textmachine/backend/internal/pipeline/contractblockers_test.go

1270 lines
53 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package pipeline
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
"textmachine/backend/internal/chunk"
"textmachine/backend/internal/obs"
)
// contractblockers_test.go: the five engine surfaces the API contract is blocked on — per-phase progress
// (backlog row 99), the persisted chapter/chunk manifest (100), the machine bank-stop table (101), the
// bank export (125) and the per-run ceiling (145). Every test here runs against the mock provider or a
// pure projection: no paid call exists in this file.
// --- row 99: per-phase progress ------------------------------------------------------------------
// TestPhaseProgressSplitsTheWaves pins the defect the row was raised on: the end-to-end unit counter needs
// BOTH waves to have resolved a unit ok, so a book whose drafts are all done but whose edits are not reads
// 0/N — indistinguishable from a book where nothing has happened. The per-wave counters have to tell those
// two states apart.
func TestPhaseProgressSplitsTheWaves(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"})
ctx := context.Background()
r := newRunner(t, bookPath)
if _, err := r.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
rep, err := r.Status(ctx)
if err != nil {
t.Fatal(err)
}
if rep.Progress.Draft != (WaveCounter{Done: 2, Total: 2}) || rep.Progress.Edit != (WaveCounter{Done: 2, Total: 2}) {
t.Fatalf("a finished book must read full on BOTH waves, got %+v", rep.Progress)
}
// Now remove the edit row of chapter 1 — the exact shape of a run stopped between the waves (the bank
// stop sits there, and it is the state the row's "0% for the whole draft wave" complaint describes).
if err := r.Store.ResetChunkStages(r.Book.BookID, 1, 0, []string{"edit"}); err != nil {
t.Fatal(err)
}
rep, err = r.Status(ctx)
if err != nil {
t.Fatal(err)
}
r.Close()
if rep.Done != 1 {
t.Fatalf("the end-to-end counter must still be 1 (the row's defect), got %d", rep.Done)
}
if rep.Progress.Draft != (WaveCounter{Done: 2, Total: 2}) {
t.Fatalf("the draft wave finished BOTH units and must say so, got %+v", rep.Progress.Draft)
}
if rep.Progress.Edit != (WaveCounter{Done: 1, Total: 2}) {
t.Fatalf("the edit wave resolved one of two units, got %+v", rep.Progress.Edit)
}
// …and per chapter (contract companion K-10: a chapter tree built on the end-to-end counter shows every
// chapter at zero for the whole draft wave).
if len(rep.Chapters) != 2 {
t.Fatalf("want 2 chapter passports, got %d", len(rep.Chapters))
}
if rep.Chapters[0].Progress.Draft != (WaveCounter{Done: 1, Total: 1}) || rep.Chapters[0].Progress.Edit != (WaveCounter{Done: 0, Total: 1}) {
t.Fatalf("chapter 1 passport progress = %+v", rep.Chapters[0].Progress)
}
if rep.Chapters[1].Progress.Edit != (WaveCounter{Done: 1, Total: 1}) {
t.Fatalf("chapter 2 passport progress = %+v", rep.Chapters[1].Progress)
}
}
// TestPhaseProgressCountsAFlaggedUnitAsResolved pins the semantics the counters are documented with: a
// wave's counter reaches its own denominator on a book with flagged units, because a flagged unit is
// finished work. Counting only ok units would leave a progress bar permanently short of 100% on any book
// with one bad chunk, and the ok/flagged split is carried unconflated by Done/Flagged.
func TestPhaseProgressCountsAFlaggedUnitAsResolved(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
if strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) {
return "Извините, я не могу перевести это.", "stop" // soft refusal → the unit is flagged
}
return draftEdit(body)
})
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА", regenerate: 0})
ctx := context.Background()
r := newRunner(t, bookPath)
defer r.Close()
if _, err := r.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
rep, err := r.Status(ctx)
if err != nil {
t.Fatal(err)
}
if rep.Done != 1 || rep.Flagged != 1 {
t.Fatalf("fixture drifted: want 1 done + 1 flagged, got %+v", rep)
}
if rep.Progress.Draft != (WaveCounter{Done: 2, Total: 2}) || rep.Progress.Edit != (WaveCounter{Done: 2, Total: 2}) {
t.Fatalf("both waves resolved both units (one ok, one flagged/skipped) and must read full: %+v", rep.Progress)
}
}
// TestPhaseProgressHasNoEditWaveOnADraftOnlyPipeline: a pipeline with no editor must report an edit total
// of 0, not a total it can never reach. A consumer reads "this phase does not exist" from the zero
// denominator; "0/N forever" would be a progress bar that is broken by design.
func TestPhaseProgressHasNoEditWaveOnADraftOnlyPipeline(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ", draftOnly: true})
ctx := context.Background()
r := newRunner(t, bookPath)
defer r.Close()
if _, err := r.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
rep, err := r.Status(ctx)
if err != nil {
t.Fatal(err)
}
if rep.Progress.Draft != (WaveCounter{Done: 2, Total: 2}) {
t.Fatalf("draft counter = %+v", rep.Progress.Draft)
}
if rep.Progress.Edit != (WaveCounter{}) {
t.Fatalf("a draft-only pipeline has no edit wave; want a zero counter, got %+v", rep.Progress.Edit)
}
}
// --- row 100: the persisted manifest -------------------------------------------------------------
// TestManifestServesTheReadModelsIdentically is the load-bearing claim of the manifest: the projections
// built from it must be the SAME projections, not merely similar ones. It runs each read model twice —
// once served by the manifest, once with the manifest removed so the code re-ingests and re-cuts — and
// compares the whole structure.
func TestManifestServesTheReadModelsIdentically(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ"})
ctx := context.Background()
r := newRunner(t, bookPath)
defer r.Close()
if _, err := r.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
if r.loadManifest() == nil {
t.Fatal("a translate must leave a valid manifest behind — the read paths depend on it")
}
withManifest, err := r.Status(ctx)
if err != nil {
t.Fatal(err)
}
expWith, err := r.Export(false)
if err != nil {
t.Fatal(err)
}
if err := os.Remove(r.manifestPath()); err != nil {
t.Fatal(err)
}
if r.loadManifest() != nil {
t.Fatal("a removed manifest must not be served from anywhere")
}
withoutManifest, err := r.Status(ctx)
if err != nil {
t.Fatal(err)
}
expWithout, err := r.Export(false)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(withManifest, withoutManifest) {
t.Fatalf("the manifest path must project the SAME status report\nwith: %+v\nwithout: %+v", withManifest, withoutManifest)
}
if !reflect.DeepEqual(expWith, expWithout) {
t.Fatalf("export differs between the manifest path and the re-chunk path")
}
// `--pairs` is the mode the manifest deliberately does NOT serve (it stores structure, not text). Pin
// that it still carries a source column, so the fast path can never silently empty the FP-measure's
// src side — the one way a text-free manifest could do real damage.
pairs, err := r.Export(true)
if err != nil {
t.Fatal(err)
}
for _, ce := range pairs.Chunks {
if strings.TrimSpace(ce.Source) == "" {
t.Fatalf("--pairs must still emit the source column: %+v", ce)
}
}
}
// TestManifestIsIgnoredWhenTheSourceChanges: staleness is decided by the validity key, so a source edit
// makes the stored manifest unusable BEFORE it can hand a read model a wrong denominator. This is the
// property that lets the fast path exist at all — the accelerator has to degrade to the slow path, never
// to a stale answer.
func TestManifestIsIgnoredWhenTheSourceChanges(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"})
ctx := context.Background()
r := newRunner(t, bookPath)
defer r.Close()
if _, err := r.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
if r.loadManifest() == nil {
t.Fatal("precondition: a fresh manifest must be valid")
}
// A third chapter appears in the source. The stored manifest still says two.
if err := os.WriteFile(r.Book.SourceFile, []byte("ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ"), 0o644); err != nil {
t.Fatal(err)
}
if m := r.loadManifest(); m != nil {
t.Fatalf("a manifest written for another source must be refused, got %d chapters", m.ChaptersTotal)
}
rep, err := r.Status(ctx)
if err != nil {
t.Fatal(err)
}
if rep.TotalUnits != 3 {
t.Fatalf("status must fall back to the re-chunk and see the new chapter, got %d units", rep.TotalUnits)
}
}
// TestManifestChapterIDSurvivesAReCutAndAnEditElsewhere is the identity guarantee the row demands, and its
// declared boundary, pinned as behaviour rather than prose:
//
// - a different CUT (chunker budget) leaves every chapter id untouched — the id is a function of the
// chapter's ingested text, which the cut does not touch;
// - editing ONE chapter re-mints that chapter's id and leaves the others alone, INCLUDING chapters whose
// dense ordinal shifts — which is exactly what a number-based id cannot do.
func TestManifestChapterIDSurvivesAReCutAndAnEditElsewhere(t *testing.T) {
chapters := []string{"ПЕРВАЯ ГЛАВА", "ВТОРАЯ ГЛАВА", "ТРЕТЬЯ ГЛАВА"}
small := chunk.SegBudget{DraftBudgetOut: 8, EditCeilingOut: 16, FertCJK: 1.2, FertOther: 0.4}
large := chunk.SegBudget{DraftBudgetOut: 4000, EditCeilingOut: 8000, FertCJK: 1.2, FertOther: 0.4}
idsOf := func(chs []string, seg chunk.SegBudget) []string {
chunks, kept, _ := chunk.SplitChunksWithChapters(chs, seg, nil, nil)
if len(chunks) == 0 {
t.Fatalf("fixture produced no chunks")
}
occurrence := map[string]int{}
out := make([]string, len(kept))
for i, txt := range kept {
occurrence[txt]++
out[i] = manifestChapterID(txt, occurrence[txt])
}
return out
}
base := idsOf(chapters, small)
if got := idsOf(chapters, large); !reflect.DeepEqual(base, got) {
t.Fatalf("a re-cut must not move chapter ids:\nsmall budget: %v\nlarge budget: %v", base, got)
}
// An INSERTION before chapter 2: every later chapter's NUMBER shifts by one, every later chapter's id
// must not move.
inserted := []string{"ПЕРВАЯ ГЛАВА", "НОВАЯ ГЛАВА", "ВТОРАЯ ГЛАВА", "ТРЕТЬЯ ГЛАВА"}
got := idsOf(inserted, small)
if len(got) != 4 {
t.Fatalf("want 4 chapters, got %d", len(got))
}
if got[0] != base[0] || got[2] != base[1] || got[3] != base[2] {
t.Fatalf("an insertion must renumber, not re-identify:\nbefore: %v\nafter: %v", base, got)
}
// …and an edit INSIDE a chapter re-mints exactly that chapter.
edited := []string{"ПЕРВАЯ ГЛАВА", "ВТОРАЯ ГЛАВА исправленная", "ТРЕТЬЯ ГЛАВА"}
got = idsOf(edited, small)
if got[0] != base[0] || got[2] != base[2] {
t.Fatalf("an edit must touch only its own chapter's id:\nbefore: %v\nafter: %v", base, got)
}
if got[1] == base[1] {
t.Fatal("an edited chapter must get a new id — a bookmark into it is no longer pointing at the same text")
}
}
// TestManifestChunksReproduceTheCut pins the reconstruction: a chunk list rebuilt from the stored
// structure must have the same positions, the same edit-unit grouping and the same chapter titles as the
// split it was written from. A drift here would show up as a silently wrong denominator, which is the one
// failure mode a read model cannot detect for itself.
func TestManifestChunksReproduceTheCut(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := zhBookWithChapterHeadings(t, srv.URL)
r := newRunner(t, bookPath)
defer r.Close()
full, err := r.bookChunks()
if err != nil {
t.Fatal(err)
}
_, chapterTexts, _ := chunk.SplitChunksWithChapters(mustIngestChapters(t, r), r.segBudget(), r.chapterRule(), r.sentenceAbbrevs())
m := r.buildManifest(denseChapters{texts: chapterTexts}, full, chunk.StructureDetected, sourceFingerprint{SHA: "sha-fixture", Bytes: 42})
got := m.chunks()
if len(got) != len(full) {
t.Fatalf("manifest reconstructs %d chunks, the cut has %d", len(got), len(full))
}
// ⛔ THE FIXTURE IS HALF THE ASSERTION, and it is the half this test used to be missing. A chapter cut
// into one chunk makes the title comparison below "" == "" on both sides; a chapter whose chunks all
// land in one edit unit makes "the chapter's first chunk" and "the unit's first chunk" the same
// position. Both are asserted, so a budget change that degenerates the cut fails here rather than
// silently emptying the pins underneath it.
for _, c := range m.Chapters {
chunksIn := 0
for _, u := range c.Units {
chunksIn += u.ChunkCount
}
if chunksIn < 2 || len(c.Units) < 2 {
t.Fatalf("degenerate fixture: chapter %d is %d chunks in %d edit units, want ≥2 of each",
c.Number, chunksIn, len(c.Units))
}
if c.Heading == "" {
t.Fatalf("degenerate fixture: chapter %d carries no title, so every title comparison here is \"\" == \"\"", c.Number)
}
// And the title must be the HEADER's number, not the chapter's position — the two differ from
// chapter 2 on in this fixture, which is what lets the comparison below see a manifest that
// re-derives the title from the ordinal instead of storing what the cut produced.
if want := zhNominal(t, c.Number); c.Heading != want {
t.Fatalf("chapter %d is titled %q, want %q — the title must carry its header's own number", c.Number, c.Heading, want)
}
}
t.Logf("cut: %d chunks in %d chapters; chapter 1 = %d units; titles %q/%q",
len(full), len(m.Chapters), len(m.Chapters[0].Units), m.Chapters[0].Heading, m.Chapters[len(m.Chapters)-1].Heading)
for i := range full {
if got[i].Chapter != full[i].Chapter || got[i].ChunkIdx != full[i].ChunkIdx ||
got[i].EditUnitID != full[i].EditUnitID || got[i].Heading != full[i].Heading {
// The four compared fields, named. %+v of the whole struct printed a chapter's worth of source
// text on the cut side against the manifest's empty one — the reconstruction carries no text by
// design, so the eye-catching half of that dump was never the difference.
t.Fatalf("chunk %d differs — manifest{chapter %d, chunk %d, unit %d, title %q} vs cut{chapter %d, chunk %d, unit %d, title %q}",
i, got[i].Chapter, got[i].ChunkIdx, got[i].EditUnitID, got[i].Heading,
full[i].Chapter, full[i].ChunkIdx, full[i].EditUnitID, full[i].Heading)
}
}
// The read path's own contract: a chapter's title is on its FIRST chunk and on no other. ⚠ This pins the
// POSITION only — the value is read back from the same manifest field the reconstruction copies, so it
// is a tautology on VALUE by construction. The value is pinned by the element-wise comparison above
// (manifest against the cut) and by the fixture's non-consecutive numbers.
for _, c := range got {
want := ""
if c.ChunkIdx == 0 {
want = headingOfChapter(m, c.Chapter)
}
if c.Heading != want {
t.Fatalf("reconstructed chapter %d chunk %d (edit unit %d) heading = %q, want %q — the read path puts a chapter's title on its FIRST chunk only",
c.Chapter, c.ChunkIdx, c.EditUnitID, c.Heading, want)
}
}
if m.ChunksTotal != len(full) || m.UnitsTotal != len(r.outputUnits(full)) {
t.Fatalf("manifest totals wrong: chunks=%d units=%d (cut: %d/%d)",
m.ChunksTotal, m.UnitsTotal, len(full), len(r.outputUnits(full)))
}
// Every unit id must be unique — it is the key a reader addresses a unit by.
seen := map[string]bool{}
for _, c := range m.Chapters {
for _, u := range c.Units {
if seen[u.ID] {
t.Fatalf("duplicate unit id %q", u.ID)
}
seen[u.ID] = true
}
}
}
// TestManifestCommandProducesATreeBeforeAnyRun: the chapter tree has to exist for a book that has been cut
// but never translated (a library shows it as parsed-not-started), and every other producer of that
// structure is a paid path. $0: no provider is reachable in this test at all.
func TestManifestCommandProducesATreeBeforeAnyRun(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ"})
srv.Close() // nothing here may reach a provider
r, err := NewReadOnlyRunner(bookPath, obs.NewLogger())
if err != nil {
t.Fatal(err)
}
defer r.Close()
m, err := r.BuildAndPersistManifest()
if err != nil {
t.Fatal(err)
}
if m.ChaptersTotal != 3 || m.UnitsTotal != 3 {
t.Fatalf("want a 3-chapter tree, got chapters=%d units=%d", m.ChaptersTotal, m.UnitsTotal)
}
for i, c := range m.Chapters {
if c.ID == "" || c.Number != i+1 || len(c.Units) == 0 {
t.Fatalf("chapter %d is not addressable: %+v", i, c)
}
}
raw, err := os.ReadFile(r.ManifestPath())
if err != nil {
t.Fatal(err)
}
var stored BookManifest
if err := json.Unmarshal(raw, &stored); err != nil {
t.Fatalf("the persisted manifest must be readable JSON: %v", err)
}
if !reflect.DeepEqual(&stored, m) {
t.Fatal("the persisted document and the returned one must be the same document")
}
if rec.count() != 0 {
t.Fatalf("the manifest command must make ZERO provider calls, made %d", rec.count())
}
}
// zhNominal is the title the fixture's chapter N must carry, looked up by ORDINAL with the bound checked.
// Bounded, not indexed straight into: the whole point of the table is that the ordinal and the number are
// different quantities, so an index built from one to reach the other has to say so when it runs off the
// end instead of panicking — a panic here takes the verdict for every catalogue entry in this package.
func zhNominal(t *testing.T, chapter int) string {
t.Helper()
if chapter < 1 || chapter > len(zhFixtureNominals) {
t.Fatalf("the cut produced chapter %d and the fixture declares %d headers — the chapter counter no longer indexes them", chapter, len(zhFixtureNominals))
}
return fmt.Sprintf("Глава %d", zhFixtureNominals[chapter-1])
}
// zhFixtureNominals are the numbers the fixture's headers carry, indexed by chapter ordinal - 1. NOT
// consecutive on purpose: the ordinal (what chunk_status is keyed on) and the number in the title are
// different quantities, and a fixture where they coincide cannot tell a title rendered from the counter
// from one rendered from the header. Measured: with 一·二 a manifest that renders the title from the
// chapter number survives the whole battery.
var zhFixtureNominals = []int{1, 3}
// zhBookWithChapterHeadings sets up a zh→ru project whose source carries real chapter headers and whose
// chapters are long enough to be cut into several draft chunks in more than one edit unit. The default
// fixture cannot stand in for it: it is a ja→ru book of Russian prose split on \f, so no source grammar
// recognises a header and no pair template renders one — the title is empty on both sides of every
// comparison the heading seam makes.
func zhBookWithChapterHeadings(t *testing.T, providerURL string) string {
t.Helper()
body := strings.Repeat("\n\n"+strings.Repeat("古月方源站在洞口。", 100), 4)
bookPath := setupProjectOpts(t, providerURL, projectOpts{
source: "第一节:纵身亡魔心仍不悔" + body + "\n\n第三节逆光阴" + body,
minMaxTokens: 64,
})
packRoot, err := filepath.Abs("../../configs/langpacks")
if err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(bookPath)
if err != nil {
t.Fatal(err)
}
bookYAML := strings.Replace(string(raw), "source_lang: ja", "source_lang: zh\nlangpack_root: "+packRoot, 1)
if !strings.Contains(bookYAML, "langpack_root:") {
t.Fatal("premise broken: the fixture book no longer says `source_lang: ja`, so this helper silently produced a ja→ru book with no langpack and every heading assertion below would compare \"\" with \"\"")
}
writeFile(t, bookPath, bookYAML)
return bookPath
}
// headingOfChapter is the title the manifest stores for a chapter number.
func headingOfChapter(m *BookManifest, chapter int) string {
for _, c := range m.Chapters {
if c.Number == chapter {
return c.Heading
}
}
return ""
}
// mustIngestChapters re-reads the book's ingested chapters for a test that needs both halves of the split.
func mustIngestChapters(t *testing.T, r *Runner) []string {
t.Helper()
doc, err := chunk.IngestEncoded(r.Book.SourceFile, r.Book.Encoding, r.Book.SourceLang, r.structure)
if err != nil {
t.Fatal(err)
}
return doc.Chapters
}
// --- row 125: the bank export --------------------------------------------------------------------
// TestBankExportCarriesEveryStatusWithStableIDs pins what the artifact is for: the WHOLE bank, all three
// statuses, with an id a consumer can hold across runs. The id must NOT be glossary.id — that column is a
// fresh autoincrement on every bank replace, so an exported one would re-point under a reader between two
// reads of an unchanged term.
func TestBankExportCarriesEveryStatusWithStableIDs(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
seed := `terms:
- src: 花海
dst: Море цветов
type: place
status: approved
- src: 方源
dst: Фан Юань
type: name
status: draft
`
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ", glossarySeed: seed})
ctx := context.Background()
r := newRunner(t, bookPath)
if _, err := r.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
first := readBankExport(t, r.bankExportPath())
r.Close()
if first.Total != 2 || first.Signed != 1 {
t.Fatalf("want 2 terms / 1 approved, got %+v", first)
}
byStatus := map[string]BankExportTerm{}
for _, tm := range first.Terms {
byStatus[tm.Status] = tm
}
if byStatus["approved"].Src != "花海" || byStatus["approved"].Kind != "place" || byStatus["approved"].Origin != "seed" {
t.Fatalf("the approved row lost a field: %+v", byStatus["approved"])
}
if byStatus["draft"].Src != "方源" || byStatus["draft"].Dst != "Фан Юань" {
t.Fatalf("the draft row must be exported too — a boolean `signed` would merge it with the auto rows: %+v", byStatus["draft"])
}
// A second run REPLACES the bank (the pipeline rebuilds it from its deterministic inputs, so every
// glossary.id is new). The exported ids must not move.
r2 := newRunner(t, bookPath)
if _, err := r2.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
second := readBankExport(t, r2.bankExportPath())
r2.Close()
if !reflect.DeepEqual(first, second) {
t.Fatalf("an unchanged bank must export byte-identically across runs:\nrun1: %+v\nrun2: %+v", first, second)
}
// …and the id is the derived one, not the row's autoincrement primary key.
for _, tm := range first.Terms {
if want := bankTermID(tm.Src, tm.Sense, tm.SinceChapter, tm.UntilChapter); tm.ID != want {
t.Fatalf("term %q carries id %q, want the key-derived %q", tm.Src, tm.ID, want)
}
}
}
// TestTheRunStartReadOutAlreadyCarriesTheSeededBank pins the ORDER of two precompute steps, which nothing
// held: seedGlossary materialises the bank, exportBank publishes it, and the publication has to come
// second. Swap them and the artifact at «run-start/seeded» is the PREVIOUS run's bank — on a first run,
// empty — while every later boundary overwrites it with a correct one, so the whole battery stays green and
// the defect is visible only to a reader who opened the file at exactly the wrong moment.
//
// The run is made to die in the draft wave on purpose: this is the state the boundary exists for, named in
// bookrun.go itself — «a run that dies before the first wave still leaves a readable bank behind».
func TestTheRunStartReadOutAlreadyCarriesTheSeededBank(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
const seed = "terms:\n - src: 方源\n dst: Фан Юань\n status: approved\n decl: { invariant: true, forms: [\"Фан Юань\"] }\n"
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "方源пришёл.", glossarySeed: seed})
r := newRunner(t, bookPath)
defer r.Close()
if _, err := r.TranslateBook(context.Background()); err == nil {
t.Fatal("test premise broken: the provider answers 500, so the run must not finish")
}
exp := readBankExport(t, r.bankExportPath())
if exp.AsOf != "run-start/seeded" {
t.Fatalf("the artifact was last written at %q — this test can only see the run-start boundary if no later one overwrote it", exp.AsOf)
}
if exp.Total != 1 || exp.Signed != 1 {
t.Fatalf("the run-start read-out carries total=%d signed=%d, want the seeded row — the bank was published before it was materialised", exp.Total, exp.Signed)
}
// The proposed section is a SIGNATURE STOP's business and this boundary is not one. ⚠ Weaker than the
// same assertion after an auto-continue (miningstop_join_test.go), and worth knowing which is which:
// mining has not run at all by this boundary, so no reordering INSIDE the stop can reach here. What this
// one catches is a projector that INVENTS or carries over a row — measured: a bankexport that fills the
// section on every boundary reddens exactly this line.
if len(exp.Proposed) != 0 {
t.Fatalf("the run-start read-out carries %d proposal(s) — that section belongs to a signature stop", len(exp.Proposed))
}
}
// readBankExport loads the artifact, failing the test if it is absent or malformed.
func readBankExport(t *testing.T, path string) BankExport {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("the bank export artifact must exist after a run: %v", err)
}
var exp BankExport
if err := json.Unmarshal(raw, &exp); err != nil {
t.Fatalf("the bank export must be readable JSON: %v", err)
}
if exp.Version != bankExportVersion {
t.Fatalf("bank export version = %q, want %q", exp.Version, bankExportVersion)
}
return exp
}
// --- row 145: the per-run ceiling ----------------------------------------------------------------
// TestRunCeilingArgumentOverridesTheBookAndBookYAMLIsNotWritten: the argument caps THIS run, the book's own
// ceiling is left where it was, and the config file is untouched — the whole point of the row is that a
// caller's number must not become a permanent record in the engine's data.
func TestRunCeilingArgumentOverridesTheBookCeiling(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ", bookUSD: 10.0})
before, err := os.ReadFile(bookPath)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
r := newRunner(t, bookPath)
r.CeilingUSD = fakeCallUSD * 1.5 // enough for one call, not for the book
_, err = r.TranslateBook(ctx)
if err == nil {
t.Fatal("a run ceiling below the book's cost must stop the run")
}
if !strings.Contains(err.Error(), "ceiling reached") {
t.Fatalf("want a ceiling stop, got %v", err)
}
if !strings.Contains(err.Error(), "--ceiling-usd") {
t.Fatalf("the stop must name the ceiling actually in force, not the book's: %v", err)
}
if r.Book.Ceilings.BookUSD != 10.0 {
t.Fatalf("the book's own ceiling must be untouched, got %v", r.Book.Ceilings.BookUSD)
}
r.Close()
after, err := os.ReadFile(bookPath)
if err != nil {
t.Fatal(err)
}
if string(before) != string(after) {
t.Fatal("book.yaml must not be written by a run-scoped ceiling")
}
}
// TestRunCeilingMovesNoHashAndResumesWhenRaised is the row's cost claim and its resume claim in one run,
// because they are the same run: the ceiling is in NEITHER BriefHash NOR either wave snapshot, so a run
// stopped by it resumes at $0 under a higher one instead of re-buying what it already paid for.
func TestRunCeilingMovesNoHashAndResumesWhenRaised(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ", bookUSD: 10.0})
ctx := context.Background()
// --- the hashes do not move ---
probe := newRunner(t, bookPath)
briefBefore := probe.Book.BriefHash()
if err := probe.seedGlossary(ctx); err != nil {
t.Fatal(err)
}
draftBefore, _, err := probe.snapshotIDForWave(waveDraft)
if err != nil {
t.Fatal(err)
}
editBefore, _, err := probe.snapshotIDForWave(waveEdit)
if err != nil {
t.Fatal(err)
}
probe.CeilingUSD = 0.01
draftAfter, _, err := probe.snapshotIDForWave(waveDraft)
if err != nil {
t.Fatal(err)
}
editAfter, _, err := probe.snapshotIDForWave(waveEdit)
if err != nil {
t.Fatal(err)
}
if probe.Book.BriefHash() != briefBefore || draftAfter != draftBefore || editAfter != editBefore {
t.Fatal("a run-scoped ceiling must move NO hash — otherwise capping a run silently re-bills the book")
}
probe.Close()
// --- a stopped run resumes for free under a raised ceiling ---
r1 := newRunner(t, bookPath)
r1.CeilingUSD = fakeCallUSD * 2.5 // two calls fit, the book needs six
if _, err := r1.TranslateBook(ctx); err == nil {
t.Fatal("the tight ceiling must stop the run")
}
paidCalls := rec.count()
r1.Close()
if paidCalls == 0 {
t.Fatal("fixture drifted: the tight run must have paid for something before stopping")
}
r2 := newRunner(t, bookPath)
defer r2.Close()
r2.CeilingUSD = 10.0
res, err := r2.TranslateBook(ctx)
if err != nil {
t.Fatalf("raising the ceiling must let the run continue: %v", err)
}
if res.Flagged != 0 {
t.Fatalf("the resumed run must finish clean, got %d flagged", res.Flagged)
}
// The already-paid attempts must have been REPLAYED, not re-bought: the second run's provider calls are
// only the ones the first never made.
rep, err := r2.Status(ctx)
if err != nil {
t.Fatal(err)
}
if rep.Done != 3 {
t.Fatalf("want 3 done units, got %d", rep.Done)
}
total := rec.count()
if total != 6 {
t.Fatalf("a book of 3 units × 2 stages must cost exactly 6 provider calls across both runs, got %d", total)
}
}
// TestCeilingsAreAbsentFromEveryHashInput is the grep-proof half of the cost claim, executed rather than
// asserted in prose: the rendered snapshot payload — the thing every request_hash folds — must not mention
// a ceiling at all, under any spelling.
func TestCeilingsAreAbsentFromEveryHashInput(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{bookUSD: 7.25}))
defer r.Close()
if err := r.seedGlossary(context.Background()); err != nil {
t.Fatal(err)
}
for _, w := range []wave{waveDraft, waveEdit} {
_, payload, err := r.snapshotIDForWave(w)
if err != nil {
t.Fatal(err)
}
// Names, not substrings: `edit_ceiling_out` is the SEGMENTATION budget and legitimately carries the
// word "ceiling" — matching on it would have made this test pass for the wrong reason forever.
for _, forbidden := range []string{"ceilings", "book_usd", "day_usd", "7.25"} {
if strings.Contains(payload, forbidden) {
t.Fatalf("the snapshot payload of wave %v carries %q — a ceiling edit would re-bill the book:\n%s", w, forbidden, payload)
}
}
}
// The brief canon is the other hash a ceiling could hide in, and it is a hash rather than a document —
// so the executed proof is that moving the ceilings does not move it.
before := r.Book.BriefHash()
r.Book.Ceilings.BookUSD, r.Book.Ceilings.DayUSD = 999, 999
if after := r.Book.BriefHash(); after != before {
t.Fatalf("BriefHash moved when the ceilings did (%s → %s) — every RequestHash of the book would move with it", before, after)
}
}
// --- row 101 + 125 + 100 together: the artifacts a run leaves behind -----------------------------
// TestRunLeavesTheContractArtifacts walks the artifacts a caller of the engine is told to read and asserts
// each one exists and parses. It is the cheapest possible guard against the failure this pack is most
// exposed to: an artifact that is written on a path no test takes.
func TestRunLeavesTheContractArtifacts(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"}))
defer r.Close()
if _, err := r.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
var m BookManifest
mustParseJSON(t, r.manifestPath(), &m)
if m.ChaptersTotal != 2 {
t.Fatalf("manifest chapters = %d", m.ChaptersTotal)
}
var b BankExport
mustParseJSON(t, r.bankExportPath(), &b)
if b.BookID != r.Book.BookID {
t.Fatalf("bank export book_id = %q", b.BookID)
}
}
func mustParseJSON(t *testing.T, path string, v any) {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("artifact %s must exist: %v", filepath.Base(path), err)
}
if err := json.Unmarshal(raw, v); err != nil {
t.Fatalf("artifact %s must be readable JSON: %v", filepath.Base(path), err)
}
}
// --- the fix-pack round: defects the adversarial review found ------------------------------------
// TestManifestIsNotWrittenWhenTheSourceMovesUnderTheRead closes the review's top finding. The structure
// is derived from the bytes ingest read; the validity key was stamped from a SECOND read of the same
// path. A source rewritten between the two (a re-upload, an operator's edit — the ingest+split of a 23 MB
// book takes ~1.4 s) produced a document describing the OLD cut under the NEW hash, which then validated
// forever and could never be detected as stale — the exact state manifest.go claims cannot exist.
func TestManifestIsNotWrittenWhenTheSourceMovesUnderTheRead(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"})
r := newRunner(t, bookPath)
defer r.Close()
// The fingerprint and the cut both come from the ORIGINAL source…
before, err := sourceSHA256(r.Book.SourceFile)
if err != nil {
t.Fatal(err)
}
chunks, chapterTexts, _ := chunk.SplitChunksWithChapters(mustIngestChapters(t, r), r.segBudget(), r.chapterRule(), r.sentenceAbbrevs())
// …and the file changes before the write lands.
if err := os.WriteFile(r.Book.SourceFile, []byte("ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := r.writeManifest(before, denseChapters{texts: chapterTexts}, chunks, chunk.StructureNone); err == nil {
t.Fatal("a manifest cut from bytes that are no longer on disk must NOT be written — it would validate forever against the new hash")
}
if _, serr := os.Stat(r.manifestPath()); serr == nil {
t.Fatal("nothing may be left on disk when the source moved under the read")
}
// And the read path is unaffected: no manifest → full re-chunk → the NEW chapter count.
rep, err := r.Status(context.Background())
if err != nil {
t.Fatal(err)
}
if rep.TotalUnits != 3 {
t.Fatalf("status must see the current source, got %d units", rep.TotalUnits)
}
}
// TestManifestKeyFollowsThePipelineShape closes the review's second finding: the document's units[] is a
// projection of `outputUnits`, which groups whole chunks into edit units for an edit pipeline and emits
// one unit per chunk for a draft-only one. Nothing else in the key moves when the editor stage is
// dropped, so without the shipping wave in the key the stored unit tree would keep validating while
// describing groups the run no longer ships.
func TestManifestKeyFollowsThePipelineShape(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: strings.Repeat("Абзац для нарезки. ", 200) + "\fВТОРАЯ", minMaxTokens: 64})
r := newRunner(t, bookPath)
if _, err := r.BuildAndPersistManifest(); err != nil {
t.Fatal(err)
}
if r.loadManifest() == nil {
t.Fatal("precondition: a fresh manifest must be valid")
}
r.Close()
// Drop the editor stage — the cut is untouched, the UNIT decomposition is not.
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
raw, err := os.ReadFile(pipePath)
if err != nil {
t.Fatal(err)
}
var kept []string
for _, line := range strings.Split(string(raw), "\n") {
if !strings.Contains(line, "name: edit,") {
kept = append(kept, line)
}
}
if len(kept) == len(strings.Split(string(raw), "\n")) {
t.Fatal("fixture drifted: no editor stage line to drop")
}
if err := os.WriteFile(pipePath, []byte(strings.Join(kept, "\n")), 0o644); err != nil {
t.Fatal(err)
}
r2 := newRunner(t, bookPath)
defer r2.Close()
if r2.finalStageWave() != waveDraft {
t.Fatal("fixture drifted: the pipeline is still an edit pipeline")
}
if m := r2.loadManifest(); m != nil {
t.Fatalf("a manifest whose unit tree was built for another pipeline shape must be refused, got %d units", m.UnitsTotal)
}
}
// TestManifestWithBrokenCountersIsRefused: the key is a stored string, so a hand-edited sidecar keeps
// matching while its counters say anything at all — and those counters are read straight into a slice
// capacity and a loop bound. A negative one panics `make` inside a $0 read command.
func TestManifestWithBrokenCountersIsRefused(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"}))
defer r.Close()
if _, err := r.BuildAndPersistManifest(); err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(r.manifestPath())
if err != nil {
t.Fatal(err)
}
var m BookManifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
for name, mutate := range map[string]func(*BookManifest){
"negative chunk total": func(x *BookManifest) { x.ChunksTotal = -1 },
"absurd chunk count": func(x *BookManifest) { x.Chapters[0].Units[0].ChunkCount = 1 << 40 },
"lying chapter total": func(x *BookManifest) { x.ChaptersTotal = 99 },
"lying unit total": func(x *BookManifest) { x.UnitsTotal = 0 },
} {
broken := m
broken.Chapters = append([]ManifestChapter(nil), m.Chapters...)
broken.Chapters[0].Units = append([]ManifestUnit(nil), m.Chapters[0].Units...)
mutate(&broken)
body, err := json.Marshal(broken)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(r.manifestPath(), body, 0o644); err != nil {
t.Fatal(err)
}
if got := r.loadManifest(); got != nil {
t.Fatalf("%s: a manifest whose counters do not describe its own contents must be refused", name)
}
// …and the read model still answers, off the full re-chunk.
if _, err := r.Status(context.Background()); err != nil {
t.Fatalf("%s: status must survive a broken sidecar: %v", name, err)
}
}
}
// TestManifestUnitIDDiesWithTheCut closes the acceptance finding F1. The declared boundary is "chapter
// ids survive a re-chunk, unit ids do not" — and the first version made that statement FALSE for exactly
// the unit a reader is most likely to hold: a chapter's FIRST unit has leader index 0 under every cut, so
// «<chapter>:0» outlived any re-chunk while naming a different span. A consumer that persists unit ids
// would read "the id is still in the manifest" as "my anchor is still valid".
func TestManifestUnitIDDiesWithTheCut(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
// A chapter long enough to hold several units, so the cut visibly changes when the budget does.
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: strings.Repeat("Это довольно длинный абзац русского текста. ", 300)})
dir := filepath.Dir(bookPath)
idsUnder := func(pairYAML string) (chapterIDs, unitIDs []string) {
t.Helper()
writeFile(t, filepath.Join(dir, "pairs", "ja-ru.yaml"), pairYAML)
r := newRunner(t, bookPath)
defer r.Close()
m, err := r.BuildAndPersistManifest()
if err != nil {
t.Fatal(err)
}
for _, c := range m.Chapters {
chapterIDs = append(chapterIDs, c.ID)
for _, u := range c.Units {
unitIDs = append(unitIDs, u.ID)
}
}
return chapterIDs, unitIDs
}
coarse := "pair: ja-ru\nprompts_root: prompts\nsegmentation:\n draft_budget_out: 4000\n edit_ceiling_out: 8000\n fertility: { cjk: 1.1978, other: 0.3852 }\n"
fine := "pair: ja-ru\nprompts_root: prompts\nsegmentation:\n draft_budget_out: 200\n edit_ceiling_out: 400\n fertility: { cjk: 1.1978, other: 0.3852 }\n"
chaptersCoarse, unitsCoarse := idsUnder(coarse)
chaptersFine, unitsFine := idsUnder(fine)
if len(unitsCoarse) == len(unitsFine) {
t.Fatalf("fixture drifted: the two budgets produced the same number of units (%d) — the cut did not change", len(unitsCoarse))
}
if !reflect.DeepEqual(chaptersCoarse, chaptersFine) {
t.Fatalf("a re-cut must NOT move chapter ids:\ncoarse: %v\nfine: %v", chaptersCoarse, chaptersFine)
}
// The load-bearing assertion: NO unit id may survive the re-cut, the chapter's first one included.
held := map[string]bool{}
for _, id := range unitsCoarse {
held[id] = true
}
for _, id := range unitsFine {
if held[id] {
t.Fatalf("unit id %q survived a re-cut — a consumer holding it would anchor onto different text", id)
}
}
}
// TestBankExportIsRefreshedWhenARedriveReSeedsAndAborts closes acceptance finding F2: a redrive re-seeds
// the bank BEFORE its destructive reset, and every guard after that point can abort the command — leaving
// the store holding one bank and the exported file describing another.
func TestBankExportIsRefreshedWhenARedriveReSeedsAndAborts(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
if strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) {
return "Извините, я не могу перевести это.", "stop"
}
return draftEdit(body)
})
defer srv.Close()
seed := "terms:\n - src: 花海\n dst: Море цветов\n type: place\n status: approved\n"
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА", regenerate: 0, glossarySeed: seed})
ctx := context.Background()
r1 := newRunner(t, bookPath)
if _, err := r1.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
r1.Close()
// The seed grows between the runs, and the redrive is made to ABORT after its re-seed by pointing the
// config at a snapshot the stored rows do not carry (no --resnapshot ⇒ the drift guard refuses).
writeFile(t, filepath.Join(filepath.Dir(bookPath), "glossary-seed.yaml"),
seed+" - src: 方源\n dst: Фан Юань\n type: name\n status: approved\n")
r2 := newRunner(t, bookPath)
defer r2.Close()
if _, _, err := r2.Redrive(ctx, RedriveSelector{Chapter: -1, ChunkIdx: -1}); err == nil {
t.Fatal("test premise broken: the redrive was expected to abort on the drift guard")
}
exp := readBankExport(t, r2.bankExportPath())
if exp.Total != 2 {
t.Fatalf("the redrive re-seeded the bank to 2 terms; the export must describe the bank the engine now holds, got %d: %+v", exp.Total, exp.Terms)
}
}
// TestManifestWithAbsurdButConsistentCountersIsRefused closes acceptance finding V2-2: the upper bound in
// selfConsistent was written but nothing exercised it — a verifier removed it and the whole battery
// stayed green. The counters in TestManifestWithBrokenCountersIsRefused are all INCONSISTENT, so they die
// on the equality checks and never reach the bound; this one is internally consistent and absurd, which
// is the only shape the bound is for.
//
// The assertion deliberately stops at loadManifest: with the bound removed the reconstruction would
// allocate the claimed number of chunks, and a test that reached it would take the machine down instead
// of failing.
func TestManifestWithAbsurdButConsistentCountersIsRefused(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ"}))
defer r.Close()
if _, err := r.BuildAndPersistManifest(); err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(r.manifestPath())
if err != nil {
t.Fatal(err)
}
var m BookManifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
// One chapter, one unit, a chunk count just past the bound — and every counter agreeing with it, so
// the equality checks pass and only the bound can refuse.
absurd := manifestMaxChunks + 1
m.Chapters = m.Chapters[:1]
m.Chapters[0].Units = m.Chapters[0].Units[:1]
m.Chapters[0].Units[0].ChunkCount = absurd
m.Chapters[0].UnitsTotal, m.Chapters[0].ChunksTotal = 1, absurd
m.ChaptersTotal, m.UnitsTotal, m.ChunksTotal = 1, 1, absurd
if !m.selfConsistentExceptTheBound() {
t.Fatal("test premise broken: the mutated document must be internally consistent, so that ONLY the bound rejects it")
}
body, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(r.manifestPath(), body, 0o644); err != nil {
t.Fatal(err)
}
if got := r.loadManifest(); got != nil {
t.Fatalf("a manifest claiming %d chunks must be refused before anything allocates them", absurd)
}
}
// selfConsistentExceptTheBound re-checks only the EQUALITIES selfConsistent enforces, so the test above
// can assert its own premise ("the equalities pass, only the bound refuses") instead of assuming it.
func (m *BookManifest) selfConsistentExceptTheBound() bool {
if m.ChaptersTotal != len(m.Chapters) {
return false
}
units, chunks := 0, 0
for _, c := range m.Chapters {
if c.Number < 1 || c.UnitsTotal != len(c.Units) {
return false
}
for _, u := range c.Units {
if u.ChunkCount < 1 || u.FirstChunkIdx < 0 {
return false
}
chunks += u.ChunkCount
}
units += len(c.Units)
}
return m.UnitsTotal == units && m.ChunksTotal == chunks
}
// TestManifestOfAnOlderDocumentVersionIsRefused closes acceptance finding V2-4: a sidecar written before
// unit ids gained the cut tag is structurally valid and would keep validating, handing a reader ids in the
// old form — the very form whose defect (a chapter's first unit outliving every re-cut) the tag closes.
func TestManifestOfAnOlderDocumentVersionIsRefused(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"}))
defer r.Close()
if _, err := r.BuildAndPersistManifest(); err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(r.manifestPath())
if err != nil {
t.Fatal(err)
}
var m BookManifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
m.Version = "tm-manifest-v1"
for ci := range m.Chapters {
for ui := range m.Chapters[ci].Units {
// …in the old id form, so the fixture is a real v1 document rather than a relabelled v2 one.
m.Chapters[ci].Units[ui].ID = m.Chapters[ci].ID + ":" + strconv.Itoa(m.Chapters[ci].Units[ui].FirstChunkIdx)
}
}
body, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(r.manifestPath(), body, 0o644); err != nil {
t.Fatal(err)
}
if got := r.loadManifest(); got != nil {
t.Fatal("a v1 sidecar carries unit ids that outlive their cut; the version bump must refuse it")
}
}
// TestBankTermIDIsUnambiguous closes acceptance finding V2-1 by execution rather than by a comment. The
// first version joined the uniqueness key with U+001F and asserted the separator "cannot occur in any of
// them" — an assumption about free text the engine never validates. These two rows are DIFFERENT terms
// whose separator-joined encodings are byte-identical, so under the old scheme they shared one id and a
// consumer addressing either would hit the other.
func TestBankTermIDIsUnambiguous(t *testing.T) {
a := bankTermID("a\x1fb", "", 0, 0) // separator-join: "a␟b␟␟0␟0"
b := bankTermID("a", "b", 0, 0) // separator-join: "a␟b␟0␟0" → collides once the empty sense folds
if a == b {
t.Fatalf("two different bank keys must not share an id (%q); the encoding has to be injective, not merely conventional", a)
}
// The same class one field over, and a control character the engine has no rule against.
if bankTermID("x", "1\x1f2", 0, 0) == bankTermID("x\x1f1", "2", 0, 0) {
t.Fatal("a control character in `sense` must not be able to impersonate a different (src, sense) split")
}
// Determinism across runs is not asserted here — it is what
// TestBankExportCarriesEveryStatusWithStableIDs measures end-to-end (two runs, byte-identical export).
}
// TestCutTagFollowsTheLangpack closes acceptance finding V2-3 on the axis that CAN be varied at runtime.
// The langpack decides the heading rule, and the heading rule can decide whether a header-only chapter
// consumes a number at all — so it re-cuts the book while chunker_version and the budget stand still. A
// manifest rebuilt after such a change must not stamp the previous cut's unit ids onto different text.
//
// The other two inputs folded by the same fix — the embedded language data and the normalization version —
// are compile-time constants of the binary and cannot be varied from a test, exactly like chunkerVersion,
// which the segmentation test cannot vary either. They are asserted by construction (they are in the
// payload) and not by execution; this is stated rather than implied.
func TestCutTagFollowsTheLangpack(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
r := newRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{}))
defer r.Close()
if r.pack == nil {
t.Fatal("fixture drifted: this project must load a real langpack")
}
withPack := r.cutTag()
r.pack = nil // the same book, cut by a binary whose pair pack is gone
if withoutPack := r.cutTag(); withoutPack == withPack {
t.Fatalf("the cut tag must move with the langpack — the heading rule can change which chapters exist; got %q both ways", withPack)
}
}
// TestManifestFastPathIsActuallyTaken closes an audit finding: NOTHING pinned the fast path itself.
// TestManifestServesTheReadModelsIdentically compares "with manifest" against "without manifest", so
// deleting the fast path made both sides the slow path and the test passed trivially — the whole benefit
// of row 100 requirement (б) rested on a wall-clock measurement in a report.
//
// The observable difference is not timing but TEXT: the manifest stores structure, so the fast path
// returns chunks with no source text, and the fallback returns the full split. That is the property the
// rest of the design is built on (which read paths may use it, which must not), so it is the one to pin.
func TestManifestFastPathIsActuallyTaken(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ ГЛАВА\fВТОРАЯ ГЛАВА"}))
defer r.Close()
if _, err := r.BuildAndPersistManifest(); err != nil {
t.Fatal(err)
}
fast, withText, err := r.readModelChunks()
if err != nil {
t.Fatal(err)
}
if len(fast) == 0 {
t.Fatal("no chunks")
}
for i, c := range fast {
if c.Text != "" {
t.Fatalf("chunk %d carries source text — the manifest fast path was NOT taken, the source was re-ingested and re-cut", i)
}
}
// …and the provider beside it still hands back text, so a caller that needs it is not stranded.
full, err := withText()
if err != nil {
t.Fatal(err)
}
if len(full) != len(fast) || full[0].Text == "" {
t.Fatalf("the text provider must return the SAME cut, with text: %d vs %d chunks, first text %q", len(full), len(fast), full[0].Text)
}
// Without a manifest the slow path runs, and its chunks DO carry text — the two branches are visibly
// different, which is what makes the assertion above meaningful rather than vacuous.
if err := os.Remove(r.manifestPath()); err != nil {
t.Fatal(err)
}
slow, _, err := r.readModelChunks()
if err != nil {
t.Fatal(err)
}
if len(slow) != len(fast) || slow[0].Text == "" {
t.Fatalf("the fallback must re-cut the book WITH text: %d vs %d chunks, first text %q", len(slow), len(fast), slow[0].Text)
}
}
// TestWriteFileAtomicLeavesNoLitter pins what a single-process test CAN observe about the artifact write:
// the replacement lands whole and no temp file survives beside it. The property the helper exists for —
// a concurrent reader never seeing a half-written document — is not observable from one process and is
// declared in the report rather than claimed as tested.
func TestWriteFileAtomicLeavesNoLitter(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "artifact.json")
if err := writeFileAtomic(path, []byte("first")); err != nil {
t.Fatal(err)
}
if err := writeFileAtomic(path, []byte("second")); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(got) != "second" {
t.Fatalf("content = %q, want the replacement", got)
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
if len(entries) != 1 || entries[0].Name() != "artifact.json" {
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
t.Fatalf("a completed write must leave exactly the artifact, got %v", names)
}
// 0644, not CreateTemp's 0600: another process has to be able to read it.
fi, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if fi.Mode().Perm() != 0o644 {
t.Fatalf("mode = %v, want 0644 (a reader in another process must not depend on running as this user)", fi.Mode().Perm())
}
}