1128 lines
45 KiB
Go
1128 lines
45 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"os"
|
||
"path/filepath"
|
||
"reflect"
|
||
"strconv"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/obs"
|
||
)
|
||
|
||
// contractblockers_test.go: the five engine surfaces the API contract is blocked on — per-phase progress
|
||
// (backlog row 99), the persisted chapter/chunk manifest (100), the machine bank-stop table (101), the
|
||
// bank export (125) and the per-run ceiling (145). Every test here runs against the mock provider or a
|
||
// pure projection: no paid call exists in this file.
|
||
|
||
// --- row 99: per-phase progress ------------------------------------------------------------------
|
||
|
||
// TestPhaseProgressSplitsTheWaves pins the defect the row was raised on: the end-to-end unit counter needs
|
||
// BOTH waves to have resolved a unit ok, so a book whose drafts are all done but whose edits are not reads
|
||
// 0/N — indistinguishable from a book where nothing has happened. The per-wave counters have to tell those
|
||
// two states apart.
|
||
func TestPhaseProgressSplitsTheWaves(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
if _, err := r.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rep, err := r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.Progress.Draft != (WaveCounter{Done: 2, Total: 2}) || rep.Progress.Edit != (WaveCounter{Done: 2, Total: 2}) {
|
||
t.Fatalf("a finished book must read full on BOTH waves, got %+v", rep.Progress)
|
||
}
|
||
|
||
// Now remove the edit row of chapter 1 — the exact shape of a run stopped between the waves (the bank
|
||
// stop sits there, and it is the state the row's "0% for the whole draft wave" complaint describes).
|
||
if err := r.Store.ResetChunkStages(r.Book.BookID, 1, 0, []string{"edit"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rep, err = r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r.Close()
|
||
if rep.Done != 1 {
|
||
t.Fatalf("the end-to-end counter must still be 1 (the row's defect), got %d", rep.Done)
|
||
}
|
||
if rep.Progress.Draft != (WaveCounter{Done: 2, Total: 2}) {
|
||
t.Fatalf("the draft wave finished BOTH units and must say so, got %+v", rep.Progress.Draft)
|
||
}
|
||
if rep.Progress.Edit != (WaveCounter{Done: 1, Total: 2}) {
|
||
t.Fatalf("the edit wave resolved one of two units, got %+v", rep.Progress.Edit)
|
||
}
|
||
// …and per chapter (contract companion K-10: a chapter tree built on the end-to-end counter shows every
|
||
// chapter at zero for the whole draft wave).
|
||
if len(rep.Chapters) != 2 {
|
||
t.Fatalf("want 2 chapter passports, got %d", len(rep.Chapters))
|
||
}
|
||
if rep.Chapters[0].Progress.Draft != (WaveCounter{Done: 1, Total: 1}) || rep.Chapters[0].Progress.Edit != (WaveCounter{Done: 0, Total: 1}) {
|
||
t.Fatalf("chapter 1 passport progress = %+v", rep.Chapters[0].Progress)
|
||
}
|
||
if rep.Chapters[1].Progress.Edit != (WaveCounter{Done: 1, Total: 1}) {
|
||
t.Fatalf("chapter 2 passport progress = %+v", rep.Chapters[1].Progress)
|
||
}
|
||
}
|
||
|
||
// TestPhaseProgressCountsAFlaggedUnitAsResolved pins the semantics the counters are documented with: a
|
||
// wave's counter reaches its own denominator on a book with flagged units, because a flagged unit is
|
||
// finished work. Counting only ok units would leave a progress bar permanently short of 100% on any book
|
||
// with one bad chunk, and the ok/flagged split is carried unconflated by Done/Flagged.
|
||
func TestPhaseProgressCountsAFlaggedUnitAsResolved(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) {
|
||
return "Извините, я не могу перевести это.", "stop" // soft refusal → the unit is flagged
|
||
}
|
||
return draftEdit(body)
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА", regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rep, err := r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.Done != 1 || rep.Flagged != 1 {
|
||
t.Fatalf("fixture drifted: want 1 done + 1 flagged, got %+v", rep)
|
||
}
|
||
if rep.Progress.Draft != (WaveCounter{Done: 2, Total: 2}) || rep.Progress.Edit != (WaveCounter{Done: 2, Total: 2}) {
|
||
t.Fatalf("both waves resolved both units (one ok, one flagged/skipped) and must read full: %+v", rep.Progress)
|
||
}
|
||
}
|
||
|
||
// TestPhaseProgressHasNoEditWaveOnADraftOnlyPipeline: a pipeline with no editor must report an edit total
|
||
// of 0, not a total it can never reach. A consumer reads "this phase does not exist" from the zero
|
||
// denominator; "0/N forever" would be a progress bar that is broken by design.
|
||
func TestPhaseProgressHasNoEditWaveOnADraftOnlyPipeline(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ", draftOnly: true})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rep, err := r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.Progress.Draft != (WaveCounter{Done: 2, Total: 2}) {
|
||
t.Fatalf("draft counter = %+v", rep.Progress.Draft)
|
||
}
|
||
if rep.Progress.Edit != (WaveCounter{}) {
|
||
t.Fatalf("a draft-only pipeline has no edit wave; want a zero counter, got %+v", rep.Progress.Edit)
|
||
}
|
||
}
|
||
|
||
// --- row 100: the persisted manifest -------------------------------------------------------------
|
||
|
||
// TestManifestServesTheReadModelsIdentically is the load-bearing claim of the manifest: the projections
|
||
// built from it must be the SAME projections, not merely similar ones. It runs each read model twice —
|
||
// once served by the manifest, once with the manifest removed so the code re-ingests and re-cuts — and
|
||
// compares the whole structure.
|
||
func TestManifestServesTheReadModelsIdentically(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ"})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if r.loadManifest() == nil {
|
||
t.Fatal("a translate must leave a valid manifest behind — the read paths depend on it")
|
||
}
|
||
|
||
withManifest, err := r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
expWith, err := r.Export(false)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.Remove(r.manifestPath()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if r.loadManifest() != nil {
|
||
t.Fatal("a removed manifest must not be served from anywhere")
|
||
}
|
||
withoutManifest, err := r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
expWithout, err := r.Export(false)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !reflect.DeepEqual(withManifest, withoutManifest) {
|
||
t.Fatalf("the manifest path must project the SAME status report\nwith: %+v\nwithout: %+v", withManifest, withoutManifest)
|
||
}
|
||
if !reflect.DeepEqual(expWith, expWithout) {
|
||
t.Fatalf("export differs between the manifest path and the re-chunk path")
|
||
}
|
||
// `--pairs` is the mode the manifest deliberately does NOT serve (it stores structure, not text). Pin
|
||
// that it still carries a source column, so the fast path can never silently empty the FP-measure's
|
||
// src side — the one way a text-free manifest could do real damage.
|
||
pairs, err := r.Export(true)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, ce := range pairs.Chunks {
|
||
if strings.TrimSpace(ce.Source) == "" {
|
||
t.Fatalf("--pairs must still emit the source column: %+v", ce)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestManifestIsIgnoredWhenTheSourceChanges: staleness is decided by the validity key, so a source edit
|
||
// makes the stored manifest unusable BEFORE it can hand a read model a wrong denominator. This is the
|
||
// property that lets the fast path exist at all — the accelerator has to degrade to the slow path, never
|
||
// to a stale answer.
|
||
func TestManifestIsIgnoredWhenTheSourceChanges(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if r.loadManifest() == nil {
|
||
t.Fatal("precondition: a fresh manifest must be valid")
|
||
}
|
||
// A third chapter appears in the source. The stored manifest still says two.
|
||
if err := os.WriteFile(r.Book.SourceFile, []byte("ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if m := r.loadManifest(); m != nil {
|
||
t.Fatalf("a manifest written for another source must be refused, got %d chapters", m.ChaptersTotal)
|
||
}
|
||
rep, err := r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.TotalUnits != 3 {
|
||
t.Fatalf("status must fall back to the re-chunk and see the new chapter, got %d units", rep.TotalUnits)
|
||
}
|
||
}
|
||
|
||
// TestManifestChapterIDSurvivesAReCutAndAnEditElsewhere is the identity guarantee the row demands, and its
|
||
// declared boundary, pinned as behaviour rather than prose:
|
||
//
|
||
// - a different CUT (chunker budget) leaves every chapter id untouched — the id is a function of the
|
||
// chapter's ingested text, which the cut does not touch;
|
||
// - editing ONE chapter re-mints that chapter's id and leaves the others alone, INCLUDING chapters whose
|
||
// dense ordinal shifts — which is exactly what a number-based id cannot do.
|
||
func TestManifestChapterIDSurvivesAReCutAndAnEditElsewhere(t *testing.T) {
|
||
chapters := []string{"ПЕРВАЯ ГЛАВА", "ВТОРАЯ ГЛАВА", "ТРЕТЬЯ ГЛАВА"}
|
||
small := chunk.SegBudget{DraftBudgetOut: 8, EditCeilingOut: 16, FertCJK: 1.2, FertOther: 0.4}
|
||
large := chunk.SegBudget{DraftBudgetOut: 4000, EditCeilingOut: 8000, FertCJK: 1.2, FertOther: 0.4}
|
||
|
||
idsOf := func(chs []string, seg chunk.SegBudget) []string {
|
||
chunks, kept, _ := chunk.SplitChunksWithChapters(chs, seg, nil, nil)
|
||
if len(chunks) == 0 {
|
||
t.Fatalf("fixture produced no chunks")
|
||
}
|
||
occurrence := map[string]int{}
|
||
out := make([]string, len(kept))
|
||
for i, txt := range kept {
|
||
occurrence[txt]++
|
||
out[i] = manifestChapterID(txt, occurrence[txt])
|
||
}
|
||
return out
|
||
}
|
||
|
||
base := idsOf(chapters, small)
|
||
if got := idsOf(chapters, large); !reflect.DeepEqual(base, got) {
|
||
t.Fatalf("a re-cut must not move chapter ids:\nsmall budget: %v\nlarge budget: %v", base, got)
|
||
}
|
||
|
||
// An INSERTION before chapter 2: every later chapter's NUMBER shifts by one, every later chapter's id
|
||
// must not move.
|
||
inserted := []string{"ПЕРВАЯ ГЛАВА", "НОВАЯ ГЛАВА", "ВТОРАЯ ГЛАВА", "ТРЕТЬЯ ГЛАВА"}
|
||
got := idsOf(inserted, small)
|
||
if len(got) != 4 {
|
||
t.Fatalf("want 4 chapters, got %d", len(got))
|
||
}
|
||
if got[0] != base[0] || got[2] != base[1] || got[3] != base[2] {
|
||
t.Fatalf("an insertion must renumber, not re-identify:\nbefore: %v\nafter: %v", base, got)
|
||
}
|
||
// …and an edit INSIDE a chapter re-mints exactly that chapter.
|
||
edited := []string{"ПЕРВАЯ ГЛАВА", "ВТОРАЯ ГЛАВА исправленная", "ТРЕТЬЯ ГЛАВА"}
|
||
got = idsOf(edited, small)
|
||
if got[0] != base[0] || got[2] != base[2] {
|
||
t.Fatalf("an edit must touch only its own chapter's id:\nbefore: %v\nafter: %v", base, got)
|
||
}
|
||
if got[1] == base[1] {
|
||
t.Fatal("an edited chapter must get a new id — a bookmark into it is no longer pointing at the same text")
|
||
}
|
||
}
|
||
|
||
// TestManifestChunksReproduceTheCut pins the reconstruction: a chunk list rebuilt from the stored
|
||
// structure must have the same positions, the same edit-unit grouping and the same chapter titles as the
|
||
// split it was written from. A drift here would show up as a silently wrong denominator, which is the one
|
||
// failure mode a read model cannot detect for itself.
|
||
func TestManifestChunksReproduceTheCut(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
// A long chapter so the cut produces several chunks and more than one edit unit per chapter.
|
||
long := strings.Repeat("Это длинный абзац русского текста для нарезки. ", 60)
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: long + "\fВТОРАЯ ГЛАВА", minMaxTokens: 64})
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
|
||
full, err := r.bookChunks()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
_, chapterTexts, _ := chunk.SplitChunksWithChapters(mustIngestChapters(t, r), r.segBudget(), r.chapterRule(), r.sentenceAbbrevs())
|
||
m := r.buildManifest(denseChapters{texts: chapterTexts}, full, chunk.StructureDetected, sourceFingerprint{SHA: "sha-fixture", Bytes: 42})
|
||
got := m.chunks()
|
||
if len(got) != len(full) {
|
||
t.Fatalf("manifest reconstructs %d chunks, the cut has %d", len(got), len(full))
|
||
}
|
||
for i := range full {
|
||
if got[i].Chapter != full[i].Chapter || got[i].ChunkIdx != full[i].ChunkIdx ||
|
||
got[i].EditUnitID != full[i].EditUnitID || got[i].Heading != full[i].Heading {
|
||
t.Fatalf("chunk %d differs:\nmanifest: %+v\ncut: %+v", i, got[i], full[i])
|
||
}
|
||
}
|
||
if m.ChunksTotal != len(full) || m.UnitsTotal != len(r.outputUnits(full)) {
|
||
t.Fatalf("manifest totals wrong: chunks=%d units=%d (cut: %d/%d)",
|
||
m.ChunksTotal, m.UnitsTotal, len(full), len(r.outputUnits(full)))
|
||
}
|
||
// Every unit id must be unique — it is the key a reader addresses a unit by.
|
||
seen := map[string]bool{}
|
||
for _, c := range m.Chapters {
|
||
for _, u := range c.Units {
|
||
if seen[u.ID] {
|
||
t.Fatalf("duplicate unit id %q", u.ID)
|
||
}
|
||
seen[u.ID] = true
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestManifestCommandProducesATreeBeforeAnyRun: the chapter tree has to exist for a book that has been cut
|
||
// but never translated (a library shows it as parsed-not-started), and every other producer of that
|
||
// structure is a paid path. $0: no provider is reachable in this test at all.
|
||
func TestManifestCommandProducesATreeBeforeAnyRun(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ"})
|
||
srv.Close() // nothing here may reach a provider
|
||
|
||
r, err := NewReadOnlyRunner(bookPath, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer r.Close()
|
||
m, err := r.BuildAndPersistManifest()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if m.ChaptersTotal != 3 || m.UnitsTotal != 3 {
|
||
t.Fatalf("want a 3-chapter tree, got chapters=%d units=%d", m.ChaptersTotal, m.UnitsTotal)
|
||
}
|
||
for i, c := range m.Chapters {
|
||
if c.ID == "" || c.Number != i+1 || len(c.Units) == 0 {
|
||
t.Fatalf("chapter %d is not addressable: %+v", i, c)
|
||
}
|
||
}
|
||
raw, err := os.ReadFile(r.ManifestPath())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var stored BookManifest
|
||
if err := json.Unmarshal(raw, &stored); err != nil {
|
||
t.Fatalf("the persisted manifest must be readable JSON: %v", err)
|
||
}
|
||
if !reflect.DeepEqual(&stored, m) {
|
||
t.Fatal("the persisted document and the returned one must be the same document")
|
||
}
|
||
if rec.count() != 0 {
|
||
t.Fatalf("the manifest command must make ZERO provider calls, made %d", rec.count())
|
||
}
|
||
}
|
||
|
||
// mustIngestChapters re-reads the book's ingested chapters for a test that needs both halves of the split.
|
||
func mustIngestChapters(t *testing.T, r *Runner) []string {
|
||
t.Helper()
|
||
doc, err := chunk.IngestEncoded(r.Book.SourceFile, r.Book.Encoding, r.Book.SourceLang, r.structure)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return doc.Chapters
|
||
}
|
||
|
||
// --- row 125: the bank export --------------------------------------------------------------------
|
||
|
||
// TestBankExportCarriesEveryStatusWithStableIDs pins what the artifact is for: the WHOLE bank, all three
|
||
// statuses, with an id a consumer can hold across runs. The id must NOT be glossary.id — that column is a
|
||
// fresh autoincrement on every bank replace, so an exported one would re-point under a reader between two
|
||
// reads of an unchanged term.
|
||
func TestBankExportCarriesEveryStatusWithStableIDs(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
seed := `terms:
|
||
- src: 花海
|
||
dst: Море цветов
|
||
type: place
|
||
status: approved
|
||
- src: 方源
|
||
dst: Фан Юань
|
||
type: name
|
||
status: draft
|
||
`
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ", glossarySeed: seed})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
if _, err := r.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
first := readBankExport(t, r.bankExportPath())
|
||
r.Close()
|
||
|
||
if first.Total != 2 || first.Signed != 1 {
|
||
t.Fatalf("want 2 terms / 1 approved, got %+v", first)
|
||
}
|
||
byStatus := map[string]BankExportTerm{}
|
||
for _, tm := range first.Terms {
|
||
byStatus[tm.Status] = tm
|
||
}
|
||
if byStatus["approved"].Src != "花海" || byStatus["approved"].Kind != "place" || byStatus["approved"].Origin != "seed" {
|
||
t.Fatalf("the approved row lost a field: %+v", byStatus["approved"])
|
||
}
|
||
if byStatus["draft"].Src != "方源" || byStatus["draft"].Dst != "Фан Юань" {
|
||
t.Fatalf("the draft row must be exported too — a boolean `signed` would merge it with the auto rows: %+v", byStatus["draft"])
|
||
}
|
||
|
||
// A second run REPLACES the bank (the pipeline rebuilds it from its deterministic inputs, so every
|
||
// glossary.id is new). The exported ids must not move.
|
||
r2 := newRunner(t, bookPath)
|
||
if _, err := r2.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
second := readBankExport(t, r2.bankExportPath())
|
||
r2.Close()
|
||
if !reflect.DeepEqual(first, second) {
|
||
t.Fatalf("an unchanged bank must export byte-identically across runs:\nrun1: %+v\nrun2: %+v", first, second)
|
||
}
|
||
// …and the id is the derived one, not the row's autoincrement primary key.
|
||
for _, tm := range first.Terms {
|
||
if want := bankTermID(tm.Src, tm.Sense, tm.SinceChapter, tm.UntilChapter); tm.ID != want {
|
||
t.Fatalf("term %q carries id %q, want the key-derived %q", tm.Src, tm.ID, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// readBankExport loads the artifact, failing the test if it is absent or malformed.
|
||
func readBankExport(t *testing.T, path string) BankExport {
|
||
t.Helper()
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatalf("the bank export artifact must exist after a run: %v", err)
|
||
}
|
||
var exp BankExport
|
||
if err := json.Unmarshal(raw, &exp); err != nil {
|
||
t.Fatalf("the bank export must be readable JSON: %v", err)
|
||
}
|
||
if exp.Version != bankExportVersion {
|
||
t.Fatalf("bank export version = %q, want %q", exp.Version, bankExportVersion)
|
||
}
|
||
return exp
|
||
}
|
||
|
||
// --- row 145: the per-run ceiling ----------------------------------------------------------------
|
||
|
||
// TestRunCeilingArgumentOverridesTheBookAndBookYAMLIsNotWritten: the argument caps THIS run, the book's own
|
||
// ceiling is left where it was, and the config file is untouched — the whole point of the row is that a
|
||
// caller's number must not become a permanent record in the engine's data.
|
||
func TestRunCeilingArgumentOverridesTheBookCeiling(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ", bookUSD: 10.0})
|
||
before, err := os.ReadFile(bookPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
r.CeilingUSD = fakeCallUSD * 1.5 // enough for one call, not for the book
|
||
_, err = r.TranslateBook(ctx)
|
||
if err == nil {
|
||
t.Fatal("a run ceiling below the book's cost must stop the run")
|
||
}
|
||
if !strings.Contains(err.Error(), "ceiling reached") {
|
||
t.Fatalf("want a ceiling stop, got %v", err)
|
||
}
|
||
if !strings.Contains(err.Error(), "--ceiling-usd") {
|
||
t.Fatalf("the stop must name the ceiling actually in force, not the book's: %v", err)
|
||
}
|
||
if r.Book.Ceilings.BookUSD != 10.0 {
|
||
t.Fatalf("the book's own ceiling must be untouched, got %v", r.Book.Ceilings.BookUSD)
|
||
}
|
||
r.Close()
|
||
after, err := os.ReadFile(bookPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if string(before) != string(after) {
|
||
t.Fatal("book.yaml must not be written by a run-scoped ceiling")
|
||
}
|
||
}
|
||
|
||
// TestRunCeilingMovesNoHashAndResumesWhenRaised is the row's cost claim and its resume claim in one run,
|
||
// because they are the same run: the ceiling is in NEITHER BriefHash NOR either wave snapshot, so a run
|
||
// stopped by it resumes at $0 under a higher one instead of re-buying what it already paid for.
|
||
func TestRunCeilingMovesNoHashAndResumesWhenRaised(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ", bookUSD: 10.0})
|
||
ctx := context.Background()
|
||
|
||
// --- the hashes do not move ---
|
||
probe := newRunner(t, bookPath)
|
||
briefBefore := probe.Book.BriefHash()
|
||
if err := probe.seedGlossary(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
draftBefore, _, err := probe.snapshotIDForWave(waveDraft)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
editBefore, _, err := probe.snapshotIDForWave(waveEdit)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
probe.CeilingUSD = 0.01
|
||
draftAfter, _, err := probe.snapshotIDForWave(waveDraft)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
editAfter, _, err := probe.snapshotIDForWave(waveEdit)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if probe.Book.BriefHash() != briefBefore || draftAfter != draftBefore || editAfter != editBefore {
|
||
t.Fatal("a run-scoped ceiling must move NO hash — otherwise capping a run silently re-bills the book")
|
||
}
|
||
probe.Close()
|
||
|
||
// --- a stopped run resumes for free under a raised ceiling ---
|
||
r1 := newRunner(t, bookPath)
|
||
r1.CeilingUSD = fakeCallUSD * 2.5 // two calls fit, the book needs six
|
||
if _, err := r1.TranslateBook(ctx); err == nil {
|
||
t.Fatal("the tight ceiling must stop the run")
|
||
}
|
||
paidCalls := rec.count()
|
||
r1.Close()
|
||
if paidCalls == 0 {
|
||
t.Fatal("fixture drifted: the tight run must have paid for something before stopping")
|
||
}
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
r2.CeilingUSD = 10.0
|
||
res, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("raising the ceiling must let the run continue: %v", err)
|
||
}
|
||
if res.Flagged != 0 {
|
||
t.Fatalf("the resumed run must finish clean, got %d flagged", res.Flagged)
|
||
}
|
||
// The already-paid attempts must have been REPLAYED, not re-bought: the second run's provider calls are
|
||
// only the ones the first never made.
|
||
rep, err := r2.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.Done != 3 {
|
||
t.Fatalf("want 3 done units, got %d", rep.Done)
|
||
}
|
||
total := rec.count()
|
||
if total != 6 {
|
||
t.Fatalf("a book of 3 units × 2 stages must cost exactly 6 provider calls across both runs, got %d", total)
|
||
}
|
||
}
|
||
|
||
// TestCeilingsAreAbsentFromEveryHashInput is the grep-proof half of the cost claim, executed rather than
|
||
// asserted in prose: the rendered snapshot payload — the thing every request_hash folds — must not mention
|
||
// a ceiling at all, under any spelling.
|
||
func TestCeilingsAreAbsentFromEveryHashInput(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{bookUSD: 7.25}))
|
||
defer r.Close()
|
||
if err := r.seedGlossary(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, w := range []wave{waveDraft, waveEdit} {
|
||
_, payload, err := r.snapshotIDForWave(w)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Names, not substrings: `edit_ceiling_out` is the SEGMENTATION budget and legitimately carries the
|
||
// word "ceiling" — matching on it would have made this test pass for the wrong reason forever.
|
||
for _, forbidden := range []string{"ceilings", "book_usd", "day_usd", "7.25"} {
|
||
if strings.Contains(payload, forbidden) {
|
||
t.Fatalf("the snapshot payload of wave %v carries %q — a ceiling edit would re-bill the book:\n%s", w, forbidden, payload)
|
||
}
|
||
}
|
||
}
|
||
// The brief canon is the other hash a ceiling could hide in, and it is a hash rather than a document —
|
||
// so the executed proof is that moving the ceilings does not move it.
|
||
before := r.Book.BriefHash()
|
||
r.Book.Ceilings.BookUSD, r.Book.Ceilings.DayUSD = 999, 999
|
||
if after := r.Book.BriefHash(); after != before {
|
||
t.Fatalf("BriefHash moved when the ceilings did (%s → %s) — every RequestHash of the book would move with it", before, after)
|
||
}
|
||
}
|
||
|
||
// --- row 101 + 125 + 100 together: the artifacts a run leaves behind -----------------------------
|
||
|
||
// TestRunLeavesTheContractArtifacts walks the artifacts a caller of the engine is told to read and asserts
|
||
// each one exists and parses. It is the cheapest possible guard against the failure this pack is most
|
||
// exposed to: an artifact that is written on a path no test takes.
|
||
func TestRunLeavesTheContractArtifacts(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"}))
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var m BookManifest
|
||
mustParseJSON(t, r.manifestPath(), &m)
|
||
if m.ChaptersTotal != 2 {
|
||
t.Fatalf("manifest chapters = %d", m.ChaptersTotal)
|
||
}
|
||
var b BankExport
|
||
mustParseJSON(t, r.bankExportPath(), &b)
|
||
if b.BookID != r.Book.BookID {
|
||
t.Fatalf("bank export book_id = %q", b.BookID)
|
||
}
|
||
}
|
||
|
||
func mustParseJSON(t *testing.T, path string, v any) {
|
||
t.Helper()
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatalf("artifact %s must exist: %v", filepath.Base(path), err)
|
||
}
|
||
if err := json.Unmarshal(raw, v); err != nil {
|
||
t.Fatalf("artifact %s must be readable JSON: %v", filepath.Base(path), err)
|
||
}
|
||
}
|
||
|
||
// --- the fix-pack round: defects the adversarial review found ------------------------------------
|
||
|
||
// TestManifestIsNotWrittenWhenTheSourceMovesUnderTheRead closes the review's top finding. The structure
|
||
// is derived from the bytes ingest read; the validity key was stamped from a SECOND read of the same
|
||
// path. A source rewritten between the two (a re-upload, an operator's edit — the ingest+split of a 23 MB
|
||
// book takes ~1.4 s) produced a document describing the OLD cut under the NEW hash, which then validated
|
||
// forever and could never be detected as stale — the exact state manifest.go claims cannot exist.
|
||
func TestManifestIsNotWrittenWhenTheSourceMovesUnderTheRead(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"})
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
|
||
// The fingerprint and the cut both come from the ORIGINAL source…
|
||
before, err := sourceSHA256(r.Book.SourceFile)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
chunks, chapterTexts, _ := chunk.SplitChunksWithChapters(mustIngestChapters(t, r), r.segBudget(), r.chapterRule(), r.sentenceAbbrevs())
|
||
// …and the file changes before the write lands.
|
||
if err := os.WriteFile(r.Book.SourceFile, []byte("ПЕРВАЯ\fВТОРАЯ\fТРЕТЬЯ"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := r.writeManifest(before, denseChapters{texts: chapterTexts}, chunks, chunk.StructureNone); err == nil {
|
||
t.Fatal("a manifest cut from bytes that are no longer on disk must NOT be written — it would validate forever against the new hash")
|
||
}
|
||
if _, serr := os.Stat(r.manifestPath()); serr == nil {
|
||
t.Fatal("nothing may be left on disk when the source moved under the read")
|
||
}
|
||
// And the read path is unaffected: no manifest → full re-chunk → the NEW chapter count.
|
||
rep, err := r.Status(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.TotalUnits != 3 {
|
||
t.Fatalf("status must see the current source, got %d units", rep.TotalUnits)
|
||
}
|
||
}
|
||
|
||
// TestManifestKeyFollowsThePipelineShape closes the review's second finding: the document's units[] is a
|
||
// projection of `outputUnits`, which groups whole chunks into edit units for an edit pipeline and emits
|
||
// one unit per chunk for a draft-only one. Nothing else in the key moves when the editor stage is
|
||
// dropped, so without the shipping wave in the key the stored unit tree would keep validating while
|
||
// describing groups the run no longer ships.
|
||
func TestManifestKeyFollowsThePipelineShape(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: strings.Repeat("Абзац для нарезки. ", 200) + "\fВТОРАЯ", minMaxTokens: 64})
|
||
r := newRunner(t, bookPath)
|
||
if _, err := r.BuildAndPersistManifest(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if r.loadManifest() == nil {
|
||
t.Fatal("precondition: a fresh manifest must be valid")
|
||
}
|
||
r.Close()
|
||
|
||
// Drop the editor stage — the cut is untouched, the UNIT decomposition is not.
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var kept []string
|
||
for _, line := range strings.Split(string(raw), "\n") {
|
||
if !strings.Contains(line, "name: edit,") {
|
||
kept = append(kept, line)
|
||
}
|
||
}
|
||
if len(kept) == len(strings.Split(string(raw), "\n")) {
|
||
t.Fatal("fixture drifted: no editor stage line to drop")
|
||
}
|
||
if err := os.WriteFile(pipePath, []byte(strings.Join(kept, "\n")), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
if r2.finalStageWave() != waveDraft {
|
||
t.Fatal("fixture drifted: the pipeline is still an edit pipeline")
|
||
}
|
||
if m := r2.loadManifest(); m != nil {
|
||
t.Fatalf("a manifest whose unit tree was built for another pipeline shape must be refused, got %d units", m.UnitsTotal)
|
||
}
|
||
}
|
||
|
||
// TestManifestWithBrokenCountersIsRefused: the key is a stored string, so a hand-edited sidecar keeps
|
||
// matching while its counters say anything at all — and those counters are read straight into a slice
|
||
// capacity and a loop bound. A negative one panics `make` inside a $0 read command.
|
||
func TestManifestWithBrokenCountersIsRefused(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"}))
|
||
defer r.Close()
|
||
if _, err := r.BuildAndPersistManifest(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
raw, err := os.ReadFile(r.manifestPath())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var m BookManifest
|
||
if err := json.Unmarshal(raw, &m); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for name, mutate := range map[string]func(*BookManifest){
|
||
"negative chunk total": func(x *BookManifest) { x.ChunksTotal = -1 },
|
||
"absurd chunk count": func(x *BookManifest) { x.Chapters[0].Units[0].ChunkCount = 1 << 40 },
|
||
"lying chapter total": func(x *BookManifest) { x.ChaptersTotal = 99 },
|
||
"lying unit total": func(x *BookManifest) { x.UnitsTotal = 0 },
|
||
} {
|
||
broken := m
|
||
broken.Chapters = append([]ManifestChapter(nil), m.Chapters...)
|
||
broken.Chapters[0].Units = append([]ManifestUnit(nil), m.Chapters[0].Units...)
|
||
mutate(&broken)
|
||
body, err := json.Marshal(broken)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(r.manifestPath(), body, 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if got := r.loadManifest(); got != nil {
|
||
t.Fatalf("%s: a manifest whose counters do not describe its own contents must be refused", name)
|
||
}
|
||
// …and the read model still answers, off the full re-chunk.
|
||
if _, err := r.Status(context.Background()); err != nil {
|
||
t.Fatalf("%s: status must survive a broken sidecar: %v", name, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestManifestUnitIDDiesWithTheCut closes the acceptance finding F1. The declared boundary is "chapter
|
||
// ids survive a re-chunk, unit ids do not" — and the first version made that statement FALSE for exactly
|
||
// the unit a reader is most likely to hold: a chapter's FIRST unit has leader index 0 under every cut, so
|
||
// «<chapter>:0» outlived any re-chunk while naming a different span. A consumer that persists unit ids
|
||
// would read "the id is still in the manifest" as "my anchor is still valid".
|
||
func TestManifestUnitIDDiesWithTheCut(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
// A chapter long enough to hold several units, so the cut visibly changes when the budget does.
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: strings.Repeat("Это довольно длинный абзац русского текста. ", 300)})
|
||
dir := filepath.Dir(bookPath)
|
||
|
||
idsUnder := func(pairYAML string) (chapterIDs, unitIDs []string) {
|
||
t.Helper()
|
||
writeFile(t, filepath.Join(dir, "pairs", "ja-ru.yaml"), pairYAML)
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
m, err := r.BuildAndPersistManifest()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, c := range m.Chapters {
|
||
chapterIDs = append(chapterIDs, c.ID)
|
||
for _, u := range c.Units {
|
||
unitIDs = append(unitIDs, u.ID)
|
||
}
|
||
}
|
||
return chapterIDs, unitIDs
|
||
}
|
||
|
||
coarse := "pair: ja-ru\nprompts_root: prompts\nsegmentation:\n draft_budget_out: 4000\n edit_ceiling_out: 8000\n fertility: { cjk: 1.1978, other: 0.3852 }\n"
|
||
fine := "pair: ja-ru\nprompts_root: prompts\nsegmentation:\n draft_budget_out: 200\n edit_ceiling_out: 400\n fertility: { cjk: 1.1978, other: 0.3852 }\n"
|
||
|
||
chaptersCoarse, unitsCoarse := idsUnder(coarse)
|
||
chaptersFine, unitsFine := idsUnder(fine)
|
||
|
||
if len(unitsCoarse) == len(unitsFine) {
|
||
t.Fatalf("fixture drifted: the two budgets produced the same number of units (%d) — the cut did not change", len(unitsCoarse))
|
||
}
|
||
if !reflect.DeepEqual(chaptersCoarse, chaptersFine) {
|
||
t.Fatalf("a re-cut must NOT move chapter ids:\ncoarse: %v\nfine: %v", chaptersCoarse, chaptersFine)
|
||
}
|
||
// The load-bearing assertion: NO unit id may survive the re-cut, the chapter's first one included.
|
||
held := map[string]bool{}
|
||
for _, id := range unitsCoarse {
|
||
held[id] = true
|
||
}
|
||
for _, id := range unitsFine {
|
||
if held[id] {
|
||
t.Fatalf("unit id %q survived a re-cut — a consumer holding it would anchor onto different text", id)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestBankExportIsRefreshedWhenARedriveReSeedsAndAborts closes acceptance finding F2: a redrive re-seeds
|
||
// the bank BEFORE its destructive reset, and every guard after that point can abort the command — leaving
|
||
// the store holding one bank and the exported file describing another.
|
||
func TestBankExportIsRefreshedWhenARedriveReSeedsAndAborts(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) {
|
||
return "Извините, я не могу перевести это.", "stop"
|
||
}
|
||
return draftEdit(body)
|
||
})
|
||
defer srv.Close()
|
||
seed := "terms:\n - src: 花海\n dst: Море цветов\n type: place\n status: approved\n"
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА", regenerate: 0, glossarySeed: seed})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
|
||
// The seed grows between the runs, and the redrive is made to ABORT after its re-seed by pointing the
|
||
// config at a snapshot the stored rows do not carry (no --resnapshot ⇒ the drift guard refuses).
|
||
writeFile(t, filepath.Join(filepath.Dir(bookPath), "glossary-seed.yaml"),
|
||
seed+" - src: 方源\n dst: Фан Юань\n type: name\n status: approved\n")
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
if _, _, err := r2.Redrive(ctx, RedriveSelector{Chapter: -1, ChunkIdx: -1}); err == nil {
|
||
t.Fatal("test premise broken: the redrive was expected to abort on the drift guard")
|
||
}
|
||
exp := readBankExport(t, r2.bankExportPath())
|
||
if exp.Total != 2 {
|
||
t.Fatalf("the redrive re-seeded the bank to 2 terms; the export must describe the bank the engine now holds, got %d: %+v", exp.Total, exp.Terms)
|
||
}
|
||
}
|
||
|
||
// TestManifestWithAbsurdButConsistentCountersIsRefused closes acceptance finding V2-2: the upper bound in
|
||
// selfConsistent was written but nothing exercised it — a verifier removed it and the whole battery
|
||
// stayed green. The counters in TestManifestWithBrokenCountersIsRefused are all INCONSISTENT, so they die
|
||
// on the equality checks and never reach the bound; this one is internally consistent and absurd, which
|
||
// is the only shape the bound is for.
|
||
//
|
||
// The assertion deliberately stops at loadManifest: with the bound removed the reconstruction would
|
||
// allocate the claimed number of chunks, and a test that reached it would take the machine down instead
|
||
// of failing.
|
||
func TestManifestWithAbsurdButConsistentCountersIsRefused(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ"}))
|
||
defer r.Close()
|
||
if _, err := r.BuildAndPersistManifest(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
raw, err := os.ReadFile(r.manifestPath())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var m BookManifest
|
||
if err := json.Unmarshal(raw, &m); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// One chapter, one unit, a chunk count just past the bound — and every counter agreeing with it, so
|
||
// the equality checks pass and only the bound can refuse.
|
||
absurd := manifestMaxChunks + 1
|
||
m.Chapters = m.Chapters[:1]
|
||
m.Chapters[0].Units = m.Chapters[0].Units[:1]
|
||
m.Chapters[0].Units[0].ChunkCount = absurd
|
||
m.Chapters[0].UnitsTotal, m.Chapters[0].ChunksTotal = 1, absurd
|
||
m.ChaptersTotal, m.UnitsTotal, m.ChunksTotal = 1, 1, absurd
|
||
if !m.selfConsistentExceptTheBound() {
|
||
t.Fatal("test premise broken: the mutated document must be internally consistent, so that ONLY the bound rejects it")
|
||
}
|
||
body, err := json.Marshal(m)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(r.manifestPath(), body, 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if got := r.loadManifest(); got != nil {
|
||
t.Fatalf("a manifest claiming %d chunks must be refused before anything allocates them", absurd)
|
||
}
|
||
}
|
||
|
||
// selfConsistentExceptTheBound re-checks only the EQUALITIES selfConsistent enforces, so the test above
|
||
// can assert its own premise ("the equalities pass, only the bound refuses") instead of assuming it.
|
||
func (m *BookManifest) selfConsistentExceptTheBound() bool {
|
||
if m.ChaptersTotal != len(m.Chapters) {
|
||
return false
|
||
}
|
||
units, chunks := 0, 0
|
||
for _, c := range m.Chapters {
|
||
if c.Number < 1 || c.UnitsTotal != len(c.Units) {
|
||
return false
|
||
}
|
||
for _, u := range c.Units {
|
||
if u.ChunkCount < 1 || u.FirstChunkIdx < 0 {
|
||
return false
|
||
}
|
||
chunks += u.ChunkCount
|
||
}
|
||
units += len(c.Units)
|
||
}
|
||
return m.UnitsTotal == units && m.ChunksTotal == chunks
|
||
}
|
||
|
||
// TestManifestOfAnOlderDocumentVersionIsRefused closes acceptance finding V2-4: a sidecar written before
|
||
// unit ids gained the cut tag is structurally valid and would keep validating, handing a reader ids in the
|
||
// old form — the very form whose defect (a chapter's first unit outliving every re-cut) the tag closes.
|
||
func TestManifestOfAnOlderDocumentVersionIsRefused(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ\fВТОРАЯ"}))
|
||
defer r.Close()
|
||
if _, err := r.BuildAndPersistManifest(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
raw, err := os.ReadFile(r.manifestPath())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var m BookManifest
|
||
if err := json.Unmarshal(raw, &m); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
m.Version = "tm-manifest-v1"
|
||
for ci := range m.Chapters {
|
||
for ui := range m.Chapters[ci].Units {
|
||
// …in the old id form, so the fixture is a real v1 document rather than a relabelled v2 one.
|
||
m.Chapters[ci].Units[ui].ID = m.Chapters[ci].ID + ":" + strconv.Itoa(m.Chapters[ci].Units[ui].FirstChunkIdx)
|
||
}
|
||
}
|
||
body, err := json.Marshal(m)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(r.manifestPath(), body, 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if got := r.loadManifest(); got != nil {
|
||
t.Fatal("a v1 sidecar carries unit ids that outlive their cut; the version bump must refuse it")
|
||
}
|
||
}
|
||
|
||
// TestBankTermIDIsUnambiguous closes acceptance finding V2-1 by execution rather than by a comment. The
|
||
// first version joined the uniqueness key with U+001F and asserted the separator "cannot occur in any of
|
||
// them" — an assumption about free text the engine never validates. These two rows are DIFFERENT terms
|
||
// whose separator-joined encodings are byte-identical, so under the old scheme they shared one id and a
|
||
// consumer addressing either would hit the other.
|
||
func TestBankTermIDIsUnambiguous(t *testing.T) {
|
||
a := bankTermID("a\x1fb", "", 0, 0) // separator-join: "a␟b␟␟0␟0"
|
||
b := bankTermID("a", "b", 0, 0) // separator-join: "a␟b␟0␟0" → collides once the empty sense folds
|
||
if a == b {
|
||
t.Fatalf("two different bank keys must not share an id (%q); the encoding has to be injective, not merely conventional", a)
|
||
}
|
||
// The same class one field over, and a control character the engine has no rule against.
|
||
if bankTermID("x", "1\x1f2", 0, 0) == bankTermID("x\x1f1", "2", 0, 0) {
|
||
t.Fatal("a control character in `sense` must not be able to impersonate a different (src, sense) split")
|
||
}
|
||
// Determinism across runs is not asserted here — it is what
|
||
// TestBankExportCarriesEveryStatusWithStableIDs measures end-to-end (two runs, byte-identical export).
|
||
}
|
||
|
||
// TestCutTagFollowsTheLangpack closes acceptance finding V2-3 on the axis that CAN be varied at runtime.
|
||
// The langpack decides the heading rule, and the heading rule can decide whether a header-only chapter
|
||
// consumes a number at all — so it re-cuts the book while chunker_version and the budget stand still. A
|
||
// manifest rebuilt after such a change must not stamp the previous cut's unit ids onto different text.
|
||
//
|
||
// The other two inputs folded by the same fix — the embedded language data and the normalization version —
|
||
// are compile-time constants of the binary and cannot be varied from a test, exactly like chunkerVersion,
|
||
// which the segmentation test cannot vary either. They are asserted by construction (they are in the
|
||
// payload) and not by execution; this is stated rather than implied.
|
||
func TestCutTagFollowsTheLangpack(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
r := newRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{}))
|
||
defer r.Close()
|
||
if r.pack == nil {
|
||
t.Fatal("fixture drifted: this project must load a real langpack")
|
||
}
|
||
withPack := r.cutTag()
|
||
r.pack = nil // the same book, cut by a binary whose pair pack is gone
|
||
if withoutPack := r.cutTag(); withoutPack == withPack {
|
||
t.Fatalf("the cut tag must move with the langpack — the heading rule can change which chapters exist; got %q both ways", withPack)
|
||
}
|
||
}
|
||
|
||
// TestManifestFastPathIsActuallyTaken closes an audit finding: NOTHING pinned the fast path itself.
|
||
// TestManifestServesTheReadModelsIdentically compares "with manifest" against "without manifest", so
|
||
// deleting the fast path made both sides the slow path and the test passed trivially — the whole benefit
|
||
// of row 100 requirement (б) rested on a wall-clock measurement in a report.
|
||
//
|
||
// The observable difference is not timing but TEXT: the manifest stores structure, so the fast path
|
||
// returns chunks with no source text, and the fallback returns the full split. That is the property the
|
||
// rest of the design is built on (which read paths may use it, which must not), so it is the one to pin.
|
||
func TestManifestFastPathIsActuallyTaken(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
r := newRunner(t, setupProjectOpts(t, srv.URL, projectOpts{source: "ПЕРВАЯ ГЛАВА\fВТОРАЯ ГЛАВА"}))
|
||
defer r.Close()
|
||
if _, err := r.BuildAndPersistManifest(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
fast, withText, err := r.readModelChunks()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(fast) == 0 {
|
||
t.Fatal("no chunks")
|
||
}
|
||
for i, c := range fast {
|
||
if c.Text != "" {
|
||
t.Fatalf("chunk %d carries source text — the manifest fast path was NOT taken, the source was re-ingested and re-cut", i)
|
||
}
|
||
}
|
||
// …and the provider beside it still hands back text, so a caller that needs it is not stranded.
|
||
full, err := withText()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(full) != len(fast) || full[0].Text == "" {
|
||
t.Fatalf("the text provider must return the SAME cut, with text: %d vs %d chunks, first text %q", len(full), len(fast), full[0].Text)
|
||
}
|
||
|
||
// Without a manifest the slow path runs, and its chunks DO carry text — the two branches are visibly
|
||
// different, which is what makes the assertion above meaningful rather than vacuous.
|
||
if err := os.Remove(r.manifestPath()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
slow, _, err := r.readModelChunks()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(slow) != len(fast) || slow[0].Text == "" {
|
||
t.Fatalf("the fallback must re-cut the book WITH text: %d vs %d chunks, first text %q", len(slow), len(fast), slow[0].Text)
|
||
}
|
||
}
|
||
|
||
// TestWriteFileAtomicLeavesNoLitter pins what a single-process test CAN observe about the artifact write:
|
||
// the replacement lands whole and no temp file survives beside it. The property the helper exists for —
|
||
// a concurrent reader never seeing a half-written document — is not observable from one process and is
|
||
// declared in the report rather than claimed as tested.
|
||
func TestWriteFileAtomicLeavesNoLitter(t *testing.T) {
|
||
dir := t.TempDir()
|
||
path := filepath.Join(dir, "artifact.json")
|
||
if err := writeFileAtomic(path, []byte("first")); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := writeFileAtomic(path, []byte("second")); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
got, err := os.ReadFile(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if string(got) != "second" {
|
||
t.Fatalf("content = %q, want the replacement", got)
|
||
}
|
||
entries, err := os.ReadDir(dir)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(entries) != 1 || entries[0].Name() != "artifact.json" {
|
||
names := make([]string, 0, len(entries))
|
||
for _, e := range entries {
|
||
names = append(names, e.Name())
|
||
}
|
||
t.Fatalf("a completed write must leave exactly the artifact, got %v", names)
|
||
}
|
||
// 0644, not CreateTemp's 0600: another process has to be able to read it.
|
||
fi, err := os.Stat(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if fi.Mode().Perm() != 0o644 {
|
||
t.Fatalf("mode = %v, want 0644 (a reader in another process must not depend on running as this user)", fi.Mode().Perm())
|
||
}
|
||
}
|