package pipeline import ( "bytes" "context" "log/slog" "os" "path/filepath" "strings" "testing" "textmachine/backend/internal/config" "textmachine/backend/internal/miner" "textmachine/backend/internal/terminology" "textmachine/backend/internal/text" ) // bankfixpack_test.go: the fix-pack's pipeline-level seams — the arbitration record on the stop table (§G3), // the signature map's alias join (§G3), the auto-bank rewrite diff (row 130) and the eviction list. // TestBankStopTableCarriesTheArbitrationRecord pins §A7's finding closed: «why does this term have THIS // dst» has to be answerable from the artifacts. The row now carries the winning variant's ranking signals, // whether the rendering is one the drafts ever proposed, how many CONVENTIONS they actually offered (as // against raw forms), and the role's own confidence. func TestBankStopTableCarriesTheArbitrationRecord(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, func(body string) (string, string) { if isTerminologyBody(body) { // A rendering NO draft proposed, with a confidence field — both the invented flag and the // confidence column have to survive the parser. return "方源\tГуюэ Фан Юань\t40", "stop" } // The drafts disagree on the CASE only: one convention written two ways. return "Фан Юань пришёл." + "\n" + bankSeparator + "\n方源\tФан Юань\tname\n方源\tфан юань\tname", "stop" }) defer srv.Close() r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})) defer r.Close() stop := runToSignatureStop(t, r) var row *BankStopRow for i := range stop.Rows { if stop.Rows[i].Src == "方源" { row = &stop.Rows[i] } } if row == nil { t.Fatalf("方源 missing from the table: %+v", stop.Rows) } if row.Conf != 40 { t.Fatalf("the role's stated confidence must reach the review table, got %d", row.Conf) } if !row.Invented { t.Fatalf("a rendering no draft proposed must say so: %+v", *row) } if row.Conventions != 1 { t.Fatalf("two spellings of one rendering are ONE convention, got %d (%v)", row.Conventions, row.Variants) } if row.Spread < 2 { t.Fatalf("the owner must still see that the drafts wrote it two ways, got spread=%d", row.Spread) } raw, err := os.ReadFile(stop.TablePath) if err != nil { t.Fatal(err) } for _, want := range []string{"conventions=", "confidence=40", "INVENTED"} { if !strings.Contains(string(raw), want) { t.Fatalf("the sidecar must carry %q:\n%s", want, raw) } } } // TestBankStopRowCarriesTheWinningVariantsSignals pins the OTHER half of §A7's answer: not only WHAT was // chosen but WHY. The signals are the §C2-3 factors that fired for the variant the ranking put first, and // they were already computed and thrown away before this pack — so the failure mode is a row that looks // complete and explains nothing. func TestBankStopRowCarriesTheWinningVariantsSignals(t *testing.T) { c := terminology.Candidate{Key: "元海", Src: "元海", Type: "term", Variants: []terminology.Variant{ {Dst: "море истинной ци", Chunks: 5, Forms: 1}, {Dst: "первозданное море", Chunks: 1, Forms: 1}, }} terminology.ScoreVariants(&c, terminology.ScoreOpts{}) if len(c.Variants[0].Signals) == 0 { t.Fatalf("test premise broken: the winner must have fired at least one factor: %+v", c.Variants) } rows := bankStopRows([]terminology.Candidate{c}, map[string]string{"元海": "море истинной ци"}, terminologyResult{}) if len(rows[0].Signals) == 0 { t.Fatalf("the winning variant's audit trail must reach the row: %+v", rows[0]) } if strings.Join(rows[0].Signals, ",") != strings.Join(c.Variants[0].Signals, ",") { t.Fatalf("the row must carry the TOP variant's signals, got %v want %v", rows[0].Signals, c.Variants[0].Signals) } if !strings.Contains(renderBankStopTable(rows), "why: ") { t.Fatalf("and the sidecar must print them:\n%s", renderBankStopTable(rows)) } // A row with no consolidation carries no confidence rather than a zero one: 0 means «the role said it was // not sure at all», which is the first row to review, not the same as silence. if rows[0].Conf >= 0 { t.Fatalf("an absent confidence must stay absent, got %d", rows[0].Conf) } } // TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply guards the seam between the two «this batch bought // nothing» signals. A batch the budget never reached has an empty reply text for a completely different // reason, and warning about an EMPTY COMPLETION there would cry wolf on every budget-bounded run — the // budget already said what happened, one line earlier. func TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply(t *testing.T) { var logBuf bytes.Buffer rec := &reqRec{} srv := newJSONProvider(rec, func(body string) (string, string) { if isTerminologyBody(body) { return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop" } return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop" }) defer srv.Close() // batch_runes tiny → several batches; a budget sized under one batch → the pass stops at the first. probe := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, batchRunes: 400})) _ = runToSignatureStop(t, probe) if probe.lastTerminology.Batches < 2 { t.Fatalf("the fixture must actually batch, got %d", probe.lastTerminology.Batches) } perBatch := probe.lastTerminology.EstimateUSD / float64(probe.lastTerminology.Batches) probe.Close() r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{ terminology: true, batchRunes: 400, budgetUSD: perBatch * 1.5, })) defer r.Close() r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) _ = runToSignatureStop(t, r) out := logBuf.String() // ⚠ THE PREMISE IS ASSERTED STRUCTURALLY, not by grepping the log. It used to look for the sentence // «budget would be exceeded», which tied a fixture-sanity check to the wording of a message — and the // money-and-honesty pack replaced that message (the pass is now CUT TO WHAT FITS before the first call // instead of aborting part-way through it). A premise that depends on a sentence is the D39.171 trap in // miniature: it goes red when the sentence improves and green when the fixture stops exercising the // budget. `BatchesDropped` is the fact itself and cannot drift with prose. THE SUBJECT OF THIS TEST — // the assertion below — is untouched. if r.lastTerminology.BatchesDropped == 0 { t.Fatalf("the fixture must actually hit the budget (batches planned=%d, dropped=%d):\n%s", r.lastTerminology.Batches, r.lastTerminology.BatchesDropped, out) } if strings.Contains(out, "EMPTY completion") { t.Fatalf("a batch the budget never reached is not a paid batch that came back empty:\n%s", out) } } // TestSelfContradictionIsReportedAtTheStop is §G2 end to end: the run consolidates a part and then renders // the compound without it, and both the log and the row say so. The canon check cannot see this — the book // has no signed rows at all — which is precisely why 18 of the 149 contradictions on the live bank were // invisible. func TestSelfContradictionIsReportedAtTheStop(t *testing.T) { var logBuf bytes.Buffer rec := &reqRec{} srv := newJSONProvider(rec, func(body string) (string, string) { if isTerminologyBody(body) { // 花家 → «клан Хуа», and the compound built on it drops «Хуа» entirely. return "花家\tклан Хуа\n花家的人\tлюди клана", "stop" } return "Фан Юань пришёл." + "\n" + bankSeparator + "\n花家\tклан Хуа\tname\n花家的人\tлюди клана\tterm", "stop" }) defer srv.Close() r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})) defer r.Close() r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) _ = runToSignatureStop(t, r) if r.lastTerminology.SelfConflicts == 0 { t.Fatalf("the compound contradicts the part the same reply consolidated: %+v", r.lastTerminology) } if !strings.Contains(logBuf.String(), "contradict each other") { t.Fatalf("a run breaking its own canon must say so:\n%s", logBuf.String()) } var flagged bool for _, row := range r.lastBankStopRows { if row.Src == "花家的人" && len(row.Contradicts) > 0 { flagged = true } } if !flagged { t.Fatalf("the contradicting row must be marked in the review table: %+v", r.lastBankStopRows) } } // TestSignatureMapJoinsAliasProposals is the acceptance finding of §G3: a rendering that reached the // terminologist through an ALIAS of a cluster showed up in the stop table (Merge routes it to the owner) // and was ABSENT from the owner's row in the signature map, which joined on the alias's own key. The map // and the table then described the same bank differently, for exactly the terms the miner clustered. func TestSignatureMapJoinsAliasProposals(t *testing.T) { mined := []terminology.Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子"}}} observed := []terminology.Observed{ {Key: "方小子", Src: "方小子", Type: "name", Proposals: []terminology.Proposal{{Dst: "малыш Фан", Chunks: 3}}}, } cands := terminology.Merge(mined, observed, text.NormalizeTargetForm) props := proposalsFromCandidates(cands) if got := props["方源"]; len(got) == 0 || got[0].Dst != "малыш Фан" { t.Fatalf("an alias-routed proposal must land on the OWNER's key, got %+v", props) } if props["方源"][0].Via != "方小子" { t.Fatalf("and it must say which surface actually proposed it, got %q", props["方源"][0].Via) } yaml, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, props) if err != nil { t.Fatal(err) } if !strings.Contains(yaml, "малыш Фан") { t.Fatalf("the signature map must carry the rendering the table showed:\n%s", yaml) } if !strings.Contains(yaml, "proposed for 方小子") { t.Fatalf("and it must not present an alias's guess as the term's own:\n%s", yaml) } // AND it must not become the term's dst. Merge routes an alias's rendering to the cluster owner so the // ranking sees all the evidence; promoting it to the owner's rendering is the clustering DECISION, which // belongs to a signature. Here NOTHING was proposed on 方源 itself, so the row carries no dst at all. if strings.Contains(yaml, "dst: малыш Фан") { t.Fatalf("an alias's rendering became the term's dst — it reaches the wire with no model and no gate:\n%s", yaml) } // With a DIRECT proposal present, that one is the dst and the alias stays evidence in the note. observed = append(observed, terminology.Observed{ Key: "方源", Src: "方源", Type: "name", Proposals: []terminology.Proposal{{Dst: "Фан Юань", Chunks: 1}}, }) direct := proposalsFromCandidates(terminology.Merge(mined, observed, text.NormalizeTargetForm)) yaml2, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, direct) if err != nil { t.Fatal(err) } if !strings.Contains(yaml2, "dst: Фан Юань") { t.Fatalf("a rendering proposed on the term's own surface must be its dst:\n%s", yaml2) } if !strings.Contains(yaml2, "малыш Фан") { t.Fatalf("and the alias proposal must still be listed as evidence:\n%s", yaml2) } } // TestAutoBankRewriteNamesDroppedTerms is the $0 minimum of backlog row 130. The auto-bank file is rewritten // WHOLE from a top-N-capped delta, so a term that was in the bank for twenty chapters can vanish from it // mid-run with nothing in the artifacts saying so. The pack does not change that behaviour — accumulating is // the owner's STOP decision, because it moves memory_version — it makes the loss visible. func TestAutoBankRewriteNamesDroppedTerms(t *testing.T) { var logBuf bytes.Buffer r := &Runner{ Log: slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})), Book: &config.Book{BookID: "test-book", ProjectDB: filepath.Join(t.TempDir(), "project.db")}, } ctx := context.Background() first := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "青茅山", Type: "place", Freq: 12}} if err := r.writeAutoBank(ctx, first, nil, nil); err != nil { t.Fatal(err) } if logBuf.Len() != 0 { t.Fatalf("the first write has nothing to diff against:\n%s", logBuf.String()) } // The next run's delta no longer holds 青茅山 — the rank cap cut it, and it silently leaves the bank. second := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "花家", Type: "name", Freq: 9}} if err := r.writeAutoBank(ctx, second, nil, nil); err != nil { t.Fatal(err) } out := logBuf.String() if !strings.Contains(out, "青茅山") { t.Fatalf("a term dropped by the rewrite must be NAMED:\n%s", out) } if strings.Contains(out, "方源") { t.Fatalf("a term that is still there is not a loss:\n%s", out) } // The boundary is intact: the file is the new delta, not a merge of both runs. raw, err := os.ReadFile(r.autoBankPath()) if err != nil { t.Fatal(err) } if strings.Contains(string(raw), "青茅山") { t.Fatalf("the pack must REPORT the drop, not accumulate the file (that moves memory_version):\n%s", raw) } // And the owner's OWN two verbs are not a loss. Promoting a term into the mined-delta (it becomes a seed // surface) or declining it in mined-rejects removes it from the next delta ON PURPOSE — warning about // that fires a false alarm on the normal signature cycle and advises the owner to do what he just did. logBuf.Reset() third := []miner.Term{{Src: "方源", Type: "name", Freq: 40}} handled := map[string]bool{text.NormalizeSourceKey("花家"): true} if err := r.writeAutoBank(ctx, third, nil, handled); err != nil { t.Fatal(err) } if strings.Contains(logBuf.String(), "花家") { t.Fatalf("a term the owner promoted or declined is not a silent loss:\n%s", logBuf.String()) } } // TestAutoBankDiffSurvivesTheEngineOwnRows is the row-130 warning through the path production takes: two // real auto-mode runs, a term that leaves the delta between them, and the warning that has to name it. // // It has to be end-to-end, because the defect lives in the CALL SITE, not in the helper. From the second // auto-mode run the stored glossary also holds the engine's own unsigned rows — seedGlossary re-seeds the // auto-bank at start (seeding.go:83-93) — so passing the raw seed makes every term the engine ever proposed // read as «the owner decided about this». The diff then empties and the warning is structurally dead in // production while every unit test stays green. Risk 2, the self-exclusion trap, arriving at the one place // in this file that did not filter for it. func TestAutoBankDiffSurvivesTheEngineOwnRows(t *testing.T) { var logBuf bytes.Buffer rec := &reqRec{} srv := newJSONProvider(rec, func(body string) (string, string) { if isTerminologyBody(body) { return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop" } return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop" }) defer srv.Close() bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}) // Run 1: the auto mode mines 花家 among others, writes the auto-bank and seeds it into the store. r1 := newRunner(t, bookPath) if _, err := r1.TranslateBook(context.Background()); err != nil { t.Fatal(err) } var had bool for _, src := range r1.autoBankSurfaces() { if src == "花家" { had = true } } if !had { t.Fatalf("test premise broken: the first run must bank 花家, got %v", r1.autoBankSurfaces()) } r1.Close() // The book changes and 花家 stops occurring, so it leaves the delta — standing in for the rank cap doing // the same thing mid-book, which is the case backlog row 130 is about. writeFile(t, filepath.Join(filepath.Dir(bookPath), "source.txt"), strings.Repeat("方源来到青茅山。方源很强。青茅山很高。", 6)) r2 := newRunner(t, bookPath) defer r2.Close() r2.Resnapshot = true // the source moved, and this fixture is about the bank diff, not the snapshot gate // ⚠ CONSENT IS EXPLICIT HERE SINCE THE ROW-238 FIX (money-and-honesty pack, 31.08). Editing the source // re-buys already-billed rows, and the re-payment consent gate now SEES that — before the fix its probe // read a manifest the run had just rewritten, so it stayed silent and this scenario passed by leaning // on a defect. Consent is orthogonal to what this test asserts; the guarantee that used to be implied // here — «an edited source proceeds without consent» — was FALSE and now lives, inverted and explicit, // in TestTheCONSENTGateSeesAnInPlaceSourceEdit (rebillsource_test.go). Scenario-only edit, sanctioned // by the orchestrator 31.08 on the acceptance of this pack; no assertion of this test is touched. r2.AcceptRebill = RebillConsent{Given: true} r2.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) if _, err := r2.TranslateBook(context.Background()); err != nil { t.Fatal(err) } // The store now holds the engine's own unsigned rows — the state that used to swallow the warning. seed, err := r2.Store.GlossaryForBook("test-book") if err != nil { t.Fatal(err) } engineRows := 0 for _, e := range seed { if e.Source == "mined" && e.Status != "approved" { engineRows++ } } if engineRows == 0 { t.Fatal("test premise broken: the store must hold the engine's own unsigned rows by the second run") } out := logBuf.String() if !strings.Contains(out, "NOT in the one this run just wrote") || !strings.Contains(out, "花家") { t.Fatalf("the row-130 warning must NAME the term the rewrite dropped, engine rows in the seed and all:\n%s", out) } } // TestEvictedBankRowsAreNamed: n_evicted counted the rows the injection budget dropped and never said which, // so «the model had no canon for this term» was an unactionable integer. The names go to the log rather than // to a new column — a per-row detail column is a schema migration, and this repository has already paid for // a non-idempotent one (backlog row 49а). func TestEvictedBankRowsAreNamed(t *testing.T) { var logBuf bytes.Buffer rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() // A budget of one token cannot fit even the first glossary line, so every matched row is evicted. bookPath := setupProjectOpts(t, srv.URL, projectOpts{ source: suzukiSource, glossarySeed: suzukiSeed, glossaryTokenBudget: 1, }) r := newRunner(t, bookPath) defer r.Close() r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) if _, err := r.TranslateBook(context.Background()); err != nil { t.Fatal(err) } rs, err := r.Store.GetRetrievalState("test-book", 1, 0) if err != nil || rs == nil { t.Fatalf("retrieval_state: %v %v", rs, err) } if rs.NEvicted == 0 { t.Fatal("test premise broken: a one-token budget must evict the matched row") } out := logBuf.String() if !strings.Contains(out, "鈴木") { t.Fatalf("the evicted rows must be NAMED, not counted:\n%s", out) } }