textmachine/backend/internal/pipeline/bankfixpack_test.go

package pipeline

import (
	"bytes"
	"context"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"textmachine/backend/internal/config"
	"textmachine/backend/internal/miner"
	"textmachine/backend/internal/terminology"
	"textmachine/backend/internal/text"
)

// bankfixpack_test.go: the fix-pack's pipeline-level seams — the arbitration record on the stop table (§G3),
// the signature map's alias join (§G3), the auto-bank rewrite diff (row 130) and the eviction list.

// TestBankStopTableCarriesTheArbitrationRecord pins §A7's finding closed: «why does this term have THIS
// dst» has to be answerable from the artifacts. The row now carries the winning variant's ranking signals,
// whether the rendering is one the drafts ever proposed, how many CONVENTIONS they actually offered (as
// against raw forms), and the role's own confidence.
func TestBankStopTableCarriesTheArbitrationRecord(t *testing.T) {
	rec := &reqRec{}
	srv := newJSONProvider(rec, func(body string) (string, string) {
		if isTerminologyBody(body) {
			// A rendering NO draft proposed, with a confidence field — both the invented flag and the
			// confidence column have to survive the parser.
			return "方源\tГуюэ Фан Юань\t40", "stop"
		}
		// The drafts disagree on the CASE only: one convention written two ways.
		return "Фан Юань пришёл." + "\n" + bankSeparator +
			"\n方源\tФан Юань\tname\n方源\tфан юань\tname", "stop"
	})
	defer srv.Close()
	r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}))
	defer r.Close()
	stop := runToSignatureStop(t, r)

	var row *BankStopRow
	for i := range stop.Rows {
		if stop.Rows[i].Src == "方源" {
			row = &stop.Rows[i]
		}
	}
	if row == nil {
		t.Fatalf("方源 missing from the table: %+v", stop.Rows)
	}
	if row.Conf != 40 {
		t.Fatalf("the role's stated confidence must reach the review table, got %d", row.Conf)
	}
	if !row.Invented {
		t.Fatalf("a rendering no draft proposed must say so: %+v", *row)
	}
	if row.Conventions != 1 {
		t.Fatalf("two spellings of one rendering are ONE convention, got %d (%v)", row.Conventions, row.Variants)
	}
	if row.Spread < 2 {
		t.Fatalf("the owner must still see that the drafts wrote it two ways, got spread=%d", row.Spread)
	}
	raw, err := os.ReadFile(stop.TablePath)
	if err != nil {
		t.Fatal(err)
	}
	for _, want := range []string{"conventions=", "confidence=40", "INVENTED"} {
		if !strings.Contains(string(raw), want) {
			t.Fatalf("the sidecar must carry %q:\n%s", want, raw)
		}
	}
}

// TestBankStopRowCarriesTheWinningVariantsSignals pins the OTHER half of §A7's answer: not only WHAT was
// chosen but WHY. The signals are the §C2-3 factors that fired for the variant the ranking put first, and
// they were already computed and thrown away before this pack — so the failure mode is a row that looks
// complete and explains nothing.
func TestBankStopRowCarriesTheWinningVariantsSignals(t *testing.T) {
	c := terminology.Candidate{Key: "元海", Src: "元海", Type: "term", Variants: []terminology.Variant{
		{Dst: "море истинной ци", Chunks: 5, Forms: 1},
		{Dst: "первозданное море", Chunks: 1, Forms: 1},
	}}
	terminology.ScoreVariants(&c, terminology.ScoreOpts{})
	if len(c.Variants[0].Signals) == 0 {
		t.Fatalf("test premise broken: the winner must have fired at least one factor: %+v", c.Variants)
	}
	rows := bankStopRows([]terminology.Candidate{c}, map[string]string{"元海": "море истинной ци"}, terminologyResult{})
	if len(rows[0].Signals) == 0 {
		t.Fatalf("the winning variant's audit trail must reach the row: %+v", rows[0])
	}
	if strings.Join(rows[0].Signals, ",") != strings.Join(c.Variants[0].Signals, ",") {
		t.Fatalf("the row must carry the TOP variant's signals, got %v want %v", rows[0].Signals, c.Variants[0].Signals)
	}
	if !strings.Contains(renderBankStopTable(rows), "why: ") {
		t.Fatalf("and the sidecar must print them:\n%s", renderBankStopTable(rows))
	}
	// A row with no consolidation carries no confidence rather than a zero one: 0 means «the role said it was
	// not sure at all», which is the first row to review, not the same as silence.
	if rows[0].Conf >= 0 {
		t.Fatalf("an absent confidence must stay absent, got %d", rows[0].Conf)
	}
}

// TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply guards the seam between the two «this batch bought
// nothing» signals. A batch the budget never reached has an empty reply text for a completely different
// reason, and warning about an EMPTY COMPLETION there would cry wolf on every budget-bounded run — the
// budget already said what happened, one line earlier.
func TestTerminologistBudgetCutIsNotReportedAsAnEmptyReply(t *testing.T) {
	var logBuf bytes.Buffer
	rec := &reqRec{}
	srv := newJSONProvider(rec, func(body string) (string, string) {
		if isTerminologyBody(body) {
			return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop"
		}
		return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
	})
	defer srv.Close()
	// batch_runes tiny → several batches; a budget sized under one batch → the pass stops at the first.
	probe := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true, batchRunes: 400}))
	_ = runToSignatureStop(t, probe)
	if probe.lastTerminology.Batches < 2 {
		t.Fatalf("the fixture must actually batch, got %d", probe.lastTerminology.Batches)
	}
	perBatch := probe.lastTerminology.EstimateUSD / float64(probe.lastTerminology.Batches)
	probe.Close()

	r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{
		terminology: true, batchRunes: 400, budgetUSD: perBatch * 1.5,
	}))
	defer r.Close()
	r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
	_ = runToSignatureStop(t, r)
	out := logBuf.String()
	// ⚠ THE PREMISE IS ASSERTED STRUCTURALLY, not by grepping the log. It used to look for the sentence
	// «budget would be exceeded», which tied a fixture-sanity check to the wording of a message — and the
	// money-and-honesty pack replaced that message (the pass is now CUT TO WHAT FITS before the first call
	// instead of aborting part-way through it). A premise that depends on a sentence is the D39.171 trap in
	// miniature: it goes red when the sentence improves and green when the fixture stops exercising the
	// budget. `BatchesDropped` is the fact itself and cannot drift with prose. THE SUBJECT OF THIS TEST —
	// the assertion below — is untouched.
	if r.lastTerminology.BatchesDropped == 0 {
		t.Fatalf("the fixture must actually hit the budget (batches planned=%d, dropped=%d):\n%s",
			r.lastTerminology.Batches, r.lastTerminology.BatchesDropped, out)
	}
	if strings.Contains(out, "EMPTY completion") {
		t.Fatalf("a batch the budget never reached is not a paid batch that came back empty:\n%s", out)
	}
}

// TestSelfContradictionIsReportedAtTheStop is §G2 end to end: the run consolidates a part and then renders
// the compound without it, and both the log and the row say so. The canon check cannot see this — the book
// has no signed rows at all — which is precisely why 18 of the 149 contradictions on the live bank were
// invisible.
func TestSelfContradictionIsReportedAtTheStop(t *testing.T) {
	var logBuf bytes.Buffer
	rec := &reqRec{}
	srv := newJSONProvider(rec, func(body string) (string, string) {
		if isTerminologyBody(body) {
			// 花家 → «клан Хуа», and the compound built on it drops «Хуа» entirely.
			return "花家\tклан Хуа\n花家的人\tлюди клана", "stop"
		}
		return "Фан Юань пришёл." + "\n" + bankSeparator +
			"\n花家\tклан Хуа\tname\n花家的人\tлюди клана\tterm", "stop"
	})
	defer srv.Close()
	r := newVerifyRunner(t, setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true}))
	defer r.Close()
	r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
	_ = runToSignatureStop(t, r)

	if r.lastTerminology.SelfConflicts == 0 {
		t.Fatalf("the compound contradicts the part the same reply consolidated: %+v", r.lastTerminology)
	}
	if !strings.Contains(logBuf.String(), "contradict each other") {
		t.Fatalf("a run breaking its own canon must say so:\n%s", logBuf.String())
	}
	var flagged bool
	for _, row := range r.lastBankStopRows {
		if row.Src == "花家的人" && len(row.Contradicts) > 0 {
			flagged = true
		}
	}
	if !flagged {
		t.Fatalf("the contradicting row must be marked in the review table: %+v", r.lastBankStopRows)
	}
}

// TestSignatureMapJoinsAliasProposals is the acceptance finding of §G3: a rendering that reached the
// terminologist through an ALIAS of a cluster showed up in the stop table (Merge routes it to the owner)
// and was ABSENT from the owner's row in the signature map, which joined on the alias's own key. The map
// and the table then described the same bank differently, for exactly the terms the miner clustered.
func TestSignatureMapJoinsAliasProposals(t *testing.T) {
	mined := []terminology.Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子"}}}
	observed := []terminology.Observed{
		{Key: "方小子", Src: "方小子", Type: "name", Proposals: []terminology.Proposal{{Dst: "малыш Фан", Chunks: 3}}},
	}
	cands := terminology.Merge(mined, observed, text.NormalizeTargetForm)
	props := proposalsFromCandidates(cands)
	if got := props["方源"]; len(got) == 0 || got[0].Dst != "малыш Фан" {
		t.Fatalf("an alias-routed proposal must land on the OWNER's key, got %+v", props)
	}
	if props["方源"][0].Via != "方小子" {
		t.Fatalf("and it must say which surface actually proposed it, got %q", props["方源"][0].Via)
	}
	yaml, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, props)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(yaml, "малыш Фан") {
		t.Fatalf("the signature map must carry the rendering the table showed:\n%s", yaml)
	}
	if !strings.Contains(yaml, "proposed for 方小子") {
		t.Fatalf("and it must not present an alias's guess as the term's own:\n%s", yaml)
	}
	// AND it must not become the term's dst. Merge routes an alias's rendering to the cluster owner so the
	// ranking sees all the evidence; promoting it to the owner's rendering is the clustering DECISION, which
	// belongs to a signature. Here NOTHING was proposed on 方源 itself, so the row carries no dst at all.
	if strings.Contains(yaml, "dst: малыш Фан") {
		t.Fatalf("an alias's rendering became the term's dst — it reaches the wire with no model and no gate:\n%s", yaml)
	}
	// With a DIRECT proposal present, that one is the dst and the alias stays evidence in the note.
	observed = append(observed, terminology.Observed{
		Key: "方源", Src: "方源", Type: "name", Proposals: []terminology.Proposal{{Dst: "Фан Юань", Chunks: 1}},
	})
	direct := proposalsFromCandidates(terminology.Merge(mined, observed, text.NormalizeTargetForm))
	yaml2, err := miner.DeltaYAML([]miner.Term{{Src: "方源", Type: "name", Freq: 9}}, direct)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(yaml2, "dst: Фан Юань") {
		t.Fatalf("a rendering proposed on the term's own surface must be its dst:\n%s", yaml2)
	}
	if !strings.Contains(yaml2, "малыш Фан") {
		t.Fatalf("and the alias proposal must still be listed as evidence:\n%s", yaml2)
	}
}

// TestAutoBankRewriteNamesDroppedTerms is the $0 minimum of backlog row 130. The auto-bank file is rewritten
// WHOLE from a top-N-capped delta, so a term that was in the bank for twenty chapters can vanish from it
// mid-run with nothing in the artifacts saying so. The pack does not change that behaviour — accumulating is
// the owner's STOP decision, because it moves memory_version — it makes the loss visible.
func TestAutoBankRewriteNamesDroppedTerms(t *testing.T) {
	var logBuf bytes.Buffer
	r := &Runner{
		Log:  slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})),
		Book: &config.Book{BookID: "test-book", ProjectDB: filepath.Join(t.TempDir(), "project.db")},
	}
	ctx := context.Background()

	first := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "青茅山", Type: "place", Freq: 12}}
	if err := r.writeAutoBank(ctx, first, nil, nil); err != nil {
		t.Fatal(err)
	}
	if logBuf.Len() != 0 {
		t.Fatalf("the first write has nothing to diff against:\n%s", logBuf.String())
	}
	// The next run's delta no longer holds 青茅山 — the rank cap cut it, and it silently leaves the bank.
	second := []miner.Term{{Src: "方源", Type: "name", Freq: 40}, {Src: "花家", Type: "name", Freq: 9}}
	if err := r.writeAutoBank(ctx, second, nil, nil); err != nil {
		t.Fatal(err)
	}
	out := logBuf.String()
	if !strings.Contains(out, "青茅山") {
		t.Fatalf("a term dropped by the rewrite must be NAMED:\n%s", out)
	}
	if strings.Contains(out, "方源") {
		t.Fatalf("a term that is still there is not a loss:\n%s", out)
	}
	// The boundary is intact: the file is the new delta, not a merge of both runs.
	raw, err := os.ReadFile(r.autoBankPath())
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(raw), "青茅山") {
		t.Fatalf("the pack must REPORT the drop, not accumulate the file (that moves memory_version):\n%s", raw)
	}

	// And the owner's OWN two verbs are not a loss. Promoting a term into the mined-delta (it becomes a seed
	// surface) or declining it in mined-rejects removes it from the next delta ON PURPOSE — warning about
	// that fires a false alarm on the normal signature cycle and advises the owner to do what he just did.
	logBuf.Reset()
	third := []miner.Term{{Src: "方源", Type: "name", Freq: 40}}
	handled := map[string]bool{text.NormalizeSourceKey("花家"): true}
	if err := r.writeAutoBank(ctx, third, nil, handled); err != nil {
		t.Fatal(err)
	}
	if strings.Contains(logBuf.String(), "花家") {
		t.Fatalf("a term the owner promoted or declined is not a silent loss:\n%s", logBuf.String())
	}
}

// TestAutoBankDiffSurvivesTheEngineOwnRows is the row-130 warning through the path production takes: two
// real auto-mode runs, a term that leaves the delta between them, and the warning that has to name it.
//
// It has to be end-to-end, because the defect lives in the CALL SITE, not in the helper. From the second
// auto-mode run the stored glossary also holds the engine's own unsigned rows — seedGlossary re-seeds the
// auto-bank at start (seeding.go:83-93) — so passing the raw seed makes every term the engine ever proposed
// read as «the owner decided about this». The diff then empties and the warning is structurally dead in
// production while every unit test stays green. Risk 2, the self-exclusion trap, arriving at the one place
// in this file that did not filter for it.
func TestAutoBankDiffSurvivesTheEngineOwnRows(t *testing.T) {
	var logBuf bytes.Buffer
	rec := &reqRec{}
	srv := newJSONProvider(rec, func(body string) (string, string) {
		if isTerminologyBody(body) {
			return "方源\tФан Юань\n花家\tклан Хуа\n青茅山\tгора Цинмао", "stop"
		}
		return "Фан Юань пришёл к горе Цинмао." + "\n" + bankBlockForMining, "stop"
	})
	defer srv.Close()
	bookPath := setupMiningStopProject(t, srv.URL, miningStopOpts{terminology: true})

	// Run 1: the auto mode mines 花家 among others, writes the auto-bank and seeds it into the store.
	r1 := newRunner(t, bookPath)
	if _, err := r1.TranslateBook(context.Background()); err != nil {
		t.Fatal(err)
	}
	var had bool
	for _, src := range r1.autoBankSurfaces() {
		if src == "花家" {
			had = true
		}
	}
	if !had {
		t.Fatalf("test premise broken: the first run must bank 花家, got %v", r1.autoBankSurfaces())
	}
	r1.Close()

	// The book changes and 花家 stops occurring, so it leaves the delta — standing in for the rank cap doing
	// the same thing mid-book, which is the case backlog row 130 is about.
	writeFile(t, filepath.Join(filepath.Dir(bookPath), "source.txt"),
		strings.Repeat("方源来到青茅山。方源很强。青茅山很高。", 6))

	r2 := newRunner(t, bookPath)
	defer r2.Close()
	r2.Resnapshot = true // the source moved, and this fixture is about the bank diff, not the snapshot gate
	// ⚠ CONSENT IS EXPLICIT HERE SINCE THE ROW-238 FIX (money-and-honesty pack, 31.08). Editing the source
	// re-buys already-billed rows, and the re-payment consent gate now SEES that — before the fix its probe
	// read a manifest the run had just rewritten, so it stayed silent and this scenario passed by leaning
	// on a defect. Consent is orthogonal to what this test asserts; the guarantee that used to be implied
	// here — «an edited source proceeds without consent» — was FALSE and now lives, inverted and explicit,
	// in TestTheCONSENTGateSeesAnInPlaceSourceEdit (rebillsource_test.go). Scenario-only edit, sanctioned
	// by the orchestrator 31.08 on the acceptance of this pack; no assertion of this test is touched.
	r2.AcceptRebill = RebillConsent{Given: true}
	r2.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
	if _, err := r2.TranslateBook(context.Background()); err != nil {
		t.Fatal(err)
	}
	// The store now holds the engine's own unsigned rows — the state that used to swallow the warning.
	seed, err := r2.Store.GlossaryForBook("test-book")
	if err != nil {
		t.Fatal(err)
	}
	engineRows := 0
	for _, e := range seed {
		if e.Source == "mined" && e.Status != "approved" {
			engineRows++
		}
	}
	if engineRows == 0 {
		t.Fatal("test premise broken: the store must hold the engine's own unsigned rows by the second run")
	}
	out := logBuf.String()
	if !strings.Contains(out, "NOT in the one this run just wrote") || !strings.Contains(out, "花家") {
		t.Fatalf("the row-130 warning must NAME the term the rewrite dropped, engine rows in the seed and all:\n%s", out)
	}
}

// TestEvictedBankRowsAreNamed: n_evicted counted the rows the injection budget dropped and never said which,
// so «the model had no canon for this term» was an unactionable integer. The names go to the log rather than
// to a new column — a per-row detail column is a schema migration, and this repository has already paid for
// a non-idempotent one (backlog row 49а).
func TestEvictedBankRowsAreNamed(t *testing.T) {
	var logBuf bytes.Buffer
	rec := &reqRec{}
	srv := newJSONProvider(rec, draftEdit)
	defer srv.Close()
	// A budget of one token cannot fit even the first glossary line, so every matched row is evicted.
	bookPath := setupProjectOpts(t, srv.URL, projectOpts{
		source: suzukiSource, glossarySeed: suzukiSeed, glossaryTokenBudget: 1,
	})
	r := newRunner(t, bookPath)
	defer r.Close()
	r.Log = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
	if _, err := r.TranslateBook(context.Background()); err != nil {
		t.Fatal(err)
	}
	rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
	if err != nil || rs == nil {
		t.Fatalf("retrieval_state: %v %v", rs, err)
	}
	if rs.NEvicted == 0 {
		t.Fatal("test premise broken: a one-token budget must evict the matched row")
	}
	out := logBuf.String()
	if !strings.Contains(out, "鈴木") {
		t.Fatalf("the evicted rows must be NAMED, not counted:\n%s", out)
	}
}