package pipeline import ( "strings" "testing" "textmachine/backend/internal/store" "textmachine/backend/internal/terminology" ) // bankbasis_test.go: the pins of the settled basis's two pure halves. Every one of them guards a MONEY // property — a fingerprint that moves too easily re-buys a bank out of a lifetime ceiling, one that moves // too rarely serves an answer decided on evidence the book no longer holds. // basisCand is the fixture candidate: enough fields that a pin can move ONE of them and see what happens. func basisCand() terminology.Candidate { return terminology.Candidate{ Key: "元石", Src: "元石", Type: "name", Freq: 4, SinceCh: 2, Aliases: []string{"元石头"}, Related: []string{"元海"}, Evidence: []string{"surname:元"}, Origin: terminology.OriginMined, Variants: []terminology.Variant{ {Dst: "юаньши", Chunks: 2}, {Dst: "камень юань", Chunks: 1}, }, KWIC: []string{"…元石…"}, } } const basisSource = "他拿出元石,看了看。元石在手中发光。这是元海的东西。" // TestBasisFingerprintIgnoresWhatTheRoleMerelyGUESSED is the pin of what must NOT be in the hash. Draft // variants, frequency and the first chapter of appearance all reach the role's prompt, and all three move // between purchases for reasons that are not new knowledge about the term — model noise, a cap, a re-cut. // Hashing any of them turns ordinary variation into a re-purchase of the whole bank. func TestBasisFingerprintIgnoresWhatTheRoleMerelyGUESSED(t *testing.T) { base := candidateBasisFingerprint(basisCand(), basisSource, nil, 8) for _, tc := range []struct { name string mut func(c *terminology.Candidate) }{ {"the drafts proposed something else", func(c *terminology.Candidate) { c.Variants = []terminology.Variant{{Dst: "энергетический камень", Chunks: 9}} }}, {"the drafts proposed nothing at all", func(c *terminology.Candidate) { c.Variants = nil }}, {"the detector counted more occurrences", func(c *terminology.Candidate) { c.Freq = 400 }}, {"the term now first appears in another chapter", func(c *terminology.Candidate) { c.SinceCh = 77 }}, {"the KWIC subset shown to the model changed", func(c *terminology.Candidate) { c.KWIC = []string{"совсем", "другие", "окна"} }}, } { t.Run(tc.name, func(t *testing.T) { c := basisCand() tc.mut(&c) got := candidateBasisFingerprint(c, basisSource, nil, 8) if got.Whole != base.Whole { t.Fatalf("the fingerprint MOVED on %q, so this book and every other one would re-buy its bank out of a lifetime ceiling for a change that is not new knowledge about the term\n before %s\n after %s", tc.name, base.Whole[:16], got.Whole[:16]) } }) } // The control that makes those five zeros readable: the fingerprint is NOT constant. Without this the // whole test passes on a function that returns the same string for everything. moved := candidateBasisFingerprint(basisCand(), strings.Replace(basisSource, "他拿出元石", "他慢慢拿出元石", 1), nil, 8) if moved.Whole == base.Whole { t.Fatal("control: the fingerprint did not move when the SOURCE around the key changed — it is constant, and the five assertions above proved nothing") } } // TestBasisFingerprintReadsEveryOccurrenceNotTheFirstFew is the pin the whole shape exists for: the // fingerprint must not depend on how many contexts the model is shown, or on which of them are chosen. // Otherwise the stratified selection that is already on the backlog, or any tuning of kwic_per_term, zeroes // the settled set of every book on earth at once. func TestBasisFingerprintReadsEveryOccurrenceNotTheFirstFew(t *testing.T) { src := strings.Repeat("前面的话。元石在这里。", 6) full := terminology.EvidenceWindows(src, "元石", 4) if len(full) != 6 { t.Fatalf("premise broken: the fixture must hold 6 occurrences for «uncapped» to mean anything, got %d", len(full)) } // KWIC over the same text, capped — the subset the role actually sees. capped := terminology.AttachKWIC([]terminology.Candidate{{Key: "元石"}}, []terminology.Chunk{{NSource: src}}, 3, 4) if len(capped[0].KWIC) != 3 { t.Fatalf("premise broken: KWIC must be CAPPED here, or the two views are the same view; got %d", len(capped[0].KWIC)) } // The last occurrence is beyond the cap: an edit there is invisible to KWIC and must be visible here. last := strings.LastIndex(src, "元石") edited := src[:last] + "新" + src[last:] if equalStrings(full, terminology.EvidenceWindows(edited, "元石", 4)) { t.Fatal("an edit at the LAST occurrence — past the KWIC cap — left the evidence windows unchanged; the fingerprint would keep serving an answer whose evidence moved") } after := terminology.AttachKWIC([]terminology.Candidate{{Key: "元石"}}, []terminology.Chunk{{NSource: edited}}, 3, 4) if !equalStrings(capped[0].KWIC, after[0].KWIC) { t.Fatal("control: the KWIC subset ALSO changed, so this fixture does not separate «what the model is shown» from «what the book says» and the assertion above measures nothing") } } // TestBasisFingerprintNamesTheSignedRowsItDependsOn pins both halves of the canon component: a signature // that relates to the candidate MOVES the fingerprint (so the term is re-asked, paid for, and judged // against the new law), and the Canon hash moves with it — which is what lets the re-purchase state its own // cause instead of being an unexplained charge. func TestBasisFingerprintNamesTheSignedRowsItDependsOn(t *testing.T) { bare := candidateBasisFingerprint(basisCand(), basisSource, nil, 8) related := []terminology.Neighbour{{Src: "元", Dst: "юань"}} withKin := candidateBasisFingerprint(basisCand(), basisSource, related, 8) if withKin.Whole == bare.Whole { t.Fatal("signing a RELATED row left the fingerprint unmoved: the book's own law could change under a term and the engine would keep serving the rendering decided before it") } if withKin.Canon == bare.Canon { t.Fatal("the Canon half did not move, so a re-purchase caused by a signature would be reported as «the source moved» — a message lying about its own cause") } // The other half: a signature with NOTHING to do with this candidate must not move it. Otherwise every // act of signing re-buys the whole book, which is the opposite of what a signature is for. unrelated := []terminology.Neighbour{{Src: "春天", Dst: "весна"}} if got := candidateBasisFingerprint(basisCand(), basisSource, unrelated, 8); got.Whole != bare.Whole { t.Fatal("signing an UNRELATED row moved the fingerprint: every signature would re-buy every term of the book") } } // TestBasisUnitsGiveEveryCandidateAUnitOfItsOwn pins the zero-value trap that produced a false measurement // before it was found: MergeUnits returns a PARTIAL map and the engine's convention reads a missing key as // «singleton» (terminology.Batch checks `id != 0`). Indexed raw, every unit-less candidate shares unit 0 — // one unsettled row among them re-buys them all. func TestBasisUnitsGiveEveryCandidateAUnitOfItsOwn(t *testing.T) { // Three candidates with no series and no family between them: nothing should join them. cands := []terminology.Candidate{{Key: "春天"}, {Key: "石头"}, {Key: "海"}} units := bankBasisUnits(cands, terminology.SeriesParams{}, terminology.FamilyParams{}) seen := map[int]string{} for _, c := range cands { id, ok := units[c.Key] if !ok { t.Fatalf("%q got no unit id at all, so a caller indexing the map reads 0 for it", c.Key) } if id == 0 { t.Fatalf("%q got unit id 0, which the engine's own convention means «no unit» — every such candidate would share one unit and block each other", c.Key) } if other, clash := seen[id]; clash { t.Fatalf("%q and %q share unit id %d although neither is in a series or a family: one being re-asked would re-buy the other", other, c.Key, id) } seen[id] = c.Key } } // TestBasisServesOnlyADecidedRow pins the predicate's conditions one at a time, and each row of the table // is a way the engine could pay for, or wrongly skip, a term. // // ⛔ THE `auto`-WITH-A-RENDERING ROW IS THE ONE TO READ. The miner really does produce it — the raw // first-chunk banknote guess lands as a dst while the status stays `auto` — a pair four engine comments // declare impossible and backlog row 487 has open. This pack does NOT fix that; it pins the premise it // depends on, so the day 487 is fixed the settled set does not move by a step nobody priced. func TestBasisServesOnlyADecidedRow(t *testing.T) { c := basisCand() fp := candidateBasisFingerprint(c, basisSource, nil, 8) rec := bankBasisRecord{Key: c.Key, FP: fp, Answer: bankBasisAnswer{Dst: "юаньши", Type: "name"}} draft := store.GlossaryEntry{Src: c.Key, Dst: "юаньши", Status: "draft", Source: "mined"} for _, tc := range []struct { name string rec *bankBasisRecord row *store.GlossaryEntry fp bankBasisFP want basisReason }{ {"decided, evidence unchanged", &rec, &draft, fp, basisServed}, {"never decided", nil, &draft, fp, basisNew}, {"the source around it moved", &rec, &draft, bankBasisFP{Whole: "other", Canon: fp.Canon}, basisMoved}, {"a relative was signed", &rec, &draft, bankBasisFP{Whole: "other", Canon: "other"}, basisKin}, {"the bank no longer holds the row", &rec, nil, fp, basisAbsent}, {"status auto WITH a rendering (backlog row 487)", &rec, &store.GlossaryEntry{Src: c.Key, Dst: "юаньши", Status: "auto", Source: "mined"}, fp, basisUnsettled}, {"status draft with NO rendering", &rec, &store.GlossaryEntry{Src: c.Key, Dst: " ", Status: "draft", Source: "mined"}, fp, basisUnsettled}, {"signed by the owner — its own class", &rec, &store.GlossaryEntry{Src: c.Key, Dst: "Юаньши", Status: "approved"}, fp, basisSigned}, } { t.Run(tc.name, func(t *testing.T) { basis := map[string]bankBasisRecord{} if tc.rec != nil { basis[c.Key] = *tc.rec } rows := map[string]store.GlossaryEntry{} if tc.row != nil { rows[c.Key] = *tc.row } got := basisVerdictFor(c, tc.fp, basis, rows, nil, "s") if got.Reason != tc.want { t.Fatalf("verdict = %q, want %q", got.Reason, tc.want) } if tc.want == basisServed && got.Answer.Dst != "юаньши" { t.Fatalf("a served candidate must carry the stored answer, got %+v — a basis that serves an EMPTY rendering is the silent regression this mechanism exists to avoid", got.Answer) } if tc.want != basisServed && got.Answer.Dst != "" { t.Fatalf("a re-asked candidate must carry NO answer, got %q: it would be stamped as consolidated without anyone consolidating it", got.Answer.Dst) } }) } } // TestBasisServesAUnitOnlyWhole pins the family rule in both directions: one unsettled member holds its // unit, and a member of ANOTHER unit does not. func TestBasisServesAUnitOnlyWhole(t *testing.T) { kin := []terminology.Candidate{{Key: "一转"}, {Key: "三转"}, {Key: "春天"}} fps := map[string]bankBasisFP{} basis := map[string]bankBasisRecord{} rows := map[string]store.GlossaryEntry{} for _, c := range kin { fps[c.Key] = bankBasisFP{Whole: "fp-" + c.Key} rows[c.Key] = store.GlossaryEntry{Src: c.Key, Dst: "передача", Status: "draft", Source: "mined"} basis[c.Key] = bankBasisRecord{Key: c.Key, FP: fps[c.Key], Answer: bankBasisAnswer{Dst: "передача"}} } units := map[string]int{"一转": 7, "三转": 7, "春天": 9} if v := bankBasisSettle(kin, fps, basis, rows, nil, units, "s"); countServed(v) != 3 { t.Fatalf("premise broken: with everything settled all three must be served, got %d — the rows below would then pass for the wrong reason", countServed(v)) } // Now one member of unit 7 loses its record. delete(basis, "一转") v := bankBasisSettle(kin, fps, basis, rows, nil, units, "s") if v["三转"].Reason != basisUnit { t.Fatalf("its unit-mate was NOT held: verdict %q — a family would be decided in halves, which is the chimera co-batching exists to prevent", v["三转"].Reason) } if v["一转"].Reason != basisNew { t.Fatalf("the member that actually lost its record must report its OWN reason, got %q", v["一转"].Reason) } if v["春天"].Reason != basisServed { t.Fatalf("a candidate in ANOTHER unit was held too (%q): the unit rule is not keyed on the unit at all", v["春天"].Reason) } } // TestBasisWidthIsPairDataWithAnEngineDefault answers the canon's standing review question — «would a pair // that is not in the repository work without a Go edit» — by execution rather than by assertion in a // report. It also pins that the width REACHES the hash: a pair datum nothing reads is a key that looks // like a decision and is none. func TestBasisWidthIsPairDataWithAnEngineDefault(t *testing.T) { r := &Runner{} if got := r.basisWidth(); got != terminologyDefaultBasisWidth { t.Fatalf("a book with no pack must fall through to the engine default, got %d", got) } narrow := candidateBasisFingerprint(basisCand(), basisSource, nil, 2) wide := candidateBasisFingerprint(basisCand(), basisSource, nil, 30) if narrow.Whole == wide.Whole { t.Fatal("the evidence width does not reach the fingerprint: the pair datum would be a key that reads as a decision and decides nothing") } // ⛔ AND THE WIDTH IS NAMED IN THE HASH ITSELF, not merely reflected through the windows it produced. // A planted mutation that dropped the width from the hash survived the two lines above, because for an // ATTESTED surface the windows differ by width anyway. They do not for a surface the source does not // contain: its window list is empty at every width, so the two fingerprints collide and the book keeps // serving a decision taken under a rule that is no longer the rule. The population is real — a // draft-invented term is exactly a candidate with no occurrence — and the cure costs one hashed field. absent := basisCand() absent.Key, absent.Src = "这个词不在原文里", "这个词不在原文里" if got := terminology.EvidenceWindows(basisSource, absent.Key, 30); len(got) != 0 { t.Fatalf("premise broken: the fixture key must NOT occur in the source, got %d window(s)", len(got)) } if candidateBasisFingerprint(absent, basisSource, nil, 2).Whole == candidateBasisFingerprint(absent, basisSource, nil, 30).Whole { t.Fatal("two different evidence widths produced ONE fingerprint for an unattested surface: the rule that decided it is not part of what is stored, so changing the rule leaves those rows silently settled under the old one") } } // TestASignedRowDoesNotHoldItsFamilyOpen pins the owner-signature rule in both of its halves, and it exists // because the two halves are easy to confuse into one. // // ⚠ ITS PREMISE, NAMED IN THE TEST RATHER THAN ASSUMED BY IT: backlog row 447 is OPEN, so a signed surface // IS still sent to the paid role — the two-condition filter pays for banked surfaces on purpose, to keep // the dispute marks on the sheet the owner signs by. This pin does not assert that it is bought (that is // the filter's subject, not the basis's); it asserts that being bought and holding a family open have been // separated. If row 447 closes, the first half stops being true and the second half must still hold — // which is why they are two assertions and not one. func TestASignedRowDoesNotHoldItsFamilyOpen(t *testing.T) { fam := []terminology.Candidate{{Key: "古月山寨"}, {Key: "古月"}, {Key: "古月族长"}} units := map[string]int{"古月山寨": 3, "古月": 3, "古月族长": 3} fps := map[string]bankBasisFP{} basis := map[string]bankBasisRecord{} rows := map[string]store.GlossaryEntry{} for _, c := range fam { fps[c.Key] = bankBasisFP{Whole: "fp-" + c.Key} rows[c.Key] = store.GlossaryEntry{Src: c.Key, Dst: "решение движка", Status: "draft", Source: "mined"} basis[c.Key] = bankBasisRecord{Key: c.Key, FP: fps[c.Key], Answer: bankBasisAnswer{Dst: "решение движка"}} } // The owner signs ONE member, and the engine's memory of it is dropped: signing is not a decision the // basis recorded, so this is the state a real book reaches — exactly what both bought runs held. rows["古月山寨"] = store.GlossaryEntry{Src: "古月山寨", Dst: "горная крепость Гуюэ", Status: "approved"} delete(basis, "古月山寨") v := bankBasisSettle(fam, fps, basis, rows, nil, units, "s") if v["古月山寨"].Reason != basisSigned { t.Fatalf("the signed row is in class %q, want %q — the owner's signature must be its own class, not «unsettled»", v["古月山寨"].Reason, basisSigned) } // HALF ONE: the owner's word is never handed back as if the engine had decided it. if got := v["古月山寨"].Answer; got.Dst != "" { t.Fatalf("the basis handed back %q for a SIGNED row: the owner's canon would be republished as an engine decision, and the two words the bank ontology keeps apart would have merged", got.Dst) } // HALF TWO: the rest of the family is served WHOLE, which is the whole point of the exemption. for _, k := range []string{"古月", "古月族长"} { if v[k].Reason != basisServed { t.Fatalf("%q was re-asked (%q) although the only member missing from the basis is one the OWNER SIGNED — his signature would re-buy the family it belongs to", k, v[k].Reason) } if v[k].Answer.Dst != "решение движка" { t.Fatalf("%q was served with %q instead of its stored answer", k, v[k].Answer.Dst) } } // The control that keeps the exemption from being a hole: an UNSIGNED member missing from the basis // still holds the family. Without this row the test passes on a predicate with no unit rule at all. delete(basis, "古月") rows["古月"] = store.GlossaryEntry{Src: "古月", Dst: "решение движка", Status: "draft", Source: "mined"} v2 := bankBasisSettle(fam, fps, basis, rows, nil, units, "s") if v2["古月族长"].Reason != basisUnit { t.Fatalf("control: an UNSIGNED member missing from the basis left its family served (%q) — the exemption has swallowed the unit rule", v2["古月族长"].Reason) } } // TestBasisRowsToStoreRecordsOnlyDecidedRows pins the pipeline half of the same guarantee the store keeps // at its write: a term the role declined, never answered, or has no fingerprint for is NOT recorded. // // ⚠ TWO GUARDS FOR ONE PROPERTY IS DELIBERATE, and each is pinned separately rather than one standing for // both. The store's refusal is the one that cannot be forgotten by a future caller; this one keeps the // decision where the reason lives — «undecided» and «decided» are a pipeline distinction, and a row // dropped here never has to be explained to a layer that does not know what a role is. func TestBasisRowsToStoreRecordsOnlyDecidedRows(t *testing.T) { cands := []terminology.Candidate{{Key: "元石"}, {Key: "青茅山"}, {Key: "方源"}, {Key: "无指纹"}} consolidated := map[string]string{ "元石": "юаньши", "青茅山": "", // the role declined it "无指纹": "передача", // answered, but no fingerprint was computed for it // 方源 is absent entirely: no reply line covered it } fps := map[string]bankBasisFP{ "元石": {Whole: "fp1", Canon: "c1"}, "青茅山": {Whole: "fp2"}, "方源": {Whole: "fp3"}, } rows := basisRowsToStore(cands, consolidated, map[string]string{"元石": "name"}, map[string]string{"元石": "male"}, fps, nil, nil, basisShape(40, "p")) if len(rows) != 1 { t.Fatalf("recorded %d row(s), want exactly the one that was DECIDED: %+v", len(rows), rows) } got := rows[0] if got.SrcKey != "元石" || got.Dst != "юаньши" || got.Type != "name" || got.Gender != "male" { t.Fatalf("the decided row lost part of its answer: %+v", got) } // ⚠ The SHAPE, not the bare version: since D1 the stored string carries the effective evidence width, // so that a width change reports its own cause instead of «the source moved». if got.FPVersion != basisShape(40, "p") || got.FPWhole != "fp1" || got.FPCanon != "c1" { t.Fatalf("the row carries no comparable fingerprint, so a later purchase could not tell whether the evidence moved: %+v", got) } }