package pipeline import ( "strings" "testing" "textmachine/backend/internal/chunk" "textmachine/backend/internal/config" ) // banknote_test.go pins the WS4 §4(д) parser invariants (byte-faithful port of exp16 banknote.py). // The 95-line corpus pin (0 parse_fail / 0 truncated over 14 saved raw outputs) is the domain of the // $0 reference eval/design11/ws4_banknote_verify.py; these unit tests cover the parser CONTRACT. // attestedIn is the test double for the src rule: the surfaces the "book" contains. The production // predicate is bankSrcAttested over bankSourceIndex; both answer the same question over the same // normalization, and TestBankSourceIndexParity pins that they agree. func attestedIn(source string) func(string) bool { r := &Runner{bankSrc: newBankSourceIndex([]chunk.Chunk{{Text: source}})} return r.bankSrcAttested("") } // srcAll is the corpus every legacy parser test above was written against. const srcAll = "方源\n蛊师\n青茅山\n古月\n蛊\n龙公\nRoseanne\n" func TestSplitBanknoteNoSeparatorIsAllClean(t *testing.T) { out := "Судзуки шёл по коридору. \n" clean, block, _ := splitBanknote(out) if clean != "Судзуки шёл по коридору." || block != "" { t.Fatalf("no-SEP: clean=%q block=%q, want the whole right-trimmed output and empty block", clean, block) } } func TestSplitBanknoteSlicesBlock(t *testing.T) { out := "Перевод... последнее предложение.\n\n" + bankSeparator + "\n方源\tФан Юань\tname\n蛊师\tгу-мастер\ttitle" clean, block, _ := splitBanknote(out) if !strings.HasSuffix(clean, "последнее предложение.") { t.Fatalf("clean tail wrong: %q", clean) } if strings.Contains(clean, bankSeparator) || strings.Contains(clean, "方源") { t.Fatalf("the banknote block leaked into the clean translation: %q", clean) } ents, flags := parseBanknote(block, false, attestedIn(srcAll)) if len(ents) != 2 || flags.ParseFail || flags.Truncated || flags.NLines != 2 { t.Fatalf("parse = %+v flags=%+v, want 2 clean entries", ents, flags) } if ents[0].Src != "方源" || ents[0].Dst != "Фан Юань" || ents[0].Type != "name" { t.Fatalf("entry[0] = %+v", ents[0]) } if ents[1].Type != "title" { t.Fatalf("entry[1] type = %q, want title", ents[1].Type) } } func TestParseBanknoteTolerantFieldSplit(t *testing.T) { // tab, ≥2 spaces, and pipe are all accepted delimiters (a model that emits spaces instead of a // real TAB still parses). type falls back to "term" when absent or unknown. block := "方源\tФан Юань\tname\n蛊师 гу-мастер title\n青茅山 | гора Цинмао | place\n古月\tГу Юэ" ents, flags := parseBanknote(block, false, attestedIn(srcAll)) if flags.ParseFail { t.Fatalf("tolerant split should not fail: %+v", flags) } if len(ents) != 4 { t.Fatalf("want 4 entries across tab/space/pipe delimiters, got %d: %+v", len(ents), ents) } if ents[3].Type != "term" { // no type field → default term t.Fatalf("missing type must default to term, got %q", ents[3].Type) } } // TestParseBanknoteSrcMustBeAttested replaces the old "src must contain a Han ideograph" pin (backlog 19, // D39.53 default). Two things changed in ONE rule: a non-ideographic src is no longer malformed by // construction (the channel was DEAD on every non-CJK source — 5/6 of the bank's candidates), and a src // the book does not contain is now malformed even when it looks Chinese (the old rule accepted invented // surfaces, which then reached the owner's sign map). func TestParseBanknoteSrcMustBeAttested(t *testing.T) { src := "方源 шёл по тропе. Roseanne ждала у Silversaint, рядом стоял San Michon.\nリン сказала: идём.\n" block := strings.Join([]string{ "方源\tФан Юань\tname", "Roseanne\tРозанна\tname", // latin src, attested → ACCEPTED (was: parse_fail) "Silversaint\tСеребряный святой\ttitle", "San Michon\tСан-Мишон\tplace", // multi-word latin src "リン\tРин\tname", // kana src }, "\n") ents, flags := parseBanknote(block, false, attestedIn(src)) if flags.ParseFail { t.Fatalf("every src here is in the source; none may be a parse fail: %+v", flags) } if len(ents) != 5 { t.Fatalf("want all 5 attested lines, got %d: %+v", len(ents), ents) } // The other half of the rule: a plausible, well-formed, Han-bearing line the book never contains. invented := "天魔宗\tСекта Небесного Демона\tplace" ents, flags = parseBanknote(invented, false, attestedIn(src)) if !flags.ParseFail || len(ents) != 0 { t.Fatalf("an invented src must be a parse fail and yield nothing, got %+v %+v", ents, flags) } // And the empty src can never be attested. if attestedIn(src)("") || attestedIn(src)(" ") { t.Fatal("an empty src must never count as attested") } } // TestParseBanknoteRecoversColumnOrder pins the cold-run finding that the attested rule also SOLVES: a // model that writes the pair in the other order (target first) used to lose the whole block. The order is // decided by which side the book attests, so the recovery is pair-blind and never guesses. func TestParseBanknoteRecoversColumnOrder(t *testing.T) { src := "学堂家老在花海。San Michon стоял рядом." block := strings.Join([]string{ "старейшина школы\t学堂家老\ttitle", // reversed by the model "花海\tМоре цветов\tplace", // declared order "выдумка\tтоже выдумка\tterm", // neither side attested → still a parse fail }, "\n") ents, flags := parseBanknote(block, false, attestedIn(src)) if !flags.ParseFail { t.Fatalf("the unattested line must still fail: %+v", flags) } if len(ents) != 2 { t.Fatalf("want the reversed line recovered and the normal one kept, got %+v", ents) } if ents[0].Src != "学堂家老" || ents[0].Dst != "старейшина школы" { t.Fatalf("reversed line not normalized to (source, target): %+v", ents[0]) } if ents[1].Src != "花海" || ents[1].Dst != "Море цветов" { t.Fatalf("declared order must be preserved: %+v", ents[1]) } // Ambiguity is resolved by the DECLARED order, never by a guess: when the book attests both fields // (a source that quotes the target language, or a same-script pair), the model's order stands. both := "San Michon\tСан-Мишон\tplace" ents, flags = parseBanknote(both, false, attestedIn(src+" Сан-Мишон")) if flags.ParseFail || len(ents) != 1 || ents[0].Src != "San Michon" { t.Fatalf("both-attested must keep the declared order, got %+v %+v", ents, flags) } } // TestBankSourceIndexParity pins the property that lets the live path check the CHUNK first: a chunk hit // implies a book hit, so the fast path can never accept a line the book-wide rule would reject (and the // re-fold path, which passes no chunk, reaches the same verdict). func TestBankSourceIndexParity(t *testing.T) { chunks := []chunk.Chunk{{Chapter: 1, ChunkIdx: 0, Text: "方源 вышел из дома."}, {Chapter: 1, ChunkIdx: 1, Text: "古月赤城 ждал во дворе."}} r := &Runner{bankSrc: newBankSourceIndex(chunks)} for _, c := range chunks { chunkPred, bookPred := r.bankSrcAttested(c.Text), r.bankSrcAttested("") for _, probe := range []string{"方源", "古月赤城", "天魔宗", "вышел", ""} { if chunkPred(probe) && !bookPred(probe) { t.Fatalf("chunk %d accepted %q that the book-wide rule rejects — the fast path is not a subset", c.ChunkIdx, probe) } } } // A term attested in ANOTHER chunk is accepted through the book fallback, not silently dropped. if !r.bankSrcAttested(chunks[0].Text)("古月赤城") { t.Fatal("a surface attested elsewhere in the book must pass the fallback") } // The boundary guard: the join must not let a key straddle two chunks. if r.bankSrcAttested("")("дома.古月赤城") { t.Fatal("a key spanning a chunk boundary must not count as attested") } } func TestParseBanknoteTruncationTolerated(t *testing.T) { // The LAST line cut by generation length is tolerated (banknote_truncated), not a parse fail — // but ONLY under truncated_generation; the same short line otherwise IS a parse fail. block := "方源\tФан Юань\tname\n蛊" // last line has no dst entsT, flagsT := parseBanknote(block, true, attestedIn(srcAll)) if !flagsT.Truncated || flagsT.ParseFail || len(entsT) != 1 { t.Fatalf("truncated=true: want 1 entry + truncated flag + no parse_fail, got %+v %+v", entsT, flagsT) } entsF, flagsF := parseBanknote(block, false, attestedIn(srcAll)) if flagsF.Truncated || !flagsF.ParseFail || len(entsF) != 1 { t.Fatalf("truncated=false: the short last line must be a parse_fail, got %+v %+v", entsF, flagsF) } } func TestParseBanknoteEmptyBlock(t *testing.T) { ents, flags := parseBanknote("", false, attestedIn(srcAll)) if len(ents) != 0 || flags.ParseFail || flags.Truncated || flags.NLines != 0 { t.Fatalf("empty block must yield no entries and clean flags, got %+v %+v", ents, flags) } } func TestBankSeparatorIsNotANoteWord(t *testing.T) { // The separator must not be a trailing-note word the sanitizer reserves (§B3-1) — the two // channels must never collide. for _, note := range []string{"Примечание", "Сноска", "Комментарий", "Note", "TN"} { if strings.Contains(bankSeparator, note) { t.Fatalf("separator %q collides with the reserved note-word %q", bankSeparator, note) } } if !strings.HasPrefix(bankSeparator, "⟦") { t.Fatalf("separator lost its unusual delimiter: %q", bankSeparator) } } // --- pack-20 (D39.42 п.4): the channel is a property of the TEXT, not of a config key ---------------- // TestBanknoteSliceIsUnconditional pins the fix for S5, the phase-1 BLOCKER. The slice used to be gated // on gates.banknote.enabled, and that key is in no shipping config — so the literal reading of "move the // block into translator.md" would have sent the raw ⟦TM-BANK-v1⟧ table into the draft, the editor and the // EXPORT with nothing to catch it (the output sanitizer is final-stage-only). Slicing on the text itself // removes the failure mode; a draft with no separator still takes the byte-identical early return. func TestBanknoteSliceIsUnconditional(t *testing.T) { r := &Runner{Pipeline: &config.Pipeline{}} // gate OFF (zero value) const clean = "Тихое утро в библиотеке." const bankTestSource = "龙公 сидел в библиотеке." raw := clean + "\n" + bankSeparator + "\n龙公\tЛун Гун\tname" got, stripped, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", bankTestSource) if got != clean || stripped != clean { t.Fatalf("the block must be sliced even with the gate off, got %q / %q", got, stripped) } if len(entries) != 1 || flags.NLines != 1 { t.Fatalf("the WHAT must still be read (parsing is free), got %+v / %+v", entries, flags) } // The byte-identical path: no separator → the raw text back, no derived export. if g, s, f, e := r.applyBanknoteWithEntries(roleTranslator, clean, "stop", bankTestSource); g != clean || s != "" || f != (bankFlags{}) || e != nil { t.Fatalf("a draft with no separator must take the unchanged path, got %q/%q/%+v/%+v", g, s, f, e) } // The editor never emits banknotes — its output is never touched. if g, s, _, _ := r.applyBanknoteWithEntries(roleEditor, raw, "stop", bankTestSource); g != raw || s != "" { t.Fatalf("the editor's output must not be sliced, got %q/%q", g, s) } } // TestBanknoteMalformedSeparatorIsCaught pins S12 — the very case §B3-1 was written for and the code // never covered. A model that TRIES to open the channel and mistypes the marker used to have its table // read as prose: it travelled into the editor and the export while banknote_parse_fail stayed 0. func TestBanknoteMalformedSeparatorIsCaught(t *testing.T) { r := &Runner{Pipeline: &config.Pipeline{}} const clean = "Тихое утро в библиотеке." const bankTestSource = "龙公 сидел в библиотеке." for _, broken := range []string{ "⟦TM-BANK-v2⟧", // a version the parser does not know "[TM-BANK-v1]", // the wrong brackets "⟦ TM-BANK-v1 ⟧", // stray spaces } { raw := clean + "\n" + broken + "\n龙公\tЛун Гун\tname" got, stripped, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", bankTestSource) if strings.Contains(got, "龙公") || strings.Contains(got, "TM-BANK") { t.Fatalf("%s: the malformed block must still be cut off the draft, got %q", broken, got) } if got != clean || stripped != clean { t.Fatalf("%s: the cleaned draft must be the translation, got %q", broken, got) } if !flags.ParseFail { t.Fatalf("%s: a malformed separator must be flagged, got %+v", broken, flags) } if len(entries) != 0 { t.Fatalf("%s: a block we could not open is not evidence, got %+v", broken, entries) } } // A translation that merely mentions nothing of the sort is untouched — no false positives. if got, _, flags, _ := r.applyBanknoteWithEntries(roleTranslator, clean, "stop", bankTestSource); got != clean || flags.ParseFail { t.Fatalf("ordinary prose must not trip the malformed-separator check: %q / %+v", got, flags) } } // TestBankTokenBudgetIsDerived pins the D39.41 acceptance finding: the reservation must be COMPUTED from // the line format under the engine's own estimator, not a round literal. The old `bankMaxLines * 12` was // 144 against a measured worst block of 142.6–147.2 — at best exactly the need, at worst already negative. func TestBankTokenBudgetIsDerived(t *testing.T) { // A worst-case block: bankMaxLines full lines plus the separator. var b strings.Builder b.WriteString(bankSeparator + "\n") for i := 0; i < bankMaxLines; i++ { b.WriteString(strings.Repeat("蛊", 8) + "\t" + strings.Repeat("я", 40) + "\tnickname\n") } need := EstimateTokens(b.String()) if bankTokenBudget < need { t.Fatalf("the reservation (%d) must cover a worst-case block (%d est-tokens) — a budget that is exactly the need truncates the channel on the densest chapters", bankTokenBudget, need) } // …and it must not be absurdly generous either: an unused ceiling is cheap, but the number should // still be recognisable as this block's cost. if bankTokenBudget > 2*need { t.Fatalf("the reservation (%d) is more than twice the worst case (%d) — that is a guess again, not a derivation", bankTokenBudget, need) } if bankMaxLines <= 12 { t.Fatalf("the line cap was raised on the mini-run measurement (2 of 9 blocks pressed against 12), got %d", bankMaxLines) } }