textmachine/backend/internal/pipeline/banknote_test.go

384 lines
21 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package pipeline
import (
"strings"
"testing"
"textmachine/backend/internal/chunk"
"textmachine/backend/internal/config"
)
// banknote_test.go pins the WS4 §4(д) parser invariants (byte-faithful port of exp16 banknote.py).
// The 95-line corpus pin (0 parse_fail / 0 truncated over 14 saved raw outputs) is the domain of the
// $0 reference eval/design11/ws4_banknote_verify.py; these unit tests cover the parser CONTRACT.
// attestedIn is the test double for the src rule: the surfaces the "book" contains. The production
// predicate is bankSrcAttested over bankSourceIndex; both answer the same question over the same
// normalization, and TestBankSourceIndexParity pins that they agree.
func attestedIn(source string) func(string) bool {
r := &Runner{bankSrc: newBankSourceIndex([]chunk.Chunk{{Text: source}})}
return r.bankSrcAttested("")
}
// srcAll is the corpus every legacy parser test above was written against.
const srcAll = "方源\n蛊师\n青茅山\n古月\n蛊\n龙公\nRoseanne\n"
func TestSplitBanknoteNoSeparatorIsAllClean(t *testing.T) {
out := "Судзуки шёл по коридору. \n"
clean, block, _ := splitBanknote(out)
if clean != "Судзуки шёл по коридору." || block != "" {
t.Fatalf("no-SEP: clean=%q block=%q, want the whole right-trimmed output and empty block", clean, block)
}
}
func TestSplitBanknoteSlicesBlock(t *testing.T) {
out := "Перевод... последнее предложение.\n\n" + bankSeparator + "\n方源\tФан Юань\tname\n蛊师\tгу-мастер\ttitle"
clean, block, _ := splitBanknote(out)
if !strings.HasSuffix(clean, "последнее предложение.") {
t.Fatalf("clean tail wrong: %q", clean)
}
if strings.Contains(clean, bankSeparator) || strings.Contains(clean, "方源") {
t.Fatalf("the banknote block leaked into the clean translation: %q", clean)
}
ents, flags := parseBanknote(block, attestedIn(srcAll))
if len(ents) != 2 || flags.ParseFail || flags.Truncated || flags.NLines != 2 {
t.Fatalf("parse = %+v flags=%+v, want 2 clean entries", ents, flags)
}
if ents[0].Src != "方源" || ents[0].Dst != "Фан Юань" || ents[0].Type != "name" {
t.Fatalf("entry[0] = %+v", ents[0])
}
if ents[1].Type != "title" {
t.Fatalf("entry[1] type = %q, want title", ents[1].Type)
}
}
func TestParseBanknoteTolerantFieldSplit(t *testing.T) {
// tab, ≥2 spaces, and pipe are all accepted delimiters (a model that emits spaces instead of a
// real TAB still parses). type falls back to "term" when absent or unknown.
block := "方源\tФан Юань\tname\n蛊师 гу-мастер title\n青茅山 | гора Цинмао | place\n古月\tГу Юэ"
ents, flags := parseBanknote(block, attestedIn(srcAll))
if flags.ParseFail {
t.Fatalf("tolerant split should not fail: %+v", flags)
}
if len(ents) != 4 {
t.Fatalf("want 4 entries across tab/space/pipe delimiters, got %d: %+v", len(ents), ents)
}
if ents[3].Type != "term" { // no type field → default term
t.Fatalf("missing type must default to term, got %q", ents[3].Type)
}
}
// TestParseBanknoteSrcMustBeAttested replaces the old "src must contain a Han ideograph" pin (backlog 19,
// D39.53 default). Two things changed in ONE rule: a non-ideographic src is no longer malformed by
// construction (the channel was DEAD on every non-CJK source — 5/6 of the bank's candidates), and a src
// the book does not contain is now malformed even when it looks Chinese (the old rule accepted invented
// surfaces, which then reached the owner's sign map).
func TestParseBanknoteSrcMustBeAttested(t *testing.T) {
src := "方源 шёл по тропе. Roseanne ждала у Silversaint, рядом стоял San Michon.\nリン сказала: идём.\n"
block := strings.Join([]string{
"方源\tФан Юань\tname",
"Roseanne\tРозанна\tname", // latin src, attested → ACCEPTED (was: parse_fail)
"Silversaint\tСеребряный святой\ttitle",
"San Michon\tСан-Мишон\tplace", // multi-word latin src
"リン\tРин\tname", // kana src
}, "\n")
ents, flags := parseBanknote(block, attestedIn(src))
if flags.ParseFail {
t.Fatalf("every src here is in the source; none may be a parse fail: %+v", flags)
}
if len(ents) != 5 {
t.Fatalf("want all 5 attested lines, got %d: %+v", len(ents), ents)
}
// The other half of the rule: a plausible, well-formed, Han-bearing line the book never contains.
invented := "天魔宗\tСекта Небесного Демона\tplace"
ents, flags = parseBanknote(invented, attestedIn(src))
if !flags.ParseFail || len(ents) != 0 {
t.Fatalf("an invented src must be a parse fail and yield nothing, got %+v %+v", ents, flags)
}
// And the empty src can never be attested.
if attestedIn(src)("") || attestedIn(src)(" ") {
t.Fatal("an empty src must never count as attested")
}
}
// TestParseBanknoteRecoversColumnOrder pins the cold-run finding that the attested rule also SOLVES: a
// model that writes the pair in the other order (target first) used to lose the whole block. The order is
// decided by which side the book attests, so the recovery is pair-blind and never guesses.
func TestParseBanknoteRecoversColumnOrder(t *testing.T) {
src := "学堂家老在花海。San Michon стоял рядом."
block := strings.Join([]string{
"старейшина школы\t学堂家老\ttitle", // reversed by the model
"花海\tМоре цветов\tplace", // declared order
"выдумка\tтоже выдумка\tterm", // neither side attested → still a parse fail
}, "\n")
ents, flags := parseBanknote(block, attestedIn(src))
if !flags.ParseFail {
t.Fatalf("the unattested line must still fail: %+v", flags)
}
if len(ents) != 2 {
t.Fatalf("want the reversed line recovered and the normal one kept, got %+v", ents)
}
if ents[0].Src != "学堂家老" || ents[0].Dst != "старейшина школы" {
t.Fatalf("reversed line not normalized to (source, target): %+v", ents[0])
}
if ents[1].Src != "花海" || ents[1].Dst != "Море цветов" {
t.Fatalf("declared order must be preserved: %+v", ents[1])
}
// Ambiguity is resolved by the DECLARED order, never by a guess: when the book attests both fields
// (a source that quotes the target language, or a same-script pair), the model's order stands.
both := "San Michon\tСан-Мишон\tplace"
ents, flags = parseBanknote(both, attestedIn(src+" Сан-Мишон"))
if flags.ParseFail || len(ents) != 1 || ents[0].Src != "San Michon" {
t.Fatalf("both-attested must keep the declared order, got %+v %+v", ents, flags)
}
}
// TestBankSourceIndexParity pins the property that lets the live path check the CHUNK first: a chunk hit
// implies a book hit, so the fast path can never accept a line the book-wide rule would reject (and the
// re-fold path, which passes no chunk, reaches the same verdict).
func TestBankSourceIndexParity(t *testing.T) {
chunks := []chunk.Chunk{{Chapter: 1, ChunkIdx: 0, Text: "方源 вышел из дома."}, {Chapter: 1, ChunkIdx: 1, Text: "古月赤城 ждал во дворе."}}
r := &Runner{bankSrc: newBankSourceIndex(chunks)}
for _, c := range chunks {
chunkPred, bookPred := r.bankSrcAttested(c.Text), r.bankSrcAttested("")
for _, probe := range []string{"方源", "古月赤城", "天魔宗", "вышел", ""} {
if chunkPred(probe) && !bookPred(probe) {
t.Fatalf("chunk %d accepted %q that the book-wide rule rejects — the fast path is not a subset", c.ChunkIdx, probe)
}
}
}
// A term attested in ANOTHER chunk is accepted through the book fallback, not silently dropped.
if !r.bankSrcAttested(chunks[0].Text)("古月赤城") {
t.Fatal("a surface attested elsewhere in the book must pass the fallback")
}
// The boundary guard: the join must not let a key straddle two chunks.
if r.bankSrcAttested("")("дома.古月赤城") {
t.Fatal("a key spanning a chunk boundary must not count as attested")
}
}
// TestParseBanknoteShortLastLineIsAParseFail: the parser's truncation TOLERANCE is gone with the fix-pack,
// and this pins what replaced it. A short last line reaches this parser only under finish=stop — a complete
// generation that simply emitted a broken line — and that is a parse fail, not a tolerated cut. The
// truncation FACT now lives where it is actually known, one level up, off the finish reason
// (TestBanknoteTruncatedIsReachable).
func TestParseBanknoteShortLastLineIsAParseFail(t *testing.T) {
block := "方源\tФан Юань\tname\n蛊" // last line has no dst
ents, flags := parseBanknote(block, attestedIn(srcAll))
if flags.Truncated || !flags.ParseFail || len(ents) != 1 {
t.Fatalf("a complete generation's short line is a parse_fail, got %+v %+v", ents, flags)
}
}
func TestParseBanknoteEmptyBlock(t *testing.T) {
ents, flags := parseBanknote("", attestedIn(srcAll))
if len(ents) != 0 || flags.ParseFail || flags.Truncated || flags.NLines != 0 {
t.Fatalf("empty block must yield no entries and clean flags, got %+v %+v", ents, flags)
}
}
func TestBankSeparatorIsNotANoteWord(t *testing.T) {
// The separator must not be a trailing-note word the sanitizer reserves (§B3-1) — the two
// channels must never collide.
for _, note := range []string{"Примечание", "Сноска", "Комментарий", "Note", "TN"} {
if strings.Contains(bankSeparator, note) {
t.Fatalf("separator %q collides with the reserved note-word %q", bankSeparator, note)
}
}
if !strings.HasPrefix(bankSeparator, "⟦") {
t.Fatalf("separator lost its unusual delimiter: %q", bankSeparator)
}
}
// --- pack-20 (D39.42 п.4): the channel is a property of the TEXT, not of a config key ----------------
// TestBanknoteSliceIsUnconditional pins the fix for S5, the phase-1 BLOCKER. The slice used to be gated
// on gates.banknote.enabled, and that key is in no shipping config — so the literal reading of "move the
// block into translator.md" would have sent the raw ⟦TM-BANK-v1⟧ table into the draft, the editor and the
// EXPORT with nothing to catch it (the output sanitizer is final-stage-only). Slicing on the text itself
// removes the failure mode; a draft with no separator still takes the byte-identical early return.
func TestBanknoteSliceIsUnconditional(t *testing.T) {
r := &Runner{Pipeline: &config.Pipeline{}} // gate OFF (zero value)
const clean = "Тихое утро в библиотеке."
const bankTestSource = "龙公 сидел в библиотеке."
raw := clean + "\n" + bankSeparator + "\n龙公\tЛун Гун\tname"
got, stripped, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", bankTestSource)
if got != clean || stripped != clean {
t.Fatalf("the block must be sliced even with the gate off, got %q / %q", got, stripped)
}
if len(entries) != 1 || flags.NLines != 1 {
t.Fatalf("the WHAT must still be read (parsing is free), got %+v / %+v", entries, flags)
}
// The byte-identical path: no separator → the raw text back, no derived export.
if g, s, f, e := r.applyBanknoteWithEntries(roleTranslator, clean, "stop", bankTestSource); g != clean || s != "" || f != (bankFlags{}) || e != nil {
t.Fatalf("a draft with no separator must take the unchanged path, got %q/%q/%+v/%+v", g, s, f, e)
}
// The editor never emits banknotes — its output is never touched.
if g, s, _, _ := r.applyBanknoteWithEntries(roleEditor, raw, "stop", bankTestSource); g != raw || s != "" {
t.Fatalf("the editor's output must not be sliced, got %q/%q", g, s)
}
}
// TestBanknoteMalformedSeparatorIsCaught pins S12 — the very case §B3-1 was written for and the code
// never covered. A model that TRIES to open the channel and mistypes the marker used to have its table
// read as prose: it travelled into the editor and the export while banknote_parse_fail stayed 0.
func TestBanknoteMalformedSeparatorIsCaught(t *testing.T) {
r := &Runner{Pipeline: &config.Pipeline{}}
const clean = "Тихое утро в библиотеке."
const bankTestSource = "龙公 сидел в библиотеке."
for _, broken := range []string{
"⟦TM-BANK-v2⟧", // a version the parser does not know
"[TM-BANK-v1]", // the wrong brackets
"⟦ TM-BANK-v1 ⟧", // stray spaces
} {
raw := clean + "\n" + broken + "\n龙公\tЛун Гун\tname"
got, stripped, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", bankTestSource)
if strings.Contains(got, "龙公") || strings.Contains(got, "TM-BANK") {
t.Fatalf("%s: the malformed block must still be cut off the draft, got %q", broken, got)
}
if got != clean || stripped != clean {
t.Fatalf("%s: the cleaned draft must be the translation, got %q", broken, got)
}
if !flags.ParseFail {
t.Fatalf("%s: a malformed separator must be flagged, got %+v", broken, flags)
}
if len(entries) != 0 {
t.Fatalf("%s: a block we could not open is not evidence, got %+v", broken, entries)
}
}
// A translation that merely mentions nothing of the sort is untouched — no false positives.
if got, _, flags, _ := r.applyBanknoteWithEntries(roleTranslator, clean, "stop", bankTestSource); got != clean || flags.ParseFail {
t.Fatalf("ordinary prose must not trip the malformed-separator check: %q / %+v", got, flags)
}
}
// TestBankTokenBudgetIsDerived pins the D39.41 acceptance finding: the reservation must be COMPUTED from
// the line format under the engine's own estimator, not a round literal. The old `bankMaxLines * 12` was
// 144 against a measured worst block of 142.6147.2 — at best exactly the need, at worst already negative.
func TestBankTokenBudgetIsDerived(t *testing.T) {
// A worst-case block: bankMaxLines full lines plus the separator.
var b strings.Builder
b.WriteString(bankSeparator + "\n")
for i := 0; i < bankMaxLines; i++ {
b.WriteString(strings.Repeat("蛊", 8) + "\t" + strings.Repeat("я", 40) + "\tnickname\n")
}
need := EstimateTokens(b.String())
if bankTokenBudget < need {
t.Fatalf("the reservation (%d) must cover a worst-case block (%d est-tokens) — a budget that is exactly the need truncates the channel on the densest chapters", bankTokenBudget, need)
}
// …and it must not be absurdly generous either: an unused ceiling is cheap, but the number should
// still be recognisable as this block's cost.
if bankTokenBudget > 2*need {
t.Fatalf("the reservation (%d) is more than twice the worst case (%d) — that is a guess again, not a derivation", bankTokenBudget, need)
}
if bankMaxLines <= 12 {
t.Fatalf("the line cap was raised on the mini-run measurement (2 of 9 blocks pressed against 12), got %d", bankMaxLines)
}
}
// TestParseBanknoteRejoinsASplitRendering is backlog row 129. The field splitter tolerates a run of ≥2
// spaces because models substitute spaces for tabs — and that same tolerance splits INSIDE a rendering the
// moment a model writes «Фан␣␣Юань». Taking parts[1] alone banked the half-name and pushed the tail into
// the type column, where it failed the closed vocabulary and silently became "term": a mangled surface on
// the owner's sign map with parse_fail still reading 0. The terminologist's parser was fixed for exactly
// this; the banknote's was not.
func TestParseBanknoteRejoinsASplitRendering(t *testing.T) {
attested := attestedIn(srcAll)
// BOTH shapes: with the type column present, and as a bare two-column line.
entries, flags := parseBanknote("方源\tФан Юань\tname\n蛊师\tмастер гу", attested)
if flags.ParseFail {
t.Fatalf("neither line is malformed: %+v", flags)
}
if len(entries) != 2 {
t.Fatalf("want both lines, got %+v", entries)
}
if entries[0].Dst != "Фан Юань" || entries[0].Type != "name" {
t.Fatalf("a split rendering must be re-joined and the type column still read: %+v", entries[0])
}
if entries[1].Dst != "мастер гу" || entries[1].Type != "term" {
t.Fatalf("with no type column the whole tail is the rendering: %+v", entries[1])
}
// THE SPACE-RUN PATH is the only one where the re-join can be observed at all: on a TAB line the
// rendering is one column, so join(fields[1:]) and fields[1] are the same string and the re-join could be
// reverted with every test still green. Here they differ, and «Фан» alone is what used to be banked.
spaced, sflags := parseBanknote("方源 Фан Юань name", attested)
if sflags.ParseFail || len(spaced) != 1 || spaced[0].Dst != "Фан Юань" || spaced[0].Type != "name" {
t.Fatalf("a space-delimited line must re-join its rendering: %+v %+v", spaced, sflags)
}
if two, _ := parseBanknote("蛊师 мастер гу", attested); len(two) != 1 || two[0].Dst != "мастер гу" {
t.Fatalf("and so must a space-delimited line with no type column: %+v", two)
}
// The recognised shapes are unchanged: a real three-field line is still src/dst/type.
plain, _ := parseBanknote("青茅山\tгора Цинмао\tplace", attested)
if len(plain) != 1 || plain[0].Dst != "гора Цинмао" || plain[0].Type != "place" {
t.Fatalf("the ordinary three-field line must be untouched: %+v", plain)
}
// A TAB-delimited third column outside the type vocabulary is a benign unknown TYPE, exactly as it was
// before the fix-pack — NOT a piece of the rendering. Guessing it into the rendering is what the first cut
// of this fix did, and it corrupted four terms and dropped two on the frozen coldrun-a reference.
odd, _ := parseBanknote("古月\tклан Гуюэ\tсемья", attested)
if len(odd) != 1 || odd[0].Dst != "клан Гуюэ" || odd[0].Type != "term" {
t.Fatalf("an unknown TYPE column must be discarded, not glued onto the rendering: %+v", odd)
}
// And the column-order recovery still works, including over a rendering the model spaced out.
rev, _ := parseBanknote("Фан Юань\t方源\tname", attested)
if len(rev) != 1 || rev[0].Src != "方源" || rev[0].Dst != "Фан Юань" {
t.Fatalf("a reversed line must still be recovered: %+v", rev)
}
}
// TestParseBanknoteReadsDeclaredColumnsPositionally is the pin the frozen reference bought. A tab or a pipe
// is a delimiter the model CHOSE — it cannot occur inside a rendering — so such a line has declared its own
// columns and is read by position, as it was before the fix-pack. Only a line delimited by a run of spaces
// is ambiguous, and only there may the type be guessed from the closed vocabulary.
//
// Measured cost of getting this wrong: replaying 56 real banknote blocks of ~/books/gu-zhenren/coldrun-a
// through the first cut of the fix gave 6 lines against HEAD and +0 — four renderings corrupted by a glued
// type word (clan/event/proverb/onomatopoeia — 1.25% of the reference carries a type outside the
// vocabulary) and two lines dropped outright, with banknote_parse_fail unchanged in every one of them.
func TestParseBanknoteReadsDeclaredColumnsPositionally(t *testing.T) {
attested := attestedIn(srcAll)
for _, tc := range []struct{ line, src, dst, typ string }{
// An unknown third column: discarded as an unknown type, never glued (the four corrupted terms).
{"古月\tГу Юэ\tclan", "古月", "Гу Юэ", "term"},
{"方源|Фан Юань|proverb", "方源", "Фан Юань", "term"},
// The same line REVERSED: the src is still found by attestation, and the unknown type still discarded.
// The first cut looked for the src in the last field, found «clan», and dropped the line (the two lost).
{"Гу Юэ\t古月\tclan", "古月", "Гу Юэ", "term"},
// A known type column is read as a type, in both orders.
{"青茅山\tгора Цинмао\tplace", "青茅山", "гора Цинмао", "place"},
// And a rendering the model spaced out inside a REAL column arrives whole and clean.
{"方源\tФан Юань\tname", "方源", "Фан Юань", "name"},
} {
got, flags := parseBanknote(tc.line, attested)
if len(got) != 1 {
t.Fatalf("%q: want one entry, got %+v (parse_fail=%v)", tc.line, got, flags.ParseFail)
}
if got[0].Src != tc.src || got[0].Dst != tc.dst || got[0].Type != tc.typ {
t.Fatalf("%q: got %+v, want src=%q dst=%q type=%q", tc.line, got[0], tc.src, tc.dst, tc.typ)
}
}
}
// TestBanknoteTruncatedIsReachable closes the hygiene finding beside row 129: banknote_truncated was
// derived inside the parser from an argument the only production caller pinned to false, so the column was
// structurally 0 while its comment claimed otherwise. The refusal itself is the signal — a block present
// under a non-stop finish was cut by generation length — and reading it off the finish reason changes
// nothing about the ratified stop-only gate: still no candidates, still no telemetry lines.
func TestBanknoteTruncatedIsReachable(t *testing.T) {
r := &Runner{bankSrc: newBankSourceIndex([]chunk.Chunk{{Text: srcAll}})}
raw := "Незаконченный перевод\n" + bankSeparator + "\n方源\tФан"
_, _, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "length", srcAll)
if !flags.Truncated {
t.Fatalf("a block under a non-stop finish IS the truncation the column was meant to record: %+v", flags)
}
if len(entries) != 0 || flags.NLines != 0 {
t.Fatalf("the stop-only gate still accepts nothing: %+v / %+v", entries, flags)
}
// A complete generation is unaffected.
_, _, ok, got := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", srcAll)
if ok.Truncated || len(got) != 1 {
t.Fatalf("a complete generation is not truncated and its block is parsed: %+v / %+v", ok, got)
}
}