279 lines
14 KiB
Go
279 lines
14 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/config"
|
||
)
|
||
|
||
// banknote_test.go pins the WS4 §4(д) parser invariants (byte-faithful port of exp16 banknote.py).
|
||
// The 95-line corpus pin (0 parse_fail / 0 truncated over 14 saved raw outputs) is the domain of the
|
||
// $0 reference eval/design11/ws4_banknote_verify.py; these unit tests cover the parser CONTRACT.
|
||
|
||
// attestedIn is the test double for the src rule: the surfaces the "book" contains. The production
|
||
// predicate is bankSrcAttested over bankSourceIndex; both answer the same question over the same
|
||
// normalization, and TestBankSourceIndexParity pins that they agree.
|
||
func attestedIn(source string) func(string) bool {
|
||
r := &Runner{bankSrc: newBankSourceIndex([]chunk.Chunk{{Text: source}})}
|
||
return r.bankSrcAttested("")
|
||
}
|
||
|
||
// srcAll is the corpus every legacy parser test above was written against.
|
||
const srcAll = "方源\n蛊师\n青茅山\n古月\n蛊\n龙公\nRoseanne\n"
|
||
|
||
func TestSplitBanknoteNoSeparatorIsAllClean(t *testing.T) {
|
||
out := "Судзуки шёл по коридору. \n"
|
||
clean, block, _ := splitBanknote(out)
|
||
if clean != "Судзуки шёл по коридору." || block != "" {
|
||
t.Fatalf("no-SEP: clean=%q block=%q, want the whole right-trimmed output and empty block", clean, block)
|
||
}
|
||
}
|
||
|
||
func TestSplitBanknoteSlicesBlock(t *testing.T) {
|
||
out := "Перевод... последнее предложение.\n\n" + bankSeparator + "\n方源\tФан Юань\tname\n蛊师\tгу-мастер\ttitle"
|
||
clean, block, _ := splitBanknote(out)
|
||
if !strings.HasSuffix(clean, "последнее предложение.") {
|
||
t.Fatalf("clean tail wrong: %q", clean)
|
||
}
|
||
if strings.Contains(clean, bankSeparator) || strings.Contains(clean, "方源") {
|
||
t.Fatalf("the banknote block leaked into the clean translation: %q", clean)
|
||
}
|
||
ents, flags := parseBanknote(block, false, attestedIn(srcAll))
|
||
if len(ents) != 2 || flags.ParseFail || flags.Truncated || flags.NLines != 2 {
|
||
t.Fatalf("parse = %+v flags=%+v, want 2 clean entries", ents, flags)
|
||
}
|
||
if ents[0].Src != "方源" || ents[0].Dst != "Фан Юань" || ents[0].Type != "name" {
|
||
t.Fatalf("entry[0] = %+v", ents[0])
|
||
}
|
||
if ents[1].Type != "title" {
|
||
t.Fatalf("entry[1] type = %q, want title", ents[1].Type)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteTolerantFieldSplit(t *testing.T) {
|
||
// tab, ≥2 spaces, and pipe are all accepted delimiters (a model that emits spaces instead of a
|
||
// real TAB still parses). type falls back to "term" when absent or unknown.
|
||
block := "方源\tФан Юань\tname\n蛊师 гу-мастер title\n青茅山 | гора Цинмао | place\n古月\tГу Юэ"
|
||
ents, flags := parseBanknote(block, false, attestedIn(srcAll))
|
||
if flags.ParseFail {
|
||
t.Fatalf("tolerant split should not fail: %+v", flags)
|
||
}
|
||
if len(ents) != 4 {
|
||
t.Fatalf("want 4 entries across tab/space/pipe delimiters, got %d: %+v", len(ents), ents)
|
||
}
|
||
if ents[3].Type != "term" { // no type field → default term
|
||
t.Fatalf("missing type must default to term, got %q", ents[3].Type)
|
||
}
|
||
}
|
||
|
||
// TestParseBanknoteSrcMustBeAttested replaces the old "src must contain a Han ideograph" pin (backlog 19,
|
||
// D39.53 default). Two things changed in ONE rule: a non-ideographic src is no longer malformed by
|
||
// construction (the channel was DEAD on every non-CJK source — 5/6 of the bank's candidates), and a src
|
||
// the book does not contain is now malformed even when it looks Chinese (the old rule accepted invented
|
||
// surfaces, which then reached the owner's sign map).
|
||
func TestParseBanknoteSrcMustBeAttested(t *testing.T) {
|
||
src := "方源 шёл по тропе. Roseanne ждала у Silversaint, рядом стоял San Michon.\nリン сказала: идём.\n"
|
||
block := strings.Join([]string{
|
||
"方源\tФан Юань\tname",
|
||
"Roseanne\tРозанна\tname", // latin src, attested → ACCEPTED (was: parse_fail)
|
||
"Silversaint\tСеребряный святой\ttitle",
|
||
"San Michon\tСан-Мишон\tplace", // multi-word latin src
|
||
"リン\tРин\tname", // kana src
|
||
}, "\n")
|
||
ents, flags := parseBanknote(block, false, attestedIn(src))
|
||
if flags.ParseFail {
|
||
t.Fatalf("every src here is in the source; none may be a parse fail: %+v", flags)
|
||
}
|
||
if len(ents) != 5 {
|
||
t.Fatalf("want all 5 attested lines, got %d: %+v", len(ents), ents)
|
||
}
|
||
// The other half of the rule: a plausible, well-formed, Han-bearing line the book never contains.
|
||
invented := "天魔宗\tСекта Небесного Демона\tplace"
|
||
ents, flags = parseBanknote(invented, false, attestedIn(src))
|
||
if !flags.ParseFail || len(ents) != 0 {
|
||
t.Fatalf("an invented src must be a parse fail and yield nothing, got %+v %+v", ents, flags)
|
||
}
|
||
// And the empty src can never be attested.
|
||
if attestedIn(src)("") || attestedIn(src)(" ") {
|
||
t.Fatal("an empty src must never count as attested")
|
||
}
|
||
}
|
||
|
||
// TestParseBanknoteRecoversColumnOrder pins the cold-run finding that the attested rule also SOLVES: a
|
||
// model that writes the pair in the other order (target first) used to lose the whole block. The order is
|
||
// decided by which side the book attests, so the recovery is pair-blind and never guesses.
|
||
func TestParseBanknoteRecoversColumnOrder(t *testing.T) {
|
||
src := "学堂家老在花海。San Michon стоял рядом."
|
||
block := strings.Join([]string{
|
||
"старейшина школы\t学堂家老\ttitle", // reversed by the model
|
||
"花海\tМоре цветов\tplace", // declared order
|
||
"выдумка\tтоже выдумка\tterm", // neither side attested → still a parse fail
|
||
}, "\n")
|
||
ents, flags := parseBanknote(block, false, attestedIn(src))
|
||
if !flags.ParseFail {
|
||
t.Fatalf("the unattested line must still fail: %+v", flags)
|
||
}
|
||
if len(ents) != 2 {
|
||
t.Fatalf("want the reversed line recovered and the normal one kept, got %+v", ents)
|
||
}
|
||
if ents[0].Src != "学堂家老" || ents[0].Dst != "старейшина школы" {
|
||
t.Fatalf("reversed line not normalized to (source, target): %+v", ents[0])
|
||
}
|
||
if ents[1].Src != "花海" || ents[1].Dst != "Море цветов" {
|
||
t.Fatalf("declared order must be preserved: %+v", ents[1])
|
||
}
|
||
// Ambiguity is resolved by the DECLARED order, never by a guess: when the book attests both fields
|
||
// (a source that quotes the target language, or a same-script pair), the model's order stands.
|
||
both := "San Michon\tСан-Мишон\tplace"
|
||
ents, flags = parseBanknote(both, false, attestedIn(src+" Сан-Мишон"))
|
||
if flags.ParseFail || len(ents) != 1 || ents[0].Src != "San Michon" {
|
||
t.Fatalf("both-attested must keep the declared order, got %+v %+v", ents, flags)
|
||
}
|
||
}
|
||
|
||
// TestBankSourceIndexParity pins the property that lets the live path check the CHUNK first: a chunk hit
|
||
// implies a book hit, so the fast path can never accept a line the book-wide rule would reject (and the
|
||
// re-fold path, which passes no chunk, reaches the same verdict).
|
||
func TestBankSourceIndexParity(t *testing.T) {
|
||
chunks := []chunk.Chunk{{Chapter: 1, ChunkIdx: 0, Text: "方源 вышел из дома."}, {Chapter: 1, ChunkIdx: 1, Text: "古月赤城 ждал во дворе."}}
|
||
r := &Runner{bankSrc: newBankSourceIndex(chunks)}
|
||
for _, c := range chunks {
|
||
chunkPred, bookPred := r.bankSrcAttested(c.Text), r.bankSrcAttested("")
|
||
for _, probe := range []string{"方源", "古月赤城", "天魔宗", "вышел", ""} {
|
||
if chunkPred(probe) && !bookPred(probe) {
|
||
t.Fatalf("chunk %d accepted %q that the book-wide rule rejects — the fast path is not a subset", c.ChunkIdx, probe)
|
||
}
|
||
}
|
||
}
|
||
// A term attested in ANOTHER chunk is accepted through the book fallback, not silently dropped.
|
||
if !r.bankSrcAttested(chunks[0].Text)("古月赤城") {
|
||
t.Fatal("a surface attested elsewhere in the book must pass the fallback")
|
||
}
|
||
// The boundary guard: the join must not let a key straddle two chunks.
|
||
if r.bankSrcAttested("")("дома.古月赤城") {
|
||
t.Fatal("a key spanning a chunk boundary must not count as attested")
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteTruncationTolerated(t *testing.T) {
|
||
// The LAST line cut by generation length is tolerated (banknote_truncated), not a parse fail —
|
||
// but ONLY under truncated_generation; the same short line otherwise IS a parse fail.
|
||
block := "方源\tФан Юань\tname\n蛊" // last line has no dst
|
||
entsT, flagsT := parseBanknote(block, true, attestedIn(srcAll))
|
||
if !flagsT.Truncated || flagsT.ParseFail || len(entsT) != 1 {
|
||
t.Fatalf("truncated=true: want 1 entry + truncated flag + no parse_fail, got %+v %+v", entsT, flagsT)
|
||
}
|
||
entsF, flagsF := parseBanknote(block, false, attestedIn(srcAll))
|
||
if flagsF.Truncated || !flagsF.ParseFail || len(entsF) != 1 {
|
||
t.Fatalf("truncated=false: the short last line must be a parse_fail, got %+v %+v", entsF, flagsF)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteEmptyBlock(t *testing.T) {
|
||
ents, flags := parseBanknote("", false, attestedIn(srcAll))
|
||
if len(ents) != 0 || flags.ParseFail || flags.Truncated || flags.NLines != 0 {
|
||
t.Fatalf("empty block must yield no entries and clean flags, got %+v %+v", ents, flags)
|
||
}
|
||
}
|
||
|
||
func TestBankSeparatorIsNotANoteWord(t *testing.T) {
|
||
// The separator must not be a trailing-note word the sanitizer reserves (§B3-1) — the two
|
||
// channels must never collide.
|
||
for _, note := range []string{"Примечание", "Сноска", "Комментарий", "Note", "TN"} {
|
||
if strings.Contains(bankSeparator, note) {
|
||
t.Fatalf("separator %q collides with the reserved note-word %q", bankSeparator, note)
|
||
}
|
||
}
|
||
if !strings.HasPrefix(bankSeparator, "⟦") {
|
||
t.Fatalf("separator lost its unusual delimiter: %q", bankSeparator)
|
||
}
|
||
}
|
||
|
||
// --- pack-20 (D39.42 п.4): the channel is a property of the TEXT, not of a config key ----------------
|
||
|
||
// TestBanknoteSliceIsUnconditional pins the fix for S5, the phase-1 BLOCKER. The slice used to be gated
|
||
// on gates.banknote.enabled, and that key is in no shipping config — so the literal reading of "move the
|
||
// block into translator.md" would have sent the raw ⟦TM-BANK-v1⟧ table into the draft, the editor and the
|
||
// EXPORT with nothing to catch it (the output sanitizer is final-stage-only). Slicing on the text itself
|
||
// removes the failure mode; a draft with no separator still takes the byte-identical early return.
|
||
func TestBanknoteSliceIsUnconditional(t *testing.T) {
|
||
r := &Runner{Pipeline: &config.Pipeline{}} // gate OFF (zero value)
|
||
const clean = "Тихое утро в библиотеке."
|
||
const bankTestSource = "龙公 сидел в библиотеке."
|
||
raw := clean + "\n" + bankSeparator + "\n龙公\tЛун Гун\tname"
|
||
|
||
got, stripped, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", bankTestSource)
|
||
if got != clean || stripped != clean {
|
||
t.Fatalf("the block must be sliced even with the gate off, got %q / %q", got, stripped)
|
||
}
|
||
if len(entries) != 1 || flags.NLines != 1 {
|
||
t.Fatalf("the WHAT must still be read (parsing is free), got %+v / %+v", entries, flags)
|
||
}
|
||
// The byte-identical path: no separator → the raw text back, no derived export.
|
||
if g, s, f, e := r.applyBanknoteWithEntries(roleTranslator, clean, "stop", bankTestSource); g != clean || s != "" || f != (bankFlags{}) || e != nil {
|
||
t.Fatalf("a draft with no separator must take the unchanged path, got %q/%q/%+v/%+v", g, s, f, e)
|
||
}
|
||
// The editor never emits banknotes — its output is never touched.
|
||
if g, s, _, _ := r.applyBanknoteWithEntries(roleEditor, raw, "stop", bankTestSource); g != raw || s != "" {
|
||
t.Fatalf("the editor's output must not be sliced, got %q/%q", g, s)
|
||
}
|
||
}
|
||
|
||
// TestBanknoteMalformedSeparatorIsCaught pins S12 — the very case §B3-1 was written for and the code
|
||
// never covered. A model that TRIES to open the channel and mistypes the marker used to have its table
|
||
// read as prose: it travelled into the editor and the export while banknote_parse_fail stayed 0.
|
||
func TestBanknoteMalformedSeparatorIsCaught(t *testing.T) {
|
||
r := &Runner{Pipeline: &config.Pipeline{}}
|
||
const clean = "Тихое утро в библиотеке."
|
||
const bankTestSource = "龙公 сидел в библиотеке."
|
||
for _, broken := range []string{
|
||
"⟦TM-BANK-v2⟧", // a version the parser does not know
|
||
"[TM-BANK-v1]", // the wrong brackets
|
||
"⟦ TM-BANK-v1 ⟧", // stray spaces
|
||
} {
|
||
raw := clean + "\n" + broken + "\n龙公\tЛун Гун\tname"
|
||
got, stripped, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "stop", bankTestSource)
|
||
if strings.Contains(got, "龙公") || strings.Contains(got, "TM-BANK") {
|
||
t.Fatalf("%s: the malformed block must still be cut off the draft, got %q", broken, got)
|
||
}
|
||
if got != clean || stripped != clean {
|
||
t.Fatalf("%s: the cleaned draft must be the translation, got %q", broken, got)
|
||
}
|
||
if !flags.ParseFail {
|
||
t.Fatalf("%s: a malformed separator must be flagged, got %+v", broken, flags)
|
||
}
|
||
if len(entries) != 0 {
|
||
t.Fatalf("%s: a block we could not open is not evidence, got %+v", broken, entries)
|
||
}
|
||
}
|
||
// A translation that merely mentions nothing of the sort is untouched — no false positives.
|
||
if got, _, flags, _ := r.applyBanknoteWithEntries(roleTranslator, clean, "stop", bankTestSource); got != clean || flags.ParseFail {
|
||
t.Fatalf("ordinary prose must not trip the malformed-separator check: %q / %+v", got, flags)
|
||
}
|
||
}
|
||
|
||
// TestBankTokenBudgetIsDerived pins the D39.41 acceptance finding: the reservation must be COMPUTED from
|
||
// the line format under the engine's own estimator, not a round literal. The old `bankMaxLines * 12` was
|
||
// 144 against a measured worst block of 142.6–147.2 — at best exactly the need, at worst already negative.
|
||
func TestBankTokenBudgetIsDerived(t *testing.T) {
|
||
// A worst-case block: bankMaxLines full lines plus the separator.
|
||
var b strings.Builder
|
||
b.WriteString(bankSeparator + "\n")
|
||
for i := 0; i < bankMaxLines; i++ {
|
||
b.WriteString(strings.Repeat("蛊", 8) + "\t" + strings.Repeat("я", 40) + "\tnickname\n")
|
||
}
|
||
need := EstimateTokens(b.String())
|
||
if bankTokenBudget < need {
|
||
t.Fatalf("the reservation (%d) must cover a worst-case block (%d est-tokens) — a budget that is exactly the need truncates the channel on the densest chapters", bankTokenBudget, need)
|
||
}
|
||
// …and it must not be absurdly generous either: an unused ceiling is cheap, but the number should
|
||
// still be recognisable as this block's cost.
|
||
if bankTokenBudget > 2*need {
|
||
t.Fatalf("the reservation (%d) is more than twice the worst case (%d) — that is a guess again, not a derivation", bankTokenBudget, need)
|
||
}
|
||
if bankMaxLines <= 12 {
|
||
t.Fatalf("the line cap was raised on the mini-run measurement (2 of 9 blocks pressed against 12), got %d", bankMaxLines)
|
||
}
|
||
}
|