188 lines
9.1 KiB
Go
188 lines
9.1 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/config"
|
||
)
|
||
|
||
// banknote_test.go pins the WS4 §4(д) parser invariants (byte-faithful port of exp16 banknote.py).
|
||
// The 95-line corpus pin (0 parse_fail / 0 truncated over 14 saved raw outputs) is the domain of the
|
||
// $0 reference eval/design11/ws4_banknote_verify.py; these unit tests cover the parser CONTRACT.
|
||
|
||
func TestSplitBanknoteNoSeparatorIsAllClean(t *testing.T) {
|
||
out := "Судзуки шёл по коридору. \n"
|
||
clean, block, _ := splitBanknote(out)
|
||
if clean != "Судзуки шёл по коридору." || block != "" {
|
||
t.Fatalf("no-SEP: clean=%q block=%q, want the whole right-trimmed output and empty block", clean, block)
|
||
}
|
||
}
|
||
|
||
func TestSplitBanknoteSlicesBlock(t *testing.T) {
|
||
out := "Перевод... последнее предложение.\n\n" + bankSeparator + "\n方源\tФан Юань\tname\n蛊师\tгу-мастер\ttitle"
|
||
clean, block, _ := splitBanknote(out)
|
||
if !strings.HasSuffix(clean, "последнее предложение.") {
|
||
t.Fatalf("clean tail wrong: %q", clean)
|
||
}
|
||
if strings.Contains(clean, bankSeparator) || strings.Contains(clean, "方源") {
|
||
t.Fatalf("the banknote block leaked into the clean translation: %q", clean)
|
||
}
|
||
ents, flags := parseBanknote(block, false)
|
||
if len(ents) != 2 || flags.ParseFail || flags.Truncated || flags.NLines != 2 {
|
||
t.Fatalf("parse = %+v flags=%+v, want 2 clean entries", ents, flags)
|
||
}
|
||
if ents[0].Src != "方源" || ents[0].Dst != "Фан Юань" || ents[0].Type != "name" {
|
||
t.Fatalf("entry[0] = %+v", ents[0])
|
||
}
|
||
if ents[1].Type != "title" {
|
||
t.Fatalf("entry[1] type = %q, want title", ents[1].Type)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteTolerantFieldSplit(t *testing.T) {
|
||
// tab, ≥2 spaces, and pipe are all accepted delimiters (a model that emits spaces instead of a
|
||
// real TAB still parses). type falls back to "term" when absent or unknown.
|
||
block := "方源\tФан Юань\tname\n蛊师 гу-мастер title\n青茅山 | гора Цинмао | place\n古月\tГу Юэ"
|
||
ents, flags := parseBanknote(block, false)
|
||
if flags.ParseFail {
|
||
t.Fatalf("tolerant split should not fail: %+v", flags)
|
||
}
|
||
if len(ents) != 4 {
|
||
t.Fatalf("want 4 entries across tab/space/pipe delimiters, got %d: %+v", len(ents), ents)
|
||
}
|
||
if ents[3].Type != "term" { // no type field → default term
|
||
t.Fatalf("missing type must default to term, got %q", ents[3].Type)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteNonHanSrcIsBad(t *testing.T) {
|
||
// A zh→ru channel line whose src has no Han ideograph is malformed → parse_fail.
|
||
block := "方源\tФан Юань\tname\nRoseanne\tРозанна\tname"
|
||
ents, flags := parseBanknote(block, false)
|
||
if !flags.ParseFail {
|
||
t.Fatalf("a non-Han src must set parse_fail")
|
||
}
|
||
if len(ents) != 1 || ents[0].Src != "方源" {
|
||
t.Fatalf("only the Han-src line should survive, got %+v", ents)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteTruncationTolerated(t *testing.T) {
|
||
// The LAST line cut by generation length is tolerated (banknote_truncated), not a parse fail —
|
||
// but ONLY under truncated_generation; the same short line otherwise IS a parse fail.
|
||
block := "方源\tФан Юань\tname\n蛊" // last line has no dst
|
||
entsT, flagsT := parseBanknote(block, true)
|
||
if !flagsT.Truncated || flagsT.ParseFail || len(entsT) != 1 {
|
||
t.Fatalf("truncated=true: want 1 entry + truncated flag + no parse_fail, got %+v %+v", entsT, flagsT)
|
||
}
|
||
entsF, flagsF := parseBanknote(block, false)
|
||
if flagsF.Truncated || !flagsF.ParseFail || len(entsF) != 1 {
|
||
t.Fatalf("truncated=false: the short last line must be a parse_fail, got %+v %+v", entsF, flagsF)
|
||
}
|
||
}
|
||
|
||
func TestParseBanknoteEmptyBlock(t *testing.T) {
|
||
ents, flags := parseBanknote("", false)
|
||
if len(ents) != 0 || flags.ParseFail || flags.Truncated || flags.NLines != 0 {
|
||
t.Fatalf("empty block must yield no entries and clean flags, got %+v %+v", ents, flags)
|
||
}
|
||
}
|
||
|
||
func TestBankSeparatorIsNotANoteWord(t *testing.T) {
|
||
// The separator must not be a trailing-note word the sanitizer reserves (§B3-1) — the two
|
||
// channels must never collide.
|
||
for _, note := range []string{"Примечание", "Сноска", "Комментарий", "Note", "TN"} {
|
||
if strings.Contains(bankSeparator, note) {
|
||
t.Fatalf("separator %q collides with the reserved note-word %q", bankSeparator, note)
|
||
}
|
||
}
|
||
if !strings.HasPrefix(bankSeparator, "⟦") {
|
||
t.Fatalf("separator lost its unusual delimiter: %q", bankSeparator)
|
||
}
|
||
}
|
||
|
||
// --- pack-20 (D39.42 п.4): the channel is a property of the TEXT, not of a config key ----------------
|
||
|
||
// TestBanknoteSliceIsUnconditional pins the fix for S5, the phase-1 BLOCKER. The slice used to be gated
|
||
// on gates.banknote.enabled, and that key is in no shipping config — so the literal reading of "move the
|
||
// block into translator.md" would have sent the raw ⟦TM-BANK-v1⟧ table into the draft, the editor and the
|
||
// EXPORT with nothing to catch it (the output sanitizer is final-stage-only). Slicing on the text itself
|
||
// removes the failure mode; a draft with no separator still takes the byte-identical early return.
|
||
func TestBanknoteSliceIsUnconditional(t *testing.T) {
|
||
r := &Runner{Pipeline: &config.Pipeline{}} // gate OFF (zero value)
|
||
const clean = "Тихое утро в библиотеке."
|
||
raw := clean + "\n" + bankSeparator + "\n龙公\tЛун Гун\tname"
|
||
|
||
got, stripped, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "stop")
|
||
if got != clean || stripped != clean {
|
||
t.Fatalf("the block must be sliced even with the gate off, got %q / %q", got, stripped)
|
||
}
|
||
if len(entries) != 1 || flags.NLines != 1 {
|
||
t.Fatalf("the WHAT must still be read (parsing is free), got %+v / %+v", entries, flags)
|
||
}
|
||
// The byte-identical path: no separator → the raw text back, no derived export.
|
||
if g, s, f, e := r.applyBanknoteWithEntries(roleTranslator, clean, "stop"); g != clean || s != "" || f != (bankFlags{}) || e != nil {
|
||
t.Fatalf("a draft with no separator must take the unchanged path, got %q/%q/%+v/%+v", g, s, f, e)
|
||
}
|
||
// The editor never emits banknotes — its output is never touched.
|
||
if g, s, _, _ := r.applyBanknoteWithEntries(roleEditor, raw, "stop"); g != raw || s != "" {
|
||
t.Fatalf("the editor's output must not be sliced, got %q/%q", g, s)
|
||
}
|
||
}
|
||
|
||
// TestBanknoteMalformedSeparatorIsCaught pins S12 — the very case §B3-1 was written for and the code
|
||
// never covered. A model that TRIES to open the channel and mistypes the marker used to have its table
|
||
// read as prose: it travelled into the editor and the export while banknote_parse_fail stayed 0.
|
||
func TestBanknoteMalformedSeparatorIsCaught(t *testing.T) {
|
||
r := &Runner{Pipeline: &config.Pipeline{}}
|
||
const clean = "Тихое утро в библиотеке."
|
||
for _, broken := range []string{
|
||
"⟦TM-BANK-v2⟧", // a version the parser does not know
|
||
"[TM-BANK-v1]", // the wrong brackets
|
||
"⟦ TM-BANK-v1 ⟧", // stray spaces
|
||
} {
|
||
raw := clean + "\n" + broken + "\n龙公\tЛун Гун\tname"
|
||
got, stripped, flags, entries := r.applyBanknoteWithEntries(roleTranslator, raw, "stop")
|
||
if strings.Contains(got, "龙公") || strings.Contains(got, "TM-BANK") {
|
||
t.Fatalf("%s: the malformed block must still be cut off the draft, got %q", broken, got)
|
||
}
|
||
if got != clean || stripped != clean {
|
||
t.Fatalf("%s: the cleaned draft must be the translation, got %q", broken, got)
|
||
}
|
||
if !flags.ParseFail {
|
||
t.Fatalf("%s: a malformed separator must be flagged, got %+v", broken, flags)
|
||
}
|
||
if len(entries) != 0 {
|
||
t.Fatalf("%s: a block we could not open is not evidence, got %+v", broken, entries)
|
||
}
|
||
}
|
||
// A translation that merely mentions nothing of the sort is untouched — no false positives.
|
||
if got, _, flags, _ := r.applyBanknoteWithEntries(roleTranslator, clean, "stop"); got != clean || flags.ParseFail {
|
||
t.Fatalf("ordinary prose must not trip the malformed-separator check: %q / %+v", got, flags)
|
||
}
|
||
}
|
||
|
||
// TestBankTokenBudgetIsDerived pins the D39.41 acceptance finding: the reservation must be COMPUTED from
|
||
// the line format under the engine's own estimator, not a round literal. The old `bankMaxLines * 12` was
|
||
// 144 against a measured worst block of 142.6–147.2 — at best exactly the need, at worst already negative.
|
||
func TestBankTokenBudgetIsDerived(t *testing.T) {
|
||
// A worst-case block: bankMaxLines full lines plus the separator.
|
||
var b strings.Builder
|
||
b.WriteString(bankSeparator + "\n")
|
||
for i := 0; i < bankMaxLines; i++ {
|
||
b.WriteString(strings.Repeat("蛊", 8) + "\t" + strings.Repeat("я", 40) + "\tnickname\n")
|
||
}
|
||
need := EstimateTokens(b.String())
|
||
if bankTokenBudget < need {
|
||
t.Fatalf("the reservation (%d) must cover a worst-case block (%d est-tokens) — a budget that is exactly the need truncates the channel on the densest chapters", bankTokenBudget, need)
|
||
}
|
||
// …and it must not be absurdly generous either: an unused ceiling is cheap, but the number should
|
||
// still be recognisable as this block's cost.
|
||
if bankTokenBudget > 2*need {
|
||
t.Fatalf("the reservation (%d) is more than twice the worst case (%d) — that is a guess again, not a derivation", bankTokenBudget, need)
|
||
}
|
||
if bankMaxLines <= 12 {
|
||
t.Fatalf("the line cap was raised on the mini-run measurement (2 of 9 blocks pressed against 12), got %d", bankMaxLines)
|
||
}
|
||
}
|