package pipeline import ( "strings" "testing" "textmachine/backend/internal/store" ) // TestTradTableCoversKnownCases asserts the embedded trad→simp table covers every // character referenced by the eval specs / research/14 live bugs. A mutation // (deleting a mapping line) fails exactly here — the guard against silently // regressing the A4 hole (爺→爷 was research/14's live "тихо пусто" bug). func TestTradTableCoversKnownCases(t *testing.T) { want := map[rune]rune{ '魯': '鲁', '鎮': '镇', '趙': '赵', '蕭': '萧', '煉': '炼', '門': '门', '闆': '板', '莊': '庄', '錢': '钱', '陳': '陈', '萬': '万', '舊': '旧', '臉': '脸', '爺': '爷', '羅': '罗', } for tr, si := range want { if got, ok := trad2simp[tr]; !ok || got != si { t.Errorf("trad2simp[%q] = %q, ok=%v; want %q", string(tr), string(got), ok, string(si)) } } if len(trad2simp) < 200 { t.Errorf("trad2simp table has only %d entries — expected the curated high-frequency set (≥200)", len(trad2simp)) } if memoryNormVersion == "" || !strings.HasPrefix(memoryNormVersion, memoryNormAlgoVersion) { t.Errorf("memoryNormVersion=%q must start with the algo tag %q", memoryNormVersion, memoryNormAlgoVersion) } } func TestNormalizeSourceKey(t *testing.T) { cases := []struct{ name, in, want string }{ {"trad→simp two-char", "魯鎮", "鲁镇"}, {"trad→simp name", "趙太爺", "赵太爷"}, {"fullwidth latin lowered", "阿Q", "阿q"}, {"halfwidth latin lowered", "阿Q", "阿q"}, {"katakana→hiragana", "スズキ", "すずき"}, {"halfwidth katakana via NFKC", "ススキ", "すすき"}, // NFKC folds half→full katakana, then kana-fold to hiragana {"plain han unchanged", "送灶", "送灶"}, {"mixed", "萧炎", "萧炎"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { if got := normalizeSourceKey(c.in); got != c.want { t.Errorf("normalizeSourceKey(%q) = %q; want %q", c.in, got, c.want) } }) } } // TestNormalizeSymmetry is the A4 invariant: a key written in one orthography // matches a chunk written in another, because normalization is applied to BOTH. func TestNormalizeSymmetry(t *testing.T) { // glossary key in Simplified, chunk text in Traditional key := normalizeSourceKey("鲁镇") // simplified text := normalizeSourceKey("魯鎮的冬天很冷。") // traditional if !strings.Contains(text, key) { t.Errorf("normalized simplified key %q not found in normalized traditional text %q", key, text) } // katakana name key vs hiragana surface k2 := normalizeSourceKey("スズキ") t2 := normalizeSourceKey("すずきさんが来た。") if !strings.Contains(t2, k2) { t.Errorf("normalized katakana key %q not found in normalized hiragana text %q", k2, t2) } } func TestNormalizeTargetForm(t *testing.T) { cases := []struct{ in, want string }{ {"А-кью", "а-кью"}, {"Вэйчжуан", "вэйчжуан"}, {"Тёмный страж", "темный страж"}, // NFC → lower → ё→е fold } for _, c := range cases { if got := normalizeTargetForm(c.in); got != c.want { t.Errorf("normalizeTargetForm(%q) = %q; want %q", c.in, got, c.want) } } // ё and е must fold to the SAME normalized form (symmetric — no false miss) if normalizeTargetForm("сёстры") != normalizeTargetForm("сестры") { t.Error("ё→е fold is not symmetric: сёстры vs сестры differ after normalization") } // self-review #2: internal whitespace runs (incl. newline/NBSP) collapse to one space. for _, v := range []string{"Бородатого\nВана", "Бородатого Вана", "Бородатого Вана"} { if got := normalizeTargetForm(v); got != "бородатого вана" { t.Errorf("whitespace not canonicalized: %q → %q", v, got) } } // self-review #8: dash/hyphen variants fold to ASCII '-'. for _, v := range []string{"А–кью", "А‑кью", "А−кью"} { if got := normalizeTargetForm(v); got != "а-кью" { t.Errorf("dash variant not unified: %q → %q", v, got) } } } func TestSignificantLen(t *testing.T) { cases := []struct { in string want int }{ {"阿q", 2}, // han + latin letter {"炎", 1}, // single han — banned as a standalone key {"气", 1}, // single han {"钱", 1}, // single han (research/14 homograph) {"送灶", 2}, // bigram — allowed {"すずき", 3}, // three kana {"q", 1}, // bare latin letter {"а-кью", 4}, // cyrillic letters, hyphen not counted: а к ь ю = 4 } for _, c := range cases { if got := significantLen(c.in); got != c.want { t.Errorf("significantLen(%q) = %d; want %d", c.in, got, c.want) } } } // TestNormalizeStripsDefaultIgnorable covers the workflow-confirmed A4 major: zero-width / // format / variation-selector code points survive NFKC/NFC and silently break a match. A // source key with an embedded ZWSP/joiner/BOM/VS must normalize identically to the clean // key (else the A4 "тихо пусто" hole re-opens); a target form with a mid-word joiner must // normalize identically to the clean form (else a correct rendering is a false MISS). // Reverting the isIgnorableForMatch strip in either normalizer fails this test. func TestNormalizeStripsDefaultIgnorable(t *testing.T) { // Explicit escapes via code points (the chars are invisible by nature — the bug being tested). zwsp := string(rune(0x200B)) // ZERO WIDTH SPACE zwj := string(rune(0x200D)) // ZERO WIDTH JOINER wj := string(rune(0x2060)) // WORD JOINER bom := string(rune(0xFEFF)) // BOM / ZWNBSP vs16 := string(rune(0xFE0F)) // VARIATION SELECTOR-16 shy := string(rune(0x00AD)) // SOFT HYPHEN (unicode.Cf) cleanSrc := normalizeSourceKey("渡邊") for _, in := range []string{ "渡" + zwsp + "邊", "渡" + zwj + "邊", "渡" + wj + "邊", "渡" + bom + "邊", "渡邊" + vs16, // VS decorating the final key char "渡" + shy + "邊", // soft hyphen dropped on the source side (no dash semantics) } { if got := normalizeSourceKey(in); got != cleanSrc { t.Errorf("source key %+q → %q, want %q (default-ignorable not stripped)", in, got, cleanSrc) } } cleanTgt := normalizeTargetForm("вана") for _, in := range []string{ "ва" + wj + "на", "ва" + zwsp + "на", "ва" + bom + "на", } { if got := normalizeTargetForm(in); got != cleanTgt { t.Errorf("target form %+q → %q, want %q (default-ignorable not stripped)", in, got, cleanTgt) } } // Regression guard for self-review #8 AND for the strip's ORDERING: the soft hyphen must // STILL fold to a real '-' on the target side (a hyphenation point), NOT be stripped — so // «А­кью» keeps matching a stored «А-кью». Placing the ignorable-strip before isDashVariant // would break this. if got, want := normalizeTargetForm("А"+shy+"кью"), normalizeTargetForm("А-кью"); got != want { t.Errorf("soft hyphen on the target side = %q, want %q (must fold to '-', not be stripped)", got, want) } } // TestApostropheFold covers the Task-6 re-audit finding: a typographic apostrophe (U+2019, // smart-quote editors) folds to ASCII on BOTH sides, so a source key O'Brien matches either // spelling (no тихо пусто) and the post-check does not false-MISS a д'Артаньян. Reverting the // isApostropheVariant folds breaks these. func TestApostropheFold(t *testing.T) { if normalizeSourceKey("O’Brien") != normalizeSourceKey("O'Brien") { t.Error("source: typographic and ASCII apostrophe must normalize identically") } if normalizeTargetForm("д’Артаньян") != normalizeTargetForm("д'Артаньян") { t.Error("target: typographic and ASCII apostrophe must normalize identically") } // End-to-end: an ASCII-apostrophe key matches a chunk written with a typographic apostrophe. e := gl("o'brien", "О'Брайен", "", "approved") b := bankFrom([]store.GlossaryEntry{e}) if _, ok := injMap(b.Select("mr o’brien arrived.", 1, nil, 0))["o'brien"]; !ok { t.Error("typographic apostrophe in the chunk must match the ASCII-apostrophe key") } }