package pipeline import ( "strings" "testing" ) // TestTradTableCoversKnownCases asserts the embedded trad→simp table covers every // character referenced by the eval specs / research/14 live bugs. A mutation // (deleting a mapping line) fails exactly here — the guard against silently // regressing the A4 hole (爺→爷 was research/14's live "тихо пусто" bug). func TestTradTableCoversKnownCases(t *testing.T) { want := map[rune]rune{ '魯': '鲁', '鎮': '镇', '趙': '赵', '蕭': '萧', '煉': '炼', '門': '门', '闆': '板', '莊': '庄', '錢': '钱', '陳': '陈', '萬': '万', '舊': '旧', '臉': '脸', '爺': '爷', '羅': '罗', } for tr, si := range want { if got, ok := trad2simp[tr]; !ok || got != si { t.Errorf("trad2simp[%q] = %q, ok=%v; want %q", string(tr), string(got), ok, string(si)) } } if len(trad2simp) < 200 { t.Errorf("trad2simp table has only %d entries — expected the curated high-frequency set (≥200)", len(trad2simp)) } if memoryNormVersion == "" || !strings.HasPrefix(memoryNormVersion, memoryNormAlgoVersion) { t.Errorf("memoryNormVersion=%q must start with the algo tag %q", memoryNormVersion, memoryNormAlgoVersion) } } func TestNormalizeSourceKey(t *testing.T) { cases := []struct{ name, in, want string }{ {"trad→simp two-char", "魯鎮", "鲁镇"}, {"trad→simp name", "趙太爺", "赵太爷"}, {"fullwidth latin lowered", "阿Q", "阿q"}, {"halfwidth latin lowered", "阿Q", "阿q"}, {"katakana→hiragana", "スズキ", "すずき"}, {"halfwidth katakana via NFKC", "ススキ", "すすき"}, // NFKC folds half→full katakana, then kana-fold to hiragana {"plain han unchanged", "送灶", "送灶"}, {"mixed", "萧炎", "萧炎"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { if got := normalizeSourceKey(c.in); got != c.want { t.Errorf("normalizeSourceKey(%q) = %q; want %q", c.in, got, c.want) } }) } } // TestNormalizeSymmetry is the A4 invariant: a key written in one orthography // matches a chunk written in another, because normalization is applied to BOTH. func TestNormalizeSymmetry(t *testing.T) { // glossary key in Simplified, chunk text in Traditional key := normalizeSourceKey("鲁镇") // simplified text := normalizeSourceKey("魯鎮的冬天很冷。") // traditional if !strings.Contains(text, key) { t.Errorf("normalized simplified key %q not found in normalized traditional text %q", key, text) } // katakana name key vs hiragana surface k2 := normalizeSourceKey("スズキ") t2 := normalizeSourceKey("すずきさんが来た。") if !strings.Contains(t2, k2) { t.Errorf("normalized katakana key %q not found in normalized hiragana text %q", k2, t2) } } func TestNormalizeTargetForm(t *testing.T) { cases := []struct{ in, want string }{ {"А-кью", "а-кью"}, {"Вэйчжуан", "вэйчжуан"}, {"Тёмный страж", "темный страж"}, // NFC → lower → ё→е fold } for _, c := range cases { if got := normalizeTargetForm(c.in); got != c.want { t.Errorf("normalizeTargetForm(%q) = %q; want %q", c.in, got, c.want) } } // ё and е must fold to the SAME normalized form (symmetric — no false miss) if normalizeTargetForm("сёстры") != normalizeTargetForm("сестры") { t.Error("ё→е fold is not symmetric: сёстры vs сестры differ after normalization") } // self-review #2: internal whitespace runs (incl. newline/NBSP) collapse to one space. for _, v := range []string{"Бородатого\nВана", "Бородатого Вана", "Бородатого Вана"} { if got := normalizeTargetForm(v); got != "бородатого вана" { t.Errorf("whitespace not canonicalized: %q → %q", v, got) } } // self-review #8: dash/hyphen variants fold to ASCII '-'. for _, v := range []string{"А–кью", "А‑кью", "А−кью"} { if got := normalizeTargetForm(v); got != "а-кью" { t.Errorf("dash variant not unified: %q → %q", v, got) } } } func TestSignificantLen(t *testing.T) { cases := []struct { in string want int }{ {"阿q", 2}, // han + latin letter {"炎", 1}, // single han — banned as a standalone key {"气", 1}, // single han {"钱", 1}, // single han (research/14 homograph) {"送灶", 2}, // bigram — allowed {"すずき", 3}, // three kana {"q", 1}, // bare latin letter {"а-кью", 4}, // cyrillic letters, hyphen not counted: а к ь ю = 4 } for _, c := range cases { if got := significantLen(c.in); got != c.want { t.Errorf("significantLen(%q) = %d; want %d", c.in, got, c.want) } } }