116 lines
4.6 KiB
Go
116 lines
4.6 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// TestTradTableCoversKnownCases asserts the embedded trad→simp table covers every
|
||
// character referenced by the eval specs / research/14 live bugs. A mutation
|
||
// (deleting a mapping line) fails exactly here — the guard against silently
|
||
// regressing the A4 hole (爺→爷 was research/14's live "тихо пусто" bug).
|
||
func TestTradTableCoversKnownCases(t *testing.T) {
|
||
want := map[rune]rune{
|
||
'魯': '鲁', '鎮': '镇', '趙': '赵', '蕭': '萧', '煉': '炼',
|
||
'門': '门', '闆': '板', '莊': '庄', '錢': '钱', '陳': '陈',
|
||
'萬': '万', '舊': '旧', '臉': '脸', '爺': '爷', '羅': '罗',
|
||
}
|
||
for tr, si := range want {
|
||
if got, ok := trad2simp[tr]; !ok || got != si {
|
||
t.Errorf("trad2simp[%q] = %q, ok=%v; want %q", string(tr), string(got), ok, string(si))
|
||
}
|
||
}
|
||
if len(trad2simp) < 200 {
|
||
t.Errorf("trad2simp table has only %d entries — expected the curated high-frequency set (≥200)", len(trad2simp))
|
||
}
|
||
if memoryNormVersion == "" || !strings.HasPrefix(memoryNormVersion, memoryNormAlgoVersion) {
|
||
t.Errorf("memoryNormVersion=%q must start with the algo tag %q", memoryNormVersion, memoryNormAlgoVersion)
|
||
}
|
||
}
|
||
|
||
func TestNormalizeSourceKey(t *testing.T) {
|
||
cases := []struct{ name, in, want string }{
|
||
{"trad→simp two-char", "魯鎮", "鲁镇"},
|
||
{"trad→simp name", "趙太爺", "赵太爷"},
|
||
{"fullwidth latin lowered", "阿Ｑ", "阿q"},
|
||
{"halfwidth latin lowered", "阿Q", "阿q"},
|
||
{"katakana→hiragana", "スズキ", "すずき"},
|
||
{"halfwidth katakana via NFKC", "ｽｽｷ", "すすき"}, // NFKC folds half→full katakana, then kana-fold to hiragana
|
||
{"plain han unchanged", "送灶", "送灶"},
|
||
{"mixed", "萧炎", "萧炎"},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
if got := normalizeSourceKey(c.in); got != c.want {
|
||
t.Errorf("normalizeSourceKey(%q) = %q; want %q", c.in, got, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestNormalizeSymmetry is the A4 invariant: a key written in one orthography
|
||
// matches a chunk written in another, because normalization is applied to BOTH.
|
||
func TestNormalizeSymmetry(t *testing.T) {
|
||
// glossary key in Simplified, chunk text in Traditional
|
||
key := normalizeSourceKey("鲁镇") // simplified
|
||
text := normalizeSourceKey("魯鎮的冬天很冷。") // traditional
|
||
if !strings.Contains(text, key) {
|
||
t.Errorf("normalized simplified key %q not found in normalized traditional text %q", key, text)
|
||
}
|
||
// katakana name key vs hiragana surface
|
||
k2 := normalizeSourceKey("スズキ")
|
||
t2 := normalizeSourceKey("すずきさんが来た。")
|
||
if !strings.Contains(t2, k2) {
|
||
t.Errorf("normalized katakana key %q not found in normalized hiragana text %q", k2, t2)
|
||
}
|
||
}
|
||
|
||
func TestNormalizeTargetForm(t *testing.T) {
|
||
cases := []struct{ in, want string }{
|
||
{"А-кью", "а-кью"},
|
||
{"Вэйчжуан", "вэйчжуан"},
|
||
{"Тёмный страж", "темный страж"}, // NFC → lower → ё→е fold
|
||
}
|
||
for _, c := range cases {
|
||
if got := normalizeTargetForm(c.in); got != c.want {
|
||
t.Errorf("normalizeTargetForm(%q) = %q; want %q", c.in, got, c.want)
|
||
}
|
||
}
|
||
// ё and е must fold to the SAME normalized form (symmetric — no false miss)
|
||
if normalizeTargetForm("сёстры") != normalizeTargetForm("сестры") {
|
||
t.Error("ё→е fold is not symmetric: сёстры vs сестры differ after normalization")
|
||
}
|
||
// self-review #2: internal whitespace runs (incl. newline/NBSP) collapse to one space.
|
||
for _, v := range []string{"Бородатого\nВана", "Бородатого Вана", "Бородатого Вана"} {
|
||
if got := normalizeTargetForm(v); got != "бородатого вана" {
|
||
t.Errorf("whitespace not canonicalized: %q → %q", v, got)
|
||
}
|
||
}
|
||
// self-review #8: dash/hyphen variants fold to ASCII '-'.
|
||
for _, v := range []string{"А–кью", "А‑кью", "А−кью"} {
|
||
if got := normalizeTargetForm(v); got != "а-кью" {
|
||
t.Errorf("dash variant not unified: %q → %q", v, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestSignificantLen(t *testing.T) {
|
||
cases := []struct {
|
||
in string
|
||
want int
|
||
}{
|
||
{"阿q", 2}, // han + latin letter
|
||
{"炎", 1}, // single han — banned as a standalone key
|
||
{"气", 1}, // single han
|
||
{"钱", 1}, // single han (research/14 homograph)
|
||
{"送灶", 2}, // bigram — allowed
|
||
{"すずき", 3}, // three kana
|
||
{"q", 1}, // bare latin letter
|
||
{"а-кью", 4}, // cyrillic letters, hyphen not counted: а к ь ю = 4
|
||
}
|
||
for _, c := range cases {
|
||
if got := significantLen(c.in); got != c.want {
|
||
t.Errorf("significantLen(%q) = %d; want %d", c.in, got, c.want)
|
||
}
|
||
}
|
||
}
|