186 lines
7.9 KiB
Go
186 lines
7.9 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// TestTradTableCoversKnownCases asserts the embedded trad→simp table covers every
|
||
// character referenced by the eval specs / research/14 live bugs. A mutation
|
||
// (deleting a mapping line) fails exactly here — the guard against silently
|
||
// regressing the A4 hole (爺→爷 was research/14's live "тихо пусто" bug).
|
||
func TestTradTableCoversKnownCases(t *testing.T) {
|
||
want := map[rune]rune{
|
||
'魯': '鲁', '鎮': '镇', '趙': '赵', '蕭': '萧', '煉': '炼',
|
||
'門': '门', '闆': '板', '莊': '庄', '錢': '钱', '陳': '陈',
|
||
'萬': '万', '舊': '旧', '臉': '脸', '爺': '爷', '羅': '罗',
|
||
}
|
||
for tr, si := range want {
|
||
if got, ok := trad2simp[tr]; !ok || got != si {
|
||
t.Errorf("trad2simp[%q] = %q, ok=%v; want %q", string(tr), string(got), ok, string(si))
|
||
}
|
||
}
|
||
if len(trad2simp) < 200 {
|
||
t.Errorf("trad2simp table has only %d entries — expected the curated high-frequency set (≥200)", len(trad2simp))
|
||
}
|
||
if memoryNormVersion == "" || !strings.HasPrefix(memoryNormVersion, memoryNormAlgoVersion) {
|
||
t.Errorf("memoryNormVersion=%q must start with the algo tag %q", memoryNormVersion, memoryNormAlgoVersion)
|
||
}
|
||
}
|
||
|
||
func TestNormalizeSourceKey(t *testing.T) {
|
||
cases := []struct{ name, in, want string }{
|
||
{"trad→simp two-char", "魯鎮", "鲁镇"},
|
||
{"trad→simp name", "趙太爺", "赵太爷"},
|
||
{"fullwidth latin lowered", "阿Ｑ", "阿q"},
|
||
{"halfwidth latin lowered", "阿Q", "阿q"},
|
||
{"katakana→hiragana", "スズキ", "すずき"},
|
||
{"halfwidth katakana via NFKC", "ｽｽｷ", "すすき"}, // NFKC folds half→full katakana, then kana-fold to hiragana
|
||
{"plain han unchanged", "送灶", "送灶"},
|
||
{"mixed", "萧炎", "萧炎"},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
if got := normalizeSourceKey(c.in); got != c.want {
|
||
t.Errorf("normalizeSourceKey(%q) = %q; want %q", c.in, got, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestNormalizeSymmetry is the A4 invariant: a key written in one orthography
|
||
// matches a chunk written in another, because normalization is applied to BOTH.
|
||
func TestNormalizeSymmetry(t *testing.T) {
|
||
// glossary key in Simplified, chunk text in Traditional
|
||
key := normalizeSourceKey("鲁镇") // simplified
|
||
text := normalizeSourceKey("魯鎮的冬天很冷。") // traditional
|
||
if !strings.Contains(text, key) {
|
||
t.Errorf("normalized simplified key %q not found in normalized traditional text %q", key, text)
|
||
}
|
||
// katakana name key vs hiragana surface
|
||
k2 := normalizeSourceKey("スズキ")
|
||
t2 := normalizeSourceKey("すずきさんが来た。")
|
||
if !strings.Contains(t2, k2) {
|
||
t.Errorf("normalized katakana key %q not found in normalized hiragana text %q", k2, t2)
|
||
}
|
||
}
|
||
|
||
func TestNormalizeTargetForm(t *testing.T) {
|
||
cases := []struct{ in, want string }{
|
||
{"А-кью", "а-кью"},
|
||
{"Вэйчжуан", "вэйчжуан"},
|
||
{"Тёмный страж", "темный страж"}, // NFC → lower → ё→е fold
|
||
}
|
||
for _, c := range cases {
|
||
if got := normalizeTargetForm(c.in); got != c.want {
|
||
t.Errorf("normalizeTargetForm(%q) = %q; want %q", c.in, got, c.want)
|
||
}
|
||
}
|
||
// ё and е must fold to the SAME normalized form (symmetric — no false miss)
|
||
if normalizeTargetForm("сёстры") != normalizeTargetForm("сестры") {
|
||
t.Error("ё→е fold is not symmetric: сёстры vs сестры differ after normalization")
|
||
}
|
||
// self-review #2: internal whitespace runs (incl. newline/NBSP) collapse to one space.
|
||
for _, v := range []string{"Бородатого\nВана", "Бородатого Вана", "Бородатого Вана"} {
|
||
if got := normalizeTargetForm(v); got != "бородатого вана" {
|
||
t.Errorf("whitespace not canonicalized: %q → %q", v, got)
|
||
}
|
||
}
|
||
// self-review #8: dash/hyphen variants fold to ASCII '-'.
|
||
for _, v := range []string{"А–кью", "А‑кью", "А−кью"} {
|
||
if got := normalizeTargetForm(v); got != "а-кью" {
|
||
t.Errorf("dash variant not unified: %q → %q", v, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestSignificantLen(t *testing.T) {
|
||
cases := []struct {
|
||
in string
|
||
want int
|
||
}{
|
||
{"阿q", 2}, // han + latin letter
|
||
{"炎", 1}, // single han — banned as a standalone key
|
||
{"气", 1}, // single han
|
||
{"钱", 1}, // single han (research/14 homograph)
|
||
{"送灶", 2}, // bigram — allowed
|
||
{"すずき", 3}, // three kana
|
||
{"q", 1}, // bare latin letter
|
||
{"а-кью", 4}, // cyrillic letters, hyphen not counted: а к ь ю = 4
|
||
}
|
||
for _, c := range cases {
|
||
if got := significantLen(c.in); got != c.want {
|
||
t.Errorf("significantLen(%q) = %d; want %d", c.in, got, c.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestNormalizeStripsDefaultIgnorable covers the workflow-confirmed A4 major: zero-width /
|
||
// format / variation-selector code points survive NFKC/NFC and silently break a match. A
|
||
// source key with an embedded ZWSP/joiner/BOM/VS must normalize identically to the clean
|
||
// key (else the A4 "тихо пусто" hole re-opens); a target form with a mid-word joiner must
|
||
// normalize identically to the clean form (else a correct rendering is a false MISS).
|
||
// Reverting the isIgnorableForMatch strip in either normalizer fails this test.
|
||
func TestNormalizeStripsDefaultIgnorable(t *testing.T) {
|
||
// Explicit escapes via code points (the chars are invisible by nature — the bug being tested).
|
||
zwsp := string(rune(0x200B)) // ZERO WIDTH SPACE
|
||
zwj := string(rune(0x200D)) // ZERO WIDTH JOINER
|
||
wj := string(rune(0x2060)) // WORD JOINER
|
||
bom := string(rune(0xFEFF)) // BOM / ZWNBSP
|
||
vs16 := string(rune(0xFE0F)) // VARIATION SELECTOR-16
|
||
shy := string(rune(0x00AD)) // SOFT HYPHEN (unicode.Cf)
|
||
|
||
cleanSrc := normalizeSourceKey("渡邊")
|
||
for _, in := range []string{
|
||
"渡" + zwsp + "邊",
|
||
"渡" + zwj + "邊",
|
||
"渡" + wj + "邊",
|
||
"渡" + bom + "邊",
|
||
"渡邊" + vs16, // VS decorating the final key char
|
||
"渡" + shy + "邊", // soft hyphen dropped on the source side (no dash semantics)
|
||
} {
|
||
if got := normalizeSourceKey(in); got != cleanSrc {
|
||
t.Errorf("source key %+q → %q, want %q (default-ignorable not stripped)", in, got, cleanSrc)
|
||
}
|
||
}
|
||
|
||
cleanTgt := normalizeTargetForm("вана")
|
||
for _, in := range []string{
|
||
"ва" + wj + "на",
|
||
"ва" + zwsp + "на",
|
||
"ва" + bom + "на",
|
||
} {
|
||
if got := normalizeTargetForm(in); got != cleanTgt {
|
||
t.Errorf("target form %+q → %q, want %q (default-ignorable not stripped)", in, got, cleanTgt)
|
||
}
|
||
}
|
||
|
||
// Regression guard for self-review #8 AND for the strip's ORDERING: the soft hyphen must
|
||
// STILL fold to a real '-' on the target side (a hyphenation point), NOT be stripped — so
|
||
// «Акью» keeps matching a stored «А-кью». Placing the ignorable-strip before isDashVariant
|
||
// would break this.
|
||
if got, want := normalizeTargetForm("А"+shy+"кью"), normalizeTargetForm("А-кью"); got != want {
|
||
t.Errorf("soft hyphen on the target side = %q, want %q (must fold to '-', not be stripped)", got, want)
|
||
}
|
||
}
|
||
|
||
// TestApostropheFold covers the Task-6 re-audit finding: a typographic apostrophe (U+2019,
|
||
// smart-quote editors) folds to ASCII on BOTH sides, so a source key O'Brien matches either
|
||
// spelling (no тихо пусто) and the post-check does not false-MISS a д'Артаньян. Reverting the
|
||
// isApostropheVariant folds breaks these.
|
||
func TestApostropheFold(t *testing.T) {
|
||
if normalizeSourceKey("O’Brien") != normalizeSourceKey("O'Brien") {
|
||
t.Error("source: typographic and ASCII apostrophe must normalize identically")
|
||
}
|
||
if normalizeTargetForm("д’Артаньян") != normalizeTargetForm("д'Артаньян") {
|
||
t.Error("target: typographic and ASCII apostrophe must normalize identically")
|
||
}
|
||
// End-to-end: an ASCII-apostrophe key matches a chunk written with a typographic apostrophe.
|
||
e := gl("o'brien", "О'Брайен", "", "approved")
|
||
b := bankFrom([]store.GlossaryEntry{e})
|
||
if _, ok := injMap(b.Select("mr o’brien arrived.", 1, nil, 0))["o'brien"]; !ok {
|
||
t.Error("typographic apostrophe in the chunk must match the ASCII-apostrophe key")
|
||
}
|
||
}
|