package lang import ( "os" "path/filepath" "strings" "testing" ) // realRoot is the in-repo langpack root, relative to this package's test cwd (internal/lang/). const realRoot = "../../configs/langpacks" // TestLoadResolvesRealZhRu loads the REAL zh→ru pack from disk and pins a spot-check of the moved data // (the const→file move must be byte-faithful; the full-book parity that pins the exact miner output is // the stand-gated TestMinerFullBookParity, so this is the CI-runnable anchor). It exercises the real // disk-loading path end to end. func TestLoadResolvesRealZhRu(t *testing.T) { p, err := Load(realRoot, "zh", "ru") if err != nil { t.Fatalf("Load(zh,ru): %v", err) } if p.Pair != "zh-ru" { t.Errorf("Pair = %q, want zh-ru", p.Pair) } // Source morphology spot-check (byte-faithful move). if !p.SurnamesSingle['赵'] || !p.SurnamesSingle['方'] { t.Error("surnames-single missing 赵/方") } if p.SurnamesSingle['凝'] { t.Error("surnames-single must NOT contain 凝 (discarded — not a surname)") } if !p.SurnamesCompound["欧阳"] { t.Error("surnames-compound missing 欧阳") } if p.SurnamesCompound["古月"] { t.Error("surnames-compound must NOT contain 古月 (pair-14 §1: a 蛊真人 clan, moved to the book overlay)") } if len(p.TitleSuffix) == 0 || p.TitleSuffix[0] != "公子" { t.Errorf("title-suffix order not preserved: %v", p.TitleSuffix) } if !p.TopoSuffix['山'] || !p.Numeral['三'] || !p.GradePrefix['甲'] || !p.AliasParticle['的'] { t.Error("rune-set membership missing an expected char (山/三/甲/的)") } // pair-14 moves: title-formant + sentence-terminator rune sets, and the Palladius phonotactic sets. if !p.TitleFormant['等'] || !p.TitleFormant['转'] || !p.TitleFormant['阶'] { t.Error("title-formant missing 等/转/阶") } if !p.SentenceTerminator['。'] || !p.SentenceTerminator['!'] || !p.SentenceTerminator['?'] { t.Error("sentence-terminator missing 。/!/?") } if !p.Palladius.Retroflex["zh"] || !p.Palladius.VFinal["v"] || !p.Palladius.VFinalInitial["j"] { t.Error("palladius phonotactics missing retroflex zh / vfinal v / vfinal_initial j") } // Pair transliteration spot-check. if p.Palladius.Initials["b"] != "б" || p.Palladius.Initials["zh"] != "чж" { t.Errorf("palladius initials wrong: b=%q zh=%q", p.Palladius.Initials["b"], p.Palladius.Initials["zh"]) } if p.Palladius.SpecialI["zhi"] != "чжи" { t.Errorf("palladius special_i zhi = %q, want чжи", p.Palladius.SpecialI["zhi"]) } if !strings.HasPrefix(p.Version(), packAlgoVersion+"-") { t.Errorf("Version() = %q, want %s-", p.Version(), packAlgoVersion) } } // TestRoutesByPairToDifferentBytes is the load-bearing generality proof: a SECOND pair, dropped as data // (no pipeline/ edit, no recompile), routes to DIFFERENT bytes and a DIFFERENT version. Proves "add a // language = data only" — the pair is a genuine routing key, not a baked-in zh constant. func TestRoutesByPairToDifferentBytes(t *testing.T) { root := t.TempDir() writeSyntheticPack(t, root, "xx", "yy") real, err := Load(realRoot, "zh", "ru") if err != nil { t.Fatalf("load real: %v", err) } synth, err := Load(root, "xx", "yy") if err != nil { t.Fatalf("load synthetic xx-yy: %v", err) } // The synthetic pair carries its OWN surnames (not zh's), proving the resolver routes by pair, not // to a hardcoded set. if !synth.SurnamesSingle['甴'] { t.Error("synthetic pack must carry its own surname 甴") } if synth.SurnamesSingle['赵'] { t.Error("synthetic pack must NOT inherit zh's surname 赵 (would mean a baked-in constant)") } if !real.SurnamesSingle['赵'] || real.SurnamesSingle['甴'] { t.Error("real zh pack routing leaked/borrowed synthetic bytes") } if real.Version() == synth.Version() { t.Errorf("different-byte packs must have different versions (real=%s synth=%s)", real.Version(), synth.Version()) } } // TestBookOverlayUnionsAndReVersions pins the pair-14 §1 book-overlay contract: an overlay UNIONS its // private canon onto the shared pack (additive — the base entries survive, the overlay entry is added) and // SHIFTS Version() (a book-canon edit is a loud --resnapshot for that book), while a no-overlay Load is // byte-identical to before. Guards the exact mechanism the miner's {古月:22} parity rides. func TestBookOverlayUnionsAndReVersions(t *testing.T) { base, err := Load(realRoot, "zh", "ru") if err != nil { t.Fatalf("load base: %v", err) } if base.SurnamesCompound["古月"] { t.Fatal("shared pack must not carry 古月 (it is a book clan)") } overlay := t.TempDir() if err := os.MkdirAll(filepath.Join(overlay, "zh"), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(overlay, "zh", "surnames-compound.txt"), []byte("# book canon\n古月\n"), 0o644); err != nil { t.Fatal(err) } ext, err := LoadWithOverlay(realRoot, "zh", "ru", overlay) if err != nil { t.Fatalf("load with overlay: %v", err) } if !ext.SurnamesCompound["古月"] { t.Error("overlay must add 古月 to the effective pack") } if !ext.SurnamesCompound["欧阳"] { t.Error("overlay must UNION (keep the shared 欧阳), not replace") } if ext.Version() == base.Version() { t.Errorf("overlay must shift Version() (base=%s overlay=%s)", base.Version(), ext.Version()) } // An empty overlayRoot is exactly Load — same version, no re-hash. same, err := LoadWithOverlay(realRoot, "zh", "ru", "") if err != nil { t.Fatal(err) } if same.Version() != base.Version() { t.Errorf("empty overlay must equal Load (%s != %s)", same.Version(), base.Version()) } // A MISNAMED overlay file must FAIL LOUD, not be silently ignored (review finding, scale lens): a typo'd // canon file that never reaches the miner would degrade recall with no signal. bad := t.TempDir() if err := os.MkdirAll(filepath.Join(bad, "zh"), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(bad, "zh", "surname-compound.txt"), []byte("古月\n"), 0o644); err != nil { // typo: missing 's' t.Fatal(err) } if _, err := LoadWithOverlay(realRoot, "zh", "ru", bad); err == nil { t.Error("a misnamed overlay file must fail loud (silently-ignored canon would degrade recall)") } else if !strings.Contains(err.Error(), "surname-compound.txt") { t.Errorf("error must name the unexpected file, got: %v", err) } } // TestFailsLoudOnMissingPair pins the fail-loud contract: a pair with no pack directory errors, naming the // missing file — never a silent empty pack (mirrors the prompt seam's PromptPathFor fail-loud). This is // the load-time invariant a live consumer relies on (fail before billing). func TestFailsLoudOnMissingPair(t *testing.T) { _, err := Load(realRoot, "ja", "ru") if err == nil { t.Fatal("Load(ja,ru) must fail loud (no ja langpack), got nil error") } if !strings.Contains(err.Error(), "ja-ru") { t.Errorf("error must name the missing pair ja-ru, got: %v", err) } } // TestFailsLoudOnEmptyTable pins the "never silently empty" contract: a present-but-empty (comment-only) // required file is a corrupt pack and must fail loud at load — not parse to an empty set that silently // disables a miner channel once packs are hand-authored (R1). func TestFailsLoudOnEmptyTable(t *testing.T) { root := t.TempDir() writeSyntheticPack(t, root, "xx", "yy") // Blank out one required table (keep the file present, drop its content). if err := os.WriteFile(filepath.Join(root, "xx", "surnames-single.txt"), []byte("# emptied by a bad edit\n"), 0o644); err != nil { t.Fatal(err) } _, err := Load(root, "xx", "yy") if err == nil { t.Fatal("an empty required table must fail loud, got nil error") } if !strings.Contains(err.Error(), "surnames-single") { t.Errorf("error must name the empty table, got: %v", err) } } // writeSyntheticPack writes a minimal, VALID pack for (src, tgt) under root with bytes distinct from zh — // enough for Load to succeed and for the routing assertion to bite. func writeSyntheticPack(t *testing.T, root, src, tgt string) { t.Helper() pair := src + "-" + tgt files := map[string]string{ filepath.Join(src, "surnames-single.txt"): "# synthetic\n甴甶甹\n", filepath.Join(src, "surnames-compound.txt"): "# synthetic\n甲乙\n", filepath.Join(src, "title-suffix.txt"): "# synthetic\n阁下\n", filepath.Join(src, "ordinal-title.txt"): "# synthetic\n第甲\n", filepath.Join(src, "rank-word.txt"): "# synthetic\n級\n", filepath.Join(src, "topo-suffix.txt"): "# synthetic\n峰\n", filepath.Join(src, "grade-prefix.txt"): "# synthetic\n子丑\n", filepath.Join(src, "numeral.txt"): "# synthetic\n壹貳\n", filepath.Join(src, "alias-particle.txt"): "# synthetic\n之乎\n", filepath.Join(src, "title-formant.txt"): "# synthetic\n甼\n", filepath.Join(src, "sentence-terminator.txt"): "# synthetic\n。\n", filepath.Join(pair, "palladius.txt"): "# synthetic\ninitials\tb\tб\nfinals\ta\tа\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n", filepath.Join(pair, "palladius-phonotactics.txt"): "# synthetic\nretroflex\tzh\nvfinal\tv\nvfinal_initial\tj\n", } for rel, body := range files { p := filepath.Join(root, rel) if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(p, []byte(body), 0o644); err != nil { t.Fatal(err) } } } // --- pack-15: the overlay's fail-loud one level UP + the Palladius category contract ----------------- // TestOverlayRootFailsLoud pins the pack-15 guards ABOVE the per-file check: langpack_extend is an // explicit opt-in, so a path that does not exist (a typo, a moved book directory) must STOP the load // rather than degrade recall in silence, and an overlay root may hold nothing but its two expected // subdirectories (a data file dropped at the root, or a mistyped pair dir, was silently ignored before). func TestOverlayRootFailsLoud(t *testing.T) { t.Run("missing root", func(t *testing.T) { _, err := LoadWithOverlay(realRoot, "zh", "ru", filepath.Join(t.TempDir(), "nope")) if err == nil { t.Fatal("a missing overlay root must fail loud (explicit opt-in — a typo would silently drop the book canon)") } if !strings.Contains(err.Error(), "nope") { t.Errorf("error must name the path it looked for, got: %v", err) } }) t.Run("file at the root", func(t *testing.T) { root := t.TempDir() if err := os.MkdirAll(filepath.Join(root, "zh"), 0o755); err != nil { t.Fatal(err) } // The canon file dropped at the ROOT instead of inside zh/ — the fold never looks here. if err := os.WriteFile(filepath.Join(root, "surnames-compound.txt"), []byte("古月\n"), 0o644); err != nil { t.Fatal(err) } if _, err := LoadWithOverlay(realRoot, "zh", "ru", root); err == nil { t.Error("a data file at the overlay root must fail loud (it is silently ignored otherwise)") } else if !strings.Contains(err.Error(), "surnames-compound.txt") { t.Errorf("error must name the unexpected entry, got: %v", err) } }) t.Run("mistyped pair directory", func(t *testing.T) { root := t.TempDir() if err := os.MkdirAll(filepath.Join(root, "zh-rus"), 0o755); err != nil { // typo: zh-rus t.Fatal(err) } if _, err := LoadWithOverlay(realRoot, "zh", "ru", root); err == nil { t.Error("a mistyped pair directory must fail loud") } else if !strings.Contains(err.Error(), "zh-rus") { t.Errorf("error must name the unexpected directory, got: %v", err) } }) } // TestOverlayKeyCollisionFailsLoud pins D39.23: an overlay may only ADD. A Palladius key the shared pair // pack already defines would have been silently overridden by the overlay's value — override is not // ratified, so the collision is loud and names the key and both values. func TestOverlayKeyCollisionFailsLoud(t *testing.T) { root := t.TempDir() writeSyntheticPack(t, root, "xx", "yy") overlay := t.TempDir() if err := os.MkdirAll(filepath.Join(overlay, "xx-yy"), 0o755); err != nil { t.Fatal(err) } // `initials b` is already б in the base pack; the overlay redefines it as п. if err := os.WriteFile(filepath.Join(overlay, "xx-yy", "palladius.txt"), []byte("initials\tb\tп\n"), 0o644); err != nil { t.Fatal(err) } _, err := LoadWithOverlay(root, "xx", "yy", overlay) if err == nil { t.Fatal("a base↔overlay key collision must fail loud (override is not ratified — D39.23)") } for _, want := range []string{"initials", `"b"`, "б", "п"} { if !strings.Contains(err.Error(), want) { t.Errorf("collision error must mention %q, got: %v", want, err) } } // An overlay that only ADDS a new key is fine (the additive contract still holds). ok := t.TempDir() if err := os.MkdirAll(filepath.Join(ok, "xx-yy"), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(ok, "xx-yy", "palladius.txt"), []byte("initials\tp\tп\n"), 0o644); err != nil { t.Fatal(err) } p, err := LoadWithOverlay(root, "xx", "yy", ok) if err != nil { t.Fatalf("an additive overlay must load: %v", err) } if p.Palladius.Initials["b"] != "б" || p.Palladius.Initials["p"] != "п" { t.Errorf("additive overlay must keep the base key and add the new one: %v", p.Palladius.Initials) } } // TestPalladiusParserToleratesUnknownCategory pins the owner addendum (24.07): the GENERIC parser is // category-agnostic — an unknown category is data it does not interpret, never a parse error. Which // categories are required (and which are consumed) is the CONSUMER's call, asserted separately below. func TestPalladiusParserToleratesUnknownCategory(t *testing.T) { cats, err := parseCategoryRows([]byte("initials\tb\tб\nfuture_channel\tx\ty\nretroflex\tzh\n")) if err != nil { t.Fatalf("the generic parser must not reject an unknown category: %v", err) } if cats["initials"]["b"] != "б" || cats["retroflex"]["zh"] != "" || cats["future_channel"]["x"] != "y" { t.Errorf("parser must return every category verbatim, got %v", cats) } // A malformed ROW is still an error (the parser's own contract), naming the physical line. if _, err := parseCategoryRows([]byte("initials\n")); err == nil { t.Error("a row with too few fields must fail loud") } } // TestPalladiusRequiredCategoriesFailLoud pins the consumer half: a required category missing from the // data (here: emptied) is caught by validate() at load, and a category NOTHING consumes (a typo like // `initals`) is caught by the consumer's merge — the two failure modes the generic parser deliberately // does not judge. func TestPalladiusRequiredCategoriesFailLoud(t *testing.T) { t.Run("required category missing", func(t *testing.T) { root := t.TempDir() writeSyntheticPack(t, root, "xx", "yy") // Drop the `finals` category (keep the file, keep the other categories). if err := os.WriteFile(filepath.Join(root, "xx-yy", "palladius.txt"), []byte("initials\tb\tб\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n"), 0o644); err != nil { t.Fatal(err) } _, err := Load(root, "xx", "yy") if err == nil { t.Fatal("a missing REQUIRED Palladius category must fail loud at load (validate)") } if !strings.Contains(err.Error(), "palladius/finals") { t.Errorf("error must name the empty required table, got: %v", err) } }) t.Run("typo'd category is not consumed", func(t *testing.T) { root := t.TempDir() writeSyntheticPack(t, root, "xx", "yy") if err := os.WriteFile(filepath.Join(root, "xx-yy", "palladius.txt"), []byte("initals\tb\tб\nfinals\ta\tа\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n"), 0o644); err != nil { // typo: initals t.Fatal(err) } _, err := Load(root, "xx", "yy") if err == nil { t.Fatal("a category nothing consumes must fail loud (it would silently leave its table empty)") } if !strings.Contains(err.Error(), "initals") { t.Errorf("error must name the unconsumed category, got: %v", err) } }) } // TestTerminatorClassesAreOneSource pins the pack-15 dedup: the source chunker and the coverage gate no // longer keep a private copy each of the terminator switches — both read these classes. The CJK class // splits zero-width; the generic class needs whitespace. A rune of either class is a terminator. func TestTerminatorClassesAreOneSource(t *testing.T) { term := DefaultTerminators() for _, r := range []rune{'。', '!', '?'} { if !term.IsCJK(r) || !term.IsTerminator(r) { t.Errorf("%q must be a CJK terminator", string(r)) } } for _, r := range []rune{'.', '!', '?', '…'} { if term.IsCJK(r) { t.Errorf("%q must NOT be zero-width-splitting (the ellipsis/ASCII class needs whitespace)", string(r)) } if !term.IsTerminator(r) { t.Errorf("%q must be a terminator", string(r)) } } for _, r := range []rune{',', '、', 'a', '「'} { if term.IsTerminator(r) { t.Errorf("%q must not be a terminator", string(r)) } } } // TestChapterMarkerIsData pins the pack-15 move of 第 out of the ingest regex literal into cjk-section: // the marker is data beside the numerals it precedes. func TestChapterMarkerIsData(t *testing.T) { sec := DefaultCJKSection() if got := sec.ChapterMarkerClass(); got != "第" { t.Errorf("chapter marker class = %q, want 第", got) } }