textmachine/backend/internal/lang/langpack_test.go

package lang

import (
	"os"
	"path/filepath"
	"strings"
	"testing"
)

// realRoot is the in-repo langpack root, relative to this package's test cwd (internal/lang/).
const realRoot = "../../configs/langpacks"

// TestLoadResolvesRealZhRu loads the REAL zh→ru pack from disk and pins a spot-check of the moved data
// (the const→file move must be byte-faithful; the full-book parity that pins the exact miner output is
// the stand-gated TestMinerFullBookParity, so this is the CI-runnable anchor). It exercises the real
// disk-loading path end to end.
func TestLoadResolvesRealZhRu(t *testing.T) {
	p, err := Load(realRoot, "zh", "ru")
	if err != nil {
		t.Fatalf("Load(zh,ru): %v", err)
	}
	if p.Pair != "zh-ru" {
		t.Errorf("Pair = %q, want zh-ru", p.Pair)
	}
	// Source morphology spot-check (byte-faithful move).
	if !p.SurnamesSingle['赵'] || !p.SurnamesSingle['方'] {
		t.Error("surnames-single missing 赵/方")
	}
	if p.SurnamesSingle['凝'] {
		t.Error("surnames-single must NOT contain 凝 (discarded — not a surname)")
	}
	if !p.SurnamesCompound["欧阳"] {
		t.Error("surnames-compound missing 欧阳")
	}
	if p.SurnamesCompound["古月"] {
		t.Error("surnames-compound must NOT contain 古月 (pair-14 §1: a 蛊真人 clan, moved to the book overlay)")
	}
	if len(p.TitleSuffix) == 0 || p.TitleSuffix[0] != "公子" {
		t.Errorf("title-suffix order not preserved: %v", p.TitleSuffix)
	}
	if !p.TopoSuffix['山'] || !p.Numeral['三'] || !p.GradePrefix['甲'] || !p.AliasParticle['的'] {
		t.Error("rune-set membership missing an expected char (山/三/甲/的)")
	}
	// pair-14 moves: title-formant + sentence-terminator rune sets, and the Palladius phonotactic sets.
	if !p.TitleFormant['等'] || !p.TitleFormant['转'] || !p.TitleFormant['阶'] {
		t.Error("title-formant missing 等/转/阶")
	}
	if !p.SentenceTerminator['。'] || !p.SentenceTerminator['！'] || !p.SentenceTerminator['？'] {
		t.Error("sentence-terminator missing 。/！/？")
	}
	if !p.Palladius.Retroflex["zh"] || !p.Palladius.VFinal["v"] || !p.Palladius.VFinalInitial["j"] {
		t.Error("palladius phonotactics missing retroflex zh / vfinal v / vfinal_initial j")
	}
	// Pair transliteration spot-check.
	if p.Palladius.Initials["b"] != "б" || p.Palladius.Initials["zh"] != "чж" {
		t.Errorf("palladius initials wrong: b=%q zh=%q", p.Palladius.Initials["b"], p.Palladius.Initials["zh"])
	}
	if p.Palladius.SpecialI["zhi"] != "чжи" {
		t.Errorf("palladius special_i zhi = %q, want чжи", p.Palladius.SpecialI["zhi"])
	}
	if !strings.HasPrefix(p.Version(), packAlgoVersion+"-") {
		t.Errorf("Version() = %q, want %s-<hash>", p.Version(), packAlgoVersion)
	}
}

// TestRoutesByPairToDifferentBytes is the load-bearing generality proof: a SECOND pair, dropped as data
// (no pipeline/ edit, no recompile), routes to DIFFERENT bytes and a DIFFERENT version. Proves "add a
// language = data only" — the pair is a genuine routing key, not a baked-in zh constant.
func TestRoutesByPairToDifferentBytes(t *testing.T) {
	root := t.TempDir()
	writeSyntheticPack(t, root, "xx", "yy")

	real, err := Load(realRoot, "zh", "ru")
	if err != nil {
		t.Fatalf("load real: %v", err)
	}
	synth, err := Load(root, "xx", "yy")
	if err != nil {
		t.Fatalf("load synthetic xx-yy: %v", err)
	}

	// The synthetic pair carries its OWN surnames (not zh's), proving the resolver routes by pair, not
	// to a hardcoded set.
	if !synth.SurnamesSingle['甴'] {
		t.Error("synthetic pack must carry its own surname 甴")
	}
	if synth.SurnamesSingle['赵'] {
		t.Error("synthetic pack must NOT inherit zh's surname 赵 (would mean a baked-in constant)")
	}
	if !real.SurnamesSingle['赵'] || real.SurnamesSingle['甴'] {
		t.Error("real zh pack routing leaked/borrowed synthetic bytes")
	}
	if real.Version() == synth.Version() {
		t.Errorf("different-byte packs must have different versions (real=%s synth=%s)", real.Version(), synth.Version())
	}
}

// TestBookOverlayUnionsAndReVersions pins the pair-14 §1 book-overlay contract: an overlay UNIONS its
// private canon onto the shared pack (additive — the base entries survive, the overlay entry is added) and
// SHIFTS Version() (a book-canon edit is a loud --resnapshot for that book), while a no-overlay Load is
// byte-identical to before. Guards the exact mechanism the miner's {古月:22} parity rides.
func TestBookOverlayUnionsAndReVersions(t *testing.T) {
	base, err := Load(realRoot, "zh", "ru")
	if err != nil {
		t.Fatalf("load base: %v", err)
	}
	if base.SurnamesCompound["古月"] {
		t.Fatal("shared pack must not carry 古月 (it is a book clan)")
	}
	overlay := t.TempDir()
	if err := os.MkdirAll(filepath.Join(overlay, "zh"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(overlay, "zh", "surnames-compound.txt"), []byte("# book canon\n古月\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	ext, err := LoadWithOverlay(realRoot, "zh", "ru", overlay)
	if err != nil {
		t.Fatalf("load with overlay: %v", err)
	}
	if !ext.SurnamesCompound["古月"] {
		t.Error("overlay must add 古月 to the effective pack")
	}
	if !ext.SurnamesCompound["欧阳"] {
		t.Error("overlay must UNION (keep the shared 欧阳), not replace")
	}
	if ext.Version() == base.Version() {
		t.Errorf("overlay must shift Version() (base=%s overlay=%s)", base.Version(), ext.Version())
	}
	// An empty overlayRoot is exactly Load — same version, no re-hash.
	same, err := LoadWithOverlay(realRoot, "zh", "ru", "")
	if err != nil {
		t.Fatal(err)
	}
	if same.Version() != base.Version() {
		t.Errorf("empty overlay must equal Load (%s != %s)", same.Version(), base.Version())
	}

	// A MISNAMED overlay file must FAIL LOUD, not be silently ignored (review finding, scale lens): a typo'd
	// canon file that never reaches the miner would degrade recall with no signal.
	bad := t.TempDir()
	if err := os.MkdirAll(filepath.Join(bad, "zh"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(bad, "zh", "surname-compound.txt"), []byte("古月\n"), 0o644); err != nil { // typo: missing 's'
		t.Fatal(err)
	}
	if _, err := LoadWithOverlay(realRoot, "zh", "ru", bad); err == nil {
		t.Error("a misnamed overlay file must fail loud (silently-ignored canon would degrade recall)")
	} else if !strings.Contains(err.Error(), "surname-compound.txt") {
		t.Errorf("error must name the unexpected file, got: %v", err)
	}
}

// TestFailsLoudOnMissingPair pins the fail-loud contract: a pair with no pack directory errors, naming the
// missing file — never a silent empty pack (mirrors the prompt seam's PromptPathFor fail-loud). This is
// the load-time invariant a live consumer relies on (fail before billing).
func TestFailsLoudOnMissingPair(t *testing.T) {
	_, err := Load(realRoot, "ja", "ru")
	if err == nil {
		t.Fatal("Load(ja,ru) must fail loud (no ja langpack), got nil error")
	}
	if !strings.Contains(err.Error(), "ja-ru") {
		t.Errorf("error must name the missing pair ja-ru, got: %v", err)
	}
}

// TestFailsLoudOnEmptyTable pins the "never silently empty" contract: a present-but-empty (comment-only)
// required file is a corrupt pack and must fail loud at load — not parse to an empty set that silently
// disables a miner channel once packs are hand-authored (R1).
func TestFailsLoudOnEmptyTable(t *testing.T) {
	root := t.TempDir()
	writeSyntheticPack(t, root, "xx", "yy")
	// Blank out one required table (keep the file present, drop its content).
	if err := os.WriteFile(filepath.Join(root, "xx", "surnames-single.txt"), []byte("# emptied by a bad edit\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	_, err := Load(root, "xx", "yy")
	if err == nil {
		t.Fatal("an empty required table must fail loud, got nil error")
	}
	if !strings.Contains(err.Error(), "surnames-single") {
		t.Errorf("error must name the empty table, got: %v", err)
	}
}

// writeSyntheticPack writes a minimal, VALID pack for (src, tgt) under root with bytes distinct from zh —
// enough for Load to succeed and for the routing assertion to bite.
func writeSyntheticPack(t *testing.T, root, src, tgt string) {
	t.Helper()
	pair := src + "-" + tgt
	files := map[string]string{
		filepath.Join(src, "surnames-single.txt"):         "# synthetic\n甴甶甹\n",
		filepath.Join(src, "surnames-compound.txt"):       "# synthetic\n甲乙\n",
		filepath.Join(src, "title-suffix.txt"):            "# synthetic\n阁下\n",
		filepath.Join(src, "ordinal-title.txt"):           "# synthetic\n第甲\n",
		filepath.Join(src, "rank-word.txt"):               "# synthetic\n級\n",
		filepath.Join(src, "topo-suffix.txt"):             "# synthetic\n峰\n",
		filepath.Join(src, "grade-prefix.txt"):            "# synthetic\n子丑\n",
		filepath.Join(src, "numeral.txt"):                 "# synthetic\n壹貳\n",
		filepath.Join(src, "alias-particle.txt"):          "# synthetic\n之乎\n",
		filepath.Join(src, "title-formant.txt"):           "# synthetic\n甼\n",
		filepath.Join(src, "sentence-terminator.txt"):     "# synthetic\n。\n",
		filepath.Join(pair, "palladius.txt"):              "# synthetic\ninitials\tb\tб\nfinals\ta\tа\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n",
		filepath.Join(pair, "palladius-phonotactics.txt"): "# synthetic\nretroflex\tzh\nvfinal\tv\nvfinal_initial\tj\n",
	}
	for rel, body := range files {
		p := filepath.Join(root, rel)
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
			t.Fatal(err)
		}
	}
}

// --- pack-15: the overlay's fail-loud one level UP + the Palladius category contract -----------------

// TestOverlayRootFailsLoud pins the pack-15 guards ABOVE the per-file check: langpack_extend is an
// explicit opt-in, so a path that does not exist (a typo, a moved book directory) must STOP the load
// rather than degrade recall in silence, and an overlay root may hold nothing but its two expected
// subdirectories (a data file dropped at the root, or a mistyped pair dir, was silently ignored before).
func TestOverlayRootFailsLoud(t *testing.T) {
	t.Run("missing root", func(t *testing.T) {
		_, err := LoadWithOverlay(realRoot, "zh", "ru", filepath.Join(t.TempDir(), "nope"))
		if err == nil {
			t.Fatal("a missing overlay root must fail loud (explicit opt-in — a typo would silently drop the book canon)")
		}
		if !strings.Contains(err.Error(), "nope") {
			t.Errorf("error must name the path it looked for, got: %v", err)
		}
	})
	t.Run("file at the root", func(t *testing.T) {
		root := t.TempDir()
		if err := os.MkdirAll(filepath.Join(root, "zh"), 0o755); err != nil {
			t.Fatal(err)
		}
		// The canon file dropped at the ROOT instead of inside zh/ — the fold never looks here.
		if err := os.WriteFile(filepath.Join(root, "surnames-compound.txt"), []byte("古月\n"), 0o644); err != nil {
			t.Fatal(err)
		}
		if _, err := LoadWithOverlay(realRoot, "zh", "ru", root); err == nil {
			t.Error("a data file at the overlay root must fail loud (it is silently ignored otherwise)")
		} else if !strings.Contains(err.Error(), "surnames-compound.txt") {
			t.Errorf("error must name the unexpected entry, got: %v", err)
		}
	})
	t.Run("mistyped pair directory", func(t *testing.T) {
		root := t.TempDir()
		if err := os.MkdirAll(filepath.Join(root, "zh-rus"), 0o755); err != nil { // typo: zh-rus
			t.Fatal(err)
		}
		if _, err := LoadWithOverlay(realRoot, "zh", "ru", root); err == nil {
			t.Error("a mistyped pair directory must fail loud")
		} else if !strings.Contains(err.Error(), "zh-rus") {
			t.Errorf("error must name the unexpected directory, got: %v", err)
		}
	})
}

// TestOverlayKeyCollisionFailsLoud pins D39.23: an overlay may only ADD. A Palladius key the shared pair
// pack already defines would have been silently overridden by the overlay's value — override is not
// ratified, so the collision is loud and names the key and both values.
func TestOverlayKeyCollisionFailsLoud(t *testing.T) {
	root := t.TempDir()
	writeSyntheticPack(t, root, "xx", "yy")
	overlay := t.TempDir()
	if err := os.MkdirAll(filepath.Join(overlay, "xx-yy"), 0o755); err != nil {
		t.Fatal(err)
	}
	// `initials b` is already б in the base pack; the overlay redefines it as п.
	if err := os.WriteFile(filepath.Join(overlay, "xx-yy", "palladius.txt"), []byte("initials\tb\tп\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	_, err := LoadWithOverlay(root, "xx", "yy", overlay)
	if err == nil {
		t.Fatal("a base↔overlay key collision must fail loud (override is not ratified — D39.23)")
	}
	for _, want := range []string{"initials", `"b"`, "б", "п"} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("collision error must mention %q, got: %v", want, err)
		}
	}
	// An overlay that only ADDS a new key is fine (the additive contract still holds).
	ok := t.TempDir()
	if err := os.MkdirAll(filepath.Join(ok, "xx-yy"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(ok, "xx-yy", "palladius.txt"), []byte("initials\tp\tп\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	p, err := LoadWithOverlay(root, "xx", "yy", ok)
	if err != nil {
		t.Fatalf("an additive overlay must load: %v", err)
	}
	if p.Palladius.Initials["b"] != "б" || p.Palladius.Initials["p"] != "п" {
		t.Errorf("additive overlay must keep the base key and add the new one: %v", p.Palladius.Initials)
	}
}

// TestPalladiusParserToleratesUnknownCategory pins the owner addendum (24.07): the GENERIC parser is
// category-agnostic — an unknown category is data it does not interpret, never a parse error. Which
// categories are required (and which are consumed) is the CONSUMER's call, asserted separately below.
func TestPalladiusParserToleratesUnknownCategory(t *testing.T) {
	cats, err := parseCategoryRows([]byte("initials\tb\tб\nfuture_channel\tx\ty\nretroflex\tzh\n"))
	if err != nil {
		t.Fatalf("the generic parser must not reject an unknown category: %v", err)
	}
	if cats["initials"]["b"] != "б" || cats["retroflex"]["zh"] != "" || cats["future_channel"]["x"] != "y" {
		t.Errorf("parser must return every category verbatim, got %v", cats)
	}
	// A malformed ROW is still an error (the parser's own contract), naming the physical line.
	if _, err := parseCategoryRows([]byte("initials\n")); err == nil {
		t.Error("a row with too few fields must fail loud")
	}
}

// TestPalladiusRequiredCategoriesFailLoud pins the consumer half: a required category missing from the
// data (here: emptied) is caught by validate() at load, and a category NOTHING consumes (a typo like
// `initals`) is caught by the consumer's merge — the two failure modes the generic parser deliberately
// does not judge.
func TestPalladiusRequiredCategoriesFailLoud(t *testing.T) {
	t.Run("required category missing", func(t *testing.T) {
		root := t.TempDir()
		writeSyntheticPack(t, root, "xx", "yy")
		// Drop the `finals` category (keep the file, keep the other categories).
		if err := os.WriteFile(filepath.Join(root, "xx-yy", "palladius.txt"),
			[]byte("initials\tb\tб\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n"), 0o644); err != nil {
			t.Fatal(err)
		}
		_, err := Load(root, "xx", "yy")
		if err == nil {
			t.Fatal("a missing REQUIRED Palladius category must fail loud at load (validate)")
		}
		if !strings.Contains(err.Error(), "palladius/finals") {
			t.Errorf("error must name the empty required table, got: %v", err)
		}
	})
	t.Run("typo'd category is not consumed", func(t *testing.T) {
		root := t.TempDir()
		writeSyntheticPack(t, root, "xx", "yy")
		if err := os.WriteFile(filepath.Join(root, "xx-yy", "palladius.txt"),
			[]byte("initals\tb\tб\nfinals\ta\tа\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n"), 0o644); err != nil { // typo: initals
			t.Fatal(err)
		}
		_, err := Load(root, "xx", "yy")
		if err == nil {
			t.Fatal("a category nothing consumes must fail loud (it would silently leave its table empty)")
		}
		if !strings.Contains(err.Error(), "initals") {
			t.Errorf("error must name the unconsumed category, got: %v", err)
		}
	})
}

// TestTerminatorClassesAreOneSource pins the pack-15 dedup: the source chunker and the coverage gate no
// longer keep a private copy each of the terminator switches — both read these classes. The CJK class
// splits zero-width; the generic class needs whitespace. A rune of either class is a terminator.
func TestTerminatorClassesAreOneSource(t *testing.T) {
	term := DefaultTerminators()
	for _, r := range []rune{'。', '！', '？'} {
		if !term.IsCJK(r) || !term.IsTerminator(r) {
			t.Errorf("%q must be a CJK terminator", string(r))
		}
	}
	for _, r := range []rune{'.', '!', '?', '…'} {
		if term.IsCJK(r) {
			t.Errorf("%q must NOT be zero-width-splitting (the ellipsis/ASCII class needs whitespace)", string(r))
		}
		if !term.IsTerminator(r) {
			t.Errorf("%q must be a terminator", string(r))
		}
	}
	for _, r := range []rune{'，', '、', 'a', '「'} {
		if term.IsTerminator(r) {
			t.Errorf("%q must not be a terminator", string(r))
		}
	}
}

// TestChapterMarkerIsData pins the pack-15 move of 第 out of the ingest regex literal into cjk-section:
// the marker is data beside the numerals it precedes.
func TestChapterMarkerIsData(t *testing.T) {
	sec := DefaultCJKSection()
	if got := sec.ChapterMarkerClass(); got != "第" {
		t.Errorf("chapter marker class = %q, want 第", got)
	}
}