textmachine/backend/internal/lang/langpack_test.go

package lang

import (
	"os"
	"path/filepath"
	"strings"
	"testing"
)

// realRoot is the in-repo langpack root, relative to this package's test cwd (internal/lang/).
const realRoot = "../../configs/langpacks"

// TestLoadResolvesRealZhRu loads the REAL zh→ru pack from disk and pins a spot-check of the moved data
// (the const→file move must be byte-faithful; the full-book parity that pins the exact miner output is
// the stand-gated TestMinerFullBookParity, so this is the CI-runnable anchor). It exercises the real
// disk-loading path end to end.
func TestLoadResolvesRealZhRu(t *testing.T) {
	p, err := Load(realRoot, "zh", "ru")
	if err != nil {
		t.Fatalf("Load(zh,ru): %v", err)
	}
	if p.Pair != "zh-ru" {
		t.Errorf("Pair = %q, want zh-ru", p.Pair)
	}
	// Source morphology spot-check (byte-faithful move).
	if !p.SurnamesSingle['赵'] || !p.SurnamesSingle['方'] {
		t.Error("surnames-single missing 赵/方")
	}
	if p.SurnamesSingle['凝'] {
		t.Error("surnames-single must NOT contain 凝 (discarded — not a surname)")
	}
	if !p.SurnamesCompound["欧阳"] {
		t.Error("surnames-compound missing 欧阳")
	}
	if p.SurnamesCompound["古月"] {
		t.Error("surnames-compound must NOT contain 古月 (pair-14 §1: a 蛊真人 clan, moved to the book overlay)")
	}
	if len(p.TitleSuffix) == 0 || p.TitleSuffix[0] != "公子" {
		t.Errorf("title-suffix order not preserved: %v", p.TitleSuffix)
	}
	if !p.TopoSuffix['山'] || !p.Numeral['三'] || !p.GradePrefix['甲'] || !p.AliasParticle['的'] {
		t.Error("rune-set membership missing an expected char (山/三/甲/的)")
	}
	// pair-14 moves: title-formant + sentence-terminator rune sets, and the Palladius phonotactic sets.
	if !p.TitleFormant['等'] || !p.TitleFormant['转'] || !p.TitleFormant['阶'] {
		t.Error("title-formant missing 等/转/阶")
	}
	if !p.SentenceTerminator['。'] || !p.SentenceTerminator['！'] || !p.SentenceTerminator['？'] {
		t.Error("sentence-terminator missing 。/！/？")
	}
	if !p.Palladius.Retroflex["zh"] || !p.Palladius.VFinal["v"] || !p.Palladius.VFinalInitial["j"] {
		t.Error("palladius phonotactics missing retroflex zh / vfinal v / vfinal_initial j")
	}
	// Pair transliteration spot-check.
	if p.Palladius.Initials["b"] != "б" || p.Palladius.Initials["zh"] != "чж" {
		t.Errorf("palladius initials wrong: b=%q zh=%q", p.Palladius.Initials["b"], p.Palladius.Initials["zh"])
	}
	if p.Palladius.SpecialI["zhi"] != "чжи" {
		t.Errorf("palladius special_i zhi = %q, want чжи", p.Palladius.SpecialI["zhi"])
	}
	if !strings.HasPrefix(p.Version(), packAlgoVersion+"-") {
		t.Errorf("Version() = %q, want %s-<hash>", p.Version(), packAlgoVersion)
	}
}

// TestRoutesByPairToDifferentBytes is the load-bearing generality proof: a SECOND pair, dropped as data
// (no pipeline/ edit, no recompile), routes to DIFFERENT bytes and a DIFFERENT version. Proves "add a
// language = data only" — the pair is a genuine routing key, not a baked-in zh constant.
func TestRoutesByPairToDifferentBytes(t *testing.T) {
	root := t.TempDir()
	writeSyntheticPack(t, root, "xx", "yy")

	real, err := Load(realRoot, "zh", "ru")
	if err != nil {
		t.Fatalf("load real: %v", err)
	}
	synth, err := Load(root, "xx", "yy")
	if err != nil {
		t.Fatalf("load synthetic xx-yy: %v", err)
	}

	// The synthetic pair carries its OWN surnames (not zh's), proving the resolver routes by pair, not
	// to a hardcoded set.
	if !synth.SurnamesSingle['甴'] {
		t.Error("synthetic pack must carry its own surname 甴")
	}
	if synth.SurnamesSingle['赵'] {
		t.Error("synthetic pack must NOT inherit zh's surname 赵 (would mean a baked-in constant)")
	}
	if !real.SurnamesSingle['赵'] || real.SurnamesSingle['甴'] {
		t.Error("real zh pack routing leaked/borrowed synthetic bytes")
	}
	if real.Version() == synth.Version() {
		t.Errorf("different-byte packs must have different versions (real=%s synth=%s)", real.Version(), synth.Version())
	}
}

// TestManifestChannelsGateRequiredFiles is the П4 acceptance (D39.60 §3.2, the hardest data-layer blocker):
// a source that declares ONLY the source-morphology channel in its manifest loads WITHOUT the pair Palladius
// files — which, absent the manifest, are mandatory and would fail the load. The miner then gets an inert
// transliteration channel. A manifest-less pack is byte-identical (its version must equal the no-manifest load).
func TestManifestChannelsGateRequiredFiles(t *testing.T) {
	// Source ships ONLY the morphology files + a manifest that opts out of transliteration; NO pair dir.
	root := t.TempDir()
	morphOnly := map[string]string{
		"manifest.txt":            "# only the name-miner morphology, no pinyin transliteration\nchannel\tsource-morphology\n",
		"surnames-single.txt":     "# synthetic\n甴甶甹\n",
		"surnames-compound.txt":   "# synthetic\n甲乙\n",
		"title-suffix.txt":        "# synthetic\n阁下\n",
		"ordinal-title.txt":       "# synthetic\n第甲\n",
		"rank-word.txt":           "# synthetic\n級\n",
		"topo-suffix.txt":         "# synthetic\n峰\n",
		"grade-prefix.txt":        "# synthetic\n子丑\n",
		"numeral.txt":             "# synthetic\n壹貳\n",
		"alias-particle.txt":      "# synthetic\n之乎\n",
		"title-formant.txt":       "# synthetic\n甼\n",
		"sentence-terminator.txt": "# synthetic\n。\n",
	}
	for name, body := range morphOnly {
		p := filepath.Join(root, "xx", name)
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	p, err := Load(root, "xx", "yy")
	if err != nil {
		t.Fatalf("a source declaring only source-morphology must load without the Palladius files: %v", err)
	}
	if p.HasChannel("transliteration") {
		t.Error("the manifest opted out of transliteration; the pack must not claim that channel")
	}
	if !p.HasMorphologyChannel() {
		t.Error("the manifest declared source-morphology")
	}
	if len(p.Palladius.Initials) != 0 {
		t.Error("no transliteration channel → the Palladius table must be empty (inert)")
	}

	// A manifest-less pack ships every channel and is BYTE-IDENTICAL to before the manifest existed.
	full := t.TempDir()
	writeSyntheticPack(t, full, "xx", "yy")
	withMani := t.TempDir()
	writeSyntheticPack(t, withMani, "xx", "yy")
	if err := os.WriteFile(filepath.Join(withMani, "xx", "manifest.txt"), []byte("channel\tsource-morphology\nchannel\ttransliteration\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	pFull, err := Load(full, "xx", "yy")
	if err != nil {
		t.Fatal(err)
	}
	pMani, err := Load(withMani, "xx", "yy")
	if err != nil {
		t.Fatal(err)
	}
	// A manifest that lists every channel STILL folds its own bytes → a different (loud) version. Both must
	// carry both channels and the same tables, though.
	if !pFull.HasChannel("transliteration") || !pMani.HasChannel("transliteration") {
		t.Error("both packs declare all channels")
	}
	if pFull.Version() == pMani.Version() {
		t.Error("a manifest's bytes must fold into the version (drift-proof), so the two versions differ")
	}
}

// TestBookOverlayUnionsAndReVersions pins the pair-14 §1 book-overlay contract: an overlay UNIONS its
// private canon onto the shared pack (additive — the base entries survive, the overlay entry is added) and
// SHIFTS Version() (a book-canon edit is a loud --resnapshot for that book), while a no-overlay Load is
// byte-identical to before. Guards the exact mechanism the miner's {古月:22} parity rides.
func TestBookOverlayUnionsAndReVersions(t *testing.T) {
	base, err := Load(realRoot, "zh", "ru")
	if err != nil {
		t.Fatalf("load base: %v", err)
	}
	if base.SurnamesCompound["古月"] {
		t.Fatal("shared pack must not carry 古月 (it is a book clan)")
	}
	overlay := t.TempDir()
	if err := os.MkdirAll(filepath.Join(overlay, "zh"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(overlay, "zh", "surnames-compound.txt"), []byte("# book canon\n古月\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	ext, err := LoadWithOverlay(realRoot, "zh", "ru", overlay)
	if err != nil {
		t.Fatalf("load with overlay: %v", err)
	}
	if !ext.SurnamesCompound["古月"] {
		t.Error("overlay must add 古月 to the effective pack")
	}
	if !ext.SurnamesCompound["欧阳"] {
		t.Error("overlay must UNION (keep the shared 欧阳), not replace")
	}
	if ext.Version() == base.Version() {
		t.Errorf("overlay must shift Version() (base=%s overlay=%s)", base.Version(), ext.Version())
	}
	// An empty overlayRoot is exactly Load — same version, no re-hash.
	same, err := LoadWithOverlay(realRoot, "zh", "ru", "")
	if err != nil {
		t.Fatal(err)
	}
	if same.Version() != base.Version() {
		t.Errorf("empty overlay must equal Load (%s != %s)", same.Version(), base.Version())
	}

	// A MISNAMED overlay file must FAIL LOUD, not be silently ignored (review finding, scale lens): a typo'd
	// canon file that never reaches the miner would degrade recall with no signal.
	bad := t.TempDir()
	if err := os.MkdirAll(filepath.Join(bad, "zh"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(bad, "zh", "surname-compound.txt"), []byte("古月\n"), 0o644); err != nil { // typo: missing 's'
		t.Fatal(err)
	}
	if _, err := LoadWithOverlay(realRoot, "zh", "ru", bad); err == nil {
		t.Error("a misnamed overlay file must fail loud (silently-ignored canon would degrade recall)")
	} else if !strings.Contains(err.Error(), "surname-compound.txt") {
		t.Errorf("error must name the unexpected file, got: %v", err)
	}
}

// TestFailsLoudOnMissingPair pins the fail-loud contract: a pair with no pack directory errors, naming the
// missing file — never a silent empty pack (mirrors the prompt seam's PromptPathFor fail-loud). This is
// the load-time invariant a live consumer relies on (fail before billing).
func TestFailsLoudOnMissingPair(t *testing.T) {
	_, err := Load(realRoot, "ja", "ru")
	if err == nil {
		t.Fatal("Load(ja,ru) must fail loud (no ja langpack), got nil error")
	}
	if !strings.Contains(err.Error(), "ja-ru") {
		t.Errorf("error must name the missing pair ja-ru, got: %v", err)
	}
}

// TestFailsLoudOnEmptyTable pins the "never silently empty" contract: a present-but-empty (comment-only)
// required file is a corrupt pack and must fail loud at load — not parse to an empty set that silently
// disables a miner channel once packs are hand-authored (R1).
func TestFailsLoudOnEmptyTable(t *testing.T) {
	root := t.TempDir()
	writeSyntheticPack(t, root, "xx", "yy")
	// Blank out one required table (keep the file present, drop its content).
	if err := os.WriteFile(filepath.Join(root, "xx", "surnames-single.txt"), []byte("# emptied by a bad edit\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	_, err := Load(root, "xx", "yy")
	if err == nil {
		t.Fatal("an empty required table must fail loud, got nil error")
	}
	if !strings.Contains(err.Error(), "surnames-single") {
		t.Errorf("error must name the empty table, got: %v", err)
	}
}

// writeSyntheticPack writes a minimal, VALID pack for (src, tgt) under root with bytes distinct from zh —
// enough for Load to succeed and for the routing assertion to bite.
func writeSyntheticPack(t *testing.T, root, src, tgt string) {
	t.Helper()
	pair := src + "-" + tgt
	files := map[string]string{
		filepath.Join(src, "surnames-single.txt"):         "# synthetic\n甴甶甹\n",
		filepath.Join(src, "surnames-compound.txt"):       "# synthetic\n甲乙\n",
		filepath.Join(src, "title-suffix.txt"):            "# synthetic\n阁下\n",
		filepath.Join(src, "ordinal-title.txt"):           "# synthetic\n第甲\n",
		filepath.Join(src, "rank-word.txt"):               "# synthetic\n級\n",
		filepath.Join(src, "topo-suffix.txt"):             "# synthetic\n峰\n",
		filepath.Join(src, "grade-prefix.txt"):            "# synthetic\n子丑\n",
		filepath.Join(src, "numeral.txt"):                 "# synthetic\n壹貳\n",
		filepath.Join(src, "alias-particle.txt"):          "# synthetic\n之乎\n",
		filepath.Join(src, "title-formant.txt"):           "# synthetic\n甼\n",
		filepath.Join(src, "sentence-terminator.txt"):     "# synthetic\n。\n",
		filepath.Join(pair, "palladius.txt"):              "# synthetic\ninitials\tb\tб\nfinals\ta\tа\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n",
		filepath.Join(pair, "palladius-phonotactics.txt"): "# synthetic\nretroflex\tzh\nvfinal\tv\nvfinal_initial\tj\n",
	}
	for rel, body := range files {
		p := filepath.Join(root, rel)
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
			t.Fatal(err)
		}
	}
}

// --- pack-15: the overlay's fail-loud one level UP + the Palladius category contract -----------------

// TestOverlayRootFailsLoud pins the pack-15 guards ABOVE the per-file check: langpack_extend is an
// explicit opt-in, so a path that does not exist (a typo, a moved book directory) must STOP the load
// rather than degrade recall in silence, and an overlay root may hold nothing but its two expected
// subdirectories (a data file dropped at the root, or a mistyped pair dir, was silently ignored before).
func TestOverlayRootFailsLoud(t *testing.T) {
	t.Run("missing root", func(t *testing.T) {
		_, err := LoadWithOverlay(realRoot, "zh", "ru", filepath.Join(t.TempDir(), "nope"))
		if err == nil {
			t.Fatal("a missing overlay root must fail loud (explicit opt-in — a typo would silently drop the book canon)")
		}
		if !strings.Contains(err.Error(), "nope") {
			t.Errorf("error must name the path it looked for, got: %v", err)
		}
	})
	t.Run("file at the root", func(t *testing.T) {
		root := t.TempDir()
		if err := os.MkdirAll(filepath.Join(root, "zh"), 0o755); err != nil {
			t.Fatal(err)
		}
		// The canon file dropped at the ROOT instead of inside zh/ — the fold never looks here.
		if err := os.WriteFile(filepath.Join(root, "surnames-compound.txt"), []byte("古月\n"), 0o644); err != nil {
			t.Fatal(err)
		}
		if _, err := LoadWithOverlay(realRoot, "zh", "ru", root); err == nil {
			t.Error("a data file at the overlay root must fail loud (it is silently ignored otherwise)")
		} else if !strings.Contains(err.Error(), "surnames-compound.txt") {
			t.Errorf("error must name the unexpected entry, got: %v", err)
		}
	})
	t.Run("mistyped pair directory", func(t *testing.T) {
		root := t.TempDir()
		if err := os.MkdirAll(filepath.Join(root, "zh-rus"), 0o755); err != nil { // typo: zh-rus
			t.Fatal(err)
		}
		if _, err := LoadWithOverlay(realRoot, "zh", "ru", root); err == nil {
			t.Error("a mistyped pair directory must fail loud")
		} else if !strings.Contains(err.Error(), "zh-rus") {
			t.Errorf("error must name the unexpected directory, got: %v", err)
		}
	})
}

// TestOverlayKeyCollisionFailsLoud pins D39.23: an overlay may only ADD. A Palladius key the shared pair
// pack already defines would have been silently overridden by the overlay's value — override is not
// ratified, so the collision is loud and names the key and both values.
func TestOverlayKeyCollisionFailsLoud(t *testing.T) {
	root := t.TempDir()
	writeSyntheticPack(t, root, "xx", "yy")
	overlay := t.TempDir()
	if err := os.MkdirAll(filepath.Join(overlay, "xx-yy"), 0o755); err != nil {
		t.Fatal(err)
	}
	// `initials b` is already б in the base pack; the overlay redefines it as п.
	if err := os.WriteFile(filepath.Join(overlay, "xx-yy", "palladius.txt"), []byte("initials\tb\tп\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	_, err := LoadWithOverlay(root, "xx", "yy", overlay)
	if err == nil {
		t.Fatal("a base↔overlay key collision must fail loud (override is not ratified — D39.23)")
	}
	for _, want := range []string{"initials", `"b"`, "б", "п"} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("collision error must mention %q, got: %v", want, err)
		}
	}
	// An overlay that only ADDS a new key is fine (the additive contract still holds).
	ok := t.TempDir()
	if err := os.MkdirAll(filepath.Join(ok, "xx-yy"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(ok, "xx-yy", "palladius.txt"), []byte("initials\tp\tп\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	p, err := LoadWithOverlay(root, "xx", "yy", ok)
	if err != nil {
		t.Fatalf("an additive overlay must load: %v", err)
	}
	if p.Palladius.Initials["b"] != "б" || p.Palladius.Initials["p"] != "п" {
		t.Errorf("additive overlay must keep the base key and add the new one: %v", p.Palladius.Initials)
	}
}

// TestPalladiusParserToleratesUnknownCategory pins the owner addendum (24.07): the GENERIC parser is
// category-agnostic — an unknown category is data it does not interpret, never a parse error. Which
// categories are required (and which are consumed) is the CONSUMER's call, asserted separately below.
func TestPalladiusParserToleratesUnknownCategory(t *testing.T) {
	cats, err := parseCategoryRows([]byte("initials\tb\tб\nfuture_channel\tx\ty\nretroflex\tzh\n"))
	if err != nil {
		t.Fatalf("the generic parser must not reject an unknown category: %v", err)
	}
	if cats["initials"]["b"] != "б" || cats["retroflex"]["zh"] != "" || cats["future_channel"]["x"] != "y" {
		t.Errorf("parser must return every category verbatim, got %v", cats)
	}
	// A malformed ROW is still an error (the parser's own contract), naming the physical line.
	if _, err := parseCategoryRows([]byte("initials\n")); err == nil {
		t.Error("a row with too few fields must fail loud")
	}
}

// TestPalladiusRequiredCategoriesFailLoud pins the consumer half: a required category missing from the
// data (here: emptied) is caught by validate() at load, and a category NOTHING consumes (a typo like
// `initals`) is caught by the consumer's merge — the two failure modes the generic parser deliberately
// does not judge.
func TestPalladiusRequiredCategoriesFailLoud(t *testing.T) {
	t.Run("required category missing", func(t *testing.T) {
		root := t.TempDir()
		writeSyntheticPack(t, root, "xx", "yy")
		// Drop the `finals` category (keep the file, keep the other categories).
		if err := os.WriteFile(filepath.Join(root, "xx-yy", "palladius.txt"),
			[]byte("initials\tb\tб\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n"), 0o644); err != nil {
			t.Fatal(err)
		}
		_, err := Load(root, "xx", "yy")
		if err == nil {
			t.Fatal("a missing REQUIRED Palladius category must fail loud at load (validate)")
		}
		if !strings.Contains(err.Error(), "palladius/finals") {
			t.Errorf("error must name the empty required table, got: %v", err)
		}
	})
	t.Run("typo'd category is not consumed", func(t *testing.T) {
		root := t.TempDir()
		writeSyntheticPack(t, root, "xx", "yy")
		if err := os.WriteFile(filepath.Join(root, "xx-yy", "palladius.txt"),
			[]byte("initals\tb\tб\nfinals\ta\tа\nyw\tyi\tи\nspecial_i\tzhi\tчжи\n"), 0o644); err != nil { // typo: initals
			t.Fatal(err)
		}
		_, err := Load(root, "xx", "yy")
		if err == nil {
			t.Fatal("a category nothing consumes must fail loud (it would silently leave its table empty)")
		}
		if !strings.Contains(err.Error(), "initals") {
			t.Errorf("error must name the unconsumed category, got: %v", err)
		}
	})
}

// TestTerminatorClassesAreOneSource pins the pack-15 dedup: the source chunker and the coverage gate no
// longer keep a private copy each of the terminator switches — both read these classes. The CJK class
// splits zero-width; the generic class needs whitespace. A rune of either class is a terminator.
func TestTerminatorClassesAreOneSource(t *testing.T) {
	term := DefaultTerminators()
	for _, r := range []rune{'。', '！', '？'} {
		if !term.IsCJK(r) || !term.IsTerminator(r) {
			t.Errorf("%q must be a CJK terminator", string(r))
		}
	}
	for _, r := range []rune{'.', '!', '?', '…'} {
		if term.IsCJK(r) {
			t.Errorf("%q must NOT be zero-width-splitting (the ellipsis/ASCII class needs whitespace)", string(r))
		}
		if !term.IsTerminator(r) {
			t.Errorf("%q must be a terminator", string(r))
		}
	}
	for _, r := range []rune{'，', '、', 'a', '「'} {
		if term.IsTerminator(r) {
			t.Errorf("%q must not be a terminator", string(r))
		}
	}
}

// TestChapterMarkerIsData pins the pack-15 move of 第 out of the ingest regex literal into cjk-section:
// the marker is data beside the numerals it precedes.
func TestChapterMarkerIsData(t *testing.T) {
	sec := DefaultCJKSection()
	if got := sec.ChapterMarkerClass(); got != "第" {
		t.Errorf("chapter marker class = %q, want 第", got)
	}
}

// TestPairDataEditMovesThePackVersion pins the FOLD, which is what makes a data edit a loud --resnapshot
// instead of a silent one: every authored pack file shapes a snapshot-folded stage, so editing any of them
// must move Version(). Runs over a COPY of the shipped pack, so it pins the real schema rather than a
// fixture that might drift from it.
//
// It was written the other way round for the genre glossary (the one optional file DELIBERATELY left out of
// the hash, so that curating an industry rendering would not re-bill every book of the pair). D39.47 removed
// that file and with it the exemption; what survives is the plain rule, and this test is the guard that no
// future optional file quietly acquires the same hole.
func TestPairDataEditMovesThePackVersion(t *testing.T) {
	root := t.TempDir()
	for _, dir := range []string{"zh", "zh-ru"} {
		copyPackDir(t, filepath.Join("../../configs/langpacks", dir), filepath.Join(root, dir))
	}
	p1, err := Load(root, "zh", "ru")
	if err != nil {
		t.Fatal(err)
	}
	sur := filepath.Join(root, "zh", "surnames-single.txt")
	b, err := os.ReadFile(sur)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(sur, append(b, []byte("\n\u4EC7\n")...), 0o644); err != nil {
		t.Fatal(err)
	}
	p2, err := Load(root, "zh", "ru")
	if err != nil {
		t.Fatal(err)
	}
	if p1.Version() == p2.Version() {
		t.Fatal("a miner-table edit MUST move the pack version (it shapes a folded stage)")
	}
	// And the pair half of the pack is folded too, not just the source half.
	pal := filepath.Join(root, "zh-ru", "palladius.txt")
	pb, err := os.ReadFile(pal)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(pal, append(pb, []byte("\n# curator note\n")...), 0o644); err != nil {
		t.Fatal(err)
	}
	p3, err := Load(root, "zh", "ru")
	if err != nil {
		t.Fatal(err)
	}
	if p3.Version() == p2.Version() {
		t.Fatal("a pair-table edit MUST move the pack version too")
	}
}

// TestTerminologySizingIsOptionalPairData pins both halves of the optional contract: a pair that ships no
// file is byte-stable (so shipping the mechanism re-bills nobody, which is why zh-ru deliberately ships
// none), and a pair that does ship one folds its bytes like every other authored file.
func TestTerminologySizingIsOptionalPairData(t *testing.T) {
	root := t.TempDir()
	for _, dir := range []string{"zh", "zh-ru"} {
		copyPackDir(t, filepath.Join("../../configs/langpacks", dir), filepath.Join(root, dir))
	}
	p1, err := Load(root, "zh", "ru")
	if err != nil {
		t.Fatal(err)
	}
	if p1.Terminology != nil {
		t.Fatal("the shipped pair ships no terminology.txt: its measured sizing equals the engine default, and the bytes would move the pack version for nothing")
	}
	shipped, err := Load("../../configs/langpacks", "zh", "ru")
	if err != nil {
		t.Fatal(err)
	}
	if shipped.Version() != p1.Version() {
		t.Fatalf("the copy and the shipped pack must hash identically: %s vs %s", shipped.Version(), p1.Version())
	}
	// The tag itself is the other half of "nobody is re-billed": adding an optional file changes no
	// existing pack's bytes, so bumping the tag here would move every pair's version for nothing.
	if packAlgoVersion != "langpack-v2" {
		t.Fatalf("the pack algorithm tag moved to %q — that re-bills both waves of every book of every pair, so it must be a deliberate decision, not a side effect of adding an optional file", packAlgoVersion)
	}

	path := filepath.Join(root, "zh-ru", "terminology.txt")
	if err := os.WriteFile(path, []byte("# pair sizing\nkwic_per_term\t5\nkwic_width\t120\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	p2, err := Load(root, "zh", "ru")
	if err != nil {
		t.Fatal(err)
	}
	if p2.Terminology == nil || p2.Terminology.KWICPerTerm != 5 || p2.Terminology.KWICWidth != 120 {
		t.Fatalf("the pair sizing must be parsed, got %+v", p2.Terminology)
	}
	if p2.Version() == p1.Version() {
		t.Fatal("a present pair file MUST fold into the version, like every other authored file")
	}
	for _, bad := range []string{"", "# only a comment\n", "kwic_width\t0\n", "kwic_width\twide\n", "kwick\t3\n"} {
		if err := os.WriteFile(path, []byte(bad), 0o644); err != nil {
			t.Fatal(err)
		}
		if _, err := Load(root, "zh", "ru"); err == nil {
			t.Fatalf("a corrupt or empty table must fail loud at load, %q did not", bad)
		}
	}
}

func copyPackDir(t *testing.T, from, to string) {
	t.Helper()
	if err := os.MkdirAll(to, 0o755); err != nil {
		t.Fatal(err)
	}
	entries, err := os.ReadDir(from)
	if err != nil {
		t.Fatal(err)
	}
	for _, e := range entries {
		if e.IsDir() {
			continue
		}
		b, err := os.ReadFile(filepath.Join(from, e.Name()))
		if err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(to, e.Name()), b, 0o644); err != nil {
			t.Fatal(err)
		}
	}
}

// TestParseDCCheckersRefusesIntraFileDupKey pins the row-53 fix: a repeated key WITHIN one dc-checkers file
// fails loud at load — it would otherwise silently shadow the earlier row (a lost numeral/pattern/message/
// ratio). The intra-file mirror of unionStringMap's cross-file collision refusal (PACK15 §Внутрифайловый).
func TestParseDCCheckersRefusesIntraFileDupKey(t *testing.T) {
	// A minimal blob satisfying the non-empty requirement (cjk_numeral + ru_hour; register_neg is OPTIONAL
	// after D39.79 Q4 but kept here to exercise the list-tolerance case below).
	base := "cjk_numeral\t一\t1\nru_hour\tчас\t1\nregister_neg\tтерем\n"
	if _, err := parseDCCheckers([]byte(base)); err != nil {
		t.Fatalf("the base blob must parse clean, got %v", err)
	}
	cases := []struct{ name, blob string }{
		{"cjk_numeral", base + "cjk_numeral\t一\t1\n"},           // 一 repeated
		{"ru_hour", base + "ru_hour\tчас\t1\n"},                 // час repeated
		{"dc_ratio", base + "dc_ratio\tr\t1\ndc_ratio\tr\t2\n"}, // r repeated → second would shadow
		{"pattern", base + "pattern\tp\ta\npattern\tp\tb\n"},    // p repeated
		{"msg", base + "msg\tm\ta\nmsg\tm\tb\n"},                // m repeated
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			_, err := parseDCCheckers([]byte(c.blob))
			if err == nil {
				t.Fatalf("a duplicate %s key must fail loud, got nil error", c.name)
			}
			if !strings.Contains(err.Error(), "duplicate") {
				t.Fatalf("the error must name the duplicate, got %q", err)
			}
		})
	}
	// register_neg is a LIST, not a keyed category — a repeated lexeme is tolerated and must NOT trip the guard.
	if _, err := parseDCCheckers([]byte(base + "register_neg\tтерем\n")); err != nil {
		t.Fatalf("a repeated register_neg lexeme (a list value, not a key) must be tolerated, got %v", err)
	}
}