textmachine/backend/internal/lang/langpack.go

// Package lang holds the language-specific DATA the translation engine reads, isolated OUT of
// internal/pipeline/ as versioned files under configs/langpacks/ (owner directive D39.15: language data
// must not live as Go constants inside the engine; horizon = hundreds of languages, data-as-files, add a
// pair = drop a directory, no recompile, no pipeline/ edits).
//
// This package carries NO behaviour. The miner / checker ALGORITHMS stay in internal/pipeline (parity-
// locked, owner: "from the engine, DATA leaves; the ALGORITHM stays") and READ these tables. The one-way
// dependency (pipeline → lang, never the reverse) is compile-enforced: nothing here imports pipeline, so
// the data/algorithm boundary is real, not a convention.
//
// A Pack is resolved by (source, target) language and content-hashed at load, mirroring the two in-repo
// precedents: the pair-keyed prompt seam (config.Stage.Prompts + PromptPathFor + PromptSHA256, resolved
// by Book.LangPair(), fail-loud on a missing pair) and the memnorm trad→simp table (a data file whose
// bytes ARE its version via a content hash). Editing a pack file changes its Version() → the pipeline
// folds that into the snapshot → a loud --resnapshot, drift-proof by mechanism, not discipline.
package lang

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
)

// packAlgoVersion tags the PARSE/layout of a pack (the file manifest + how each file is read). Bump it on
// a schema change (a new file, a format change). The DATA content is versioned separately by hashing the
// files into Version(), so editing a table also invalidates — you cannot forget to bump a version when
// you change the data, because the data's bytes ARE the version (the text/norm.go drift-proofing).
// v2 (pair-14): new pair/source files (title-formant, sentence-terminator, palladius-phonotactics, dc-checkers)
// + new formats (the pattern/category rows, the generic Palladius parser) — a schema change, so the tag bumps.
//
// "Schema change" means the READING of bytes that already exist changes. Adding a file that every current
// pair lacks does NOT: an absent optional file writes nothing into the hash, so those packs load to the
// identical version and nobody is re-billed for a mechanism they do not use. Bump the tag when a REQUIRED
// file joins the manifest, or when any existing file starts parsing differently.
const packAlgoVersion = "langpack-v2"

// Pack is a loaded, versioned language-data pack for one source→target pair. Fields are the DATA the
// pipeline algorithms read; the zero value is unusable (load via Load). Maps are membership sets / lookup
// tables (order-free); slices preserve their authored order.
type Pack struct {
	Pair string // "<src>-<tgt>", e.g. "zh-ru" (== config.Book.LangPair())

	// <src> source morphology (configs/langpacks/<src>/) — the miner's typed-candidate channels.
	SurnamesSingle   map[rune]bool   // 百家姓 single-char surnames
	SurnamesCompound map[string]bool // two-char compound surnames
	TitleSuffix      []string        // title suffixes → title (ordered)
	OrdinalTitle     []string        // ordinal titles 一代/第一… (ordered)
	RankWord         []string        // rank/measure words 等/转… (ordered)
	TopoSuffix       map[rune]bool   // topographic suffix chars → place
	GradePrefix      map[rune]bool   // grade/stem prefix chars 甲乙丙…
	Numeral          map[rune]bool   // CJK numeral chars
	AliasParticle    map[rune]bool   // trailing-particle set marking a boundary fragment
	// TitleFormant: the rank/measure formant chars a miner classifies as a TITLE (patterns.formant_type
	// 等/转/阶 → title). Pair-14: moved out of the pipeline formant switch, sitting beside TopoSuffix (its
	// place-side neighbour). A rune SET.
	TitleFormant map[rune]bool
	// SentenceTerminator: the source sentence-ending marks the alias miner splits on (。！？, alias.
	// cooccur_same_sentence). Pair-14: moved out of the pipeline literal. A rune SET; the miner adds "\n"
	// (a structural newline, not a language mark) itself.
	SentenceTerminator map[rune]bool

	// Palladius is the <src>-<tgt> transliteration table (configs/langpacks/<pair>/palladius.txt +
	// palladius-phonotactics.txt) — the pinyin→Cyrillic maps and the syllable-generator's phonotactic
	// constraints, as ONE typed value (owner addendum 24.07: a struct, not four+three parallel Pack fields).
	Palladius Palladius

	// DCCheckers is the OPTIONAL pair data for the WS5 defect-class checkers (configs/langpacks/<pair>/
	// dc-checkers.txt, checkers_zh_ru.go). nil when the pair ships no file — the checker ALGORITHMS then read
	// empty tables and fire 0 (a pair that does not opt in is never flagged). DATA only (pair-14 §6: the
	// LOOKUP TABLES move out of the pipeline; the regex DETECTION patterns stay as the checker algorithm).
	DCCheckers *DCCheckerData

	// Heading is the OPTIONAL chapter-heading rule (configs/langpacks/<pair>/heading.txt). nil when the pair
	// carries no heading.txt — the chapter-title feature is then inert (the chunker keeps the source header
	// as-is), so a pair that does not opt in is never re-billed for it. It is DATA only: the detect/strip/
	// render ALGORITHM lives in the chunker (internal/pipeline), which reads this table — the same
	// data/algorithm boundary the miner tables keep.
	Heading *HeadingRule

	// SpeechCue is the OPTIONAL SOURCE-side direct-speech alphabet (configs/langpacks/<src>/speech-cue.txt);
	// nil when the source ships no file, and then the source-side reply count is simply 0. It is source
	// data because "a quoted turn is 「…」 with 说/道 beside it" is a fact about the source language, not
	// about the pair or the algorithm.
	SpeechCue *SpeechCue

	// Terminology is the OPTIONAL pair sizing of the terminologist's source contexts
	// (configs/langpacks/<pair>/terminology.txt); nil when the pair ships no file. It exists because the
	// KWIC width is measured in RUNES, and N runes buy a different amount of context in every script, so
	// the number is a fact about the pair and belongs in pair data rather than in a Go constant.
	Terminology *TerminologySizing

	// channels is the set of DATA CHANNELS the source declares in its OPTIONAL manifest.txt. A pack with NO
	// manifest ships EVERY channel (the original contract, so an existing pack is byte-identical). A source
	// that supports only some — a ja source with its OWN name morphology but no pinyin transliteration —
	// declares them, so Load reads (and validate requires) ONLY the declared channels' files and the miner
	// gets an inert channel for an undeclared one (D39.60 §3.2, the hardest data-layer blocker).
	channels map[string]bool

	version string
}

// The pack DATA CHANNELS. Each groups the manifest files a channel needs; a source declares which it ships.
const (
	channelMorphology      = "source-morphology" // srcFiles: the name-miner's typed-candidate tables
	channelTransliteration = "transliteration"   // pairFiles: the Palladius pinyin→Cyrillic maps
)

// allChannels is the full channel set — the default when a source ships NO manifest (backward-compatible).
var allChannels = []string{channelMorphology, channelTransliteration}

// HasChannel reports whether the pack declares a data channel (a manifest-less pack declares all of them).
func (p *Pack) HasChannel(name string) bool { return p != nil && p.channels[name] }

// HasMorphologyChannel reports whether the source ships the name-miner morphology tables; when false the miner
// has an inert channel (empty tables → no candidates) and the caller warns rather than mining silently.
func (p *Pack) HasMorphologyChannel() bool { return p.HasChannel(channelMorphology) }

// SpeechCue is the source language's direct-speech alphabet: the attribution cues and the quote marks
// that bound a quoted turn, plus how far from a quote mark a cue still counts as attributing it. Data
// only — the counting ALGORITHM lives in the pipeline, like every other table here.
type SpeechCue struct {
	Cues       []string      // attribution cues in authored order (说/道/问 …)
	QuoteOpen  map[rune]bool // opening quote marks
	QuoteClose map[rune]bool // closing quote marks
	Window     int           // runes on either side of a quote mark within which a cue attributes it
}

// TerminologySizing is the pair's own "how much source context is one context". A zero field means
// unstated and falls through to the engine default; the order lives in pipeline.terminologyOpts.
type TerminologySizing struct {
	KWICPerTerm int
	KWICWidth   int
}

// Palladius is the pair transliteration table: the pinyin→Cyrillic maps and the syllable-generator's
// phonotactic constraint sets (owner addendum 24.07 — one typed struct instead of four+three parallel Pack
// fields). Populated from palladius.txt (Initials/Finals/YW/SpecialI) + palladius-phonotactics.txt
// (Retroflex/VFinal/VFinalInitial) by the generic category parser; which categories are REQUIRED is enforced
// by validate(), not the parser (an unknown category is collected, never a parse error).
type Palladius struct {
	Initials, Finals, YW, SpecialI   map[string]string // pinyin→Cyrillic
	Retroflex, VFinal, VFinalInitial map[string]bool   // phonotactic constraint sets (pinyin tokens)
}

// newPalladius returns a Palladius with all tables allocated (so merge/union is nil-safe).
func newPalladius() Palladius {
	return Palladius{
		Initials: map[string]string{}, Finals: map[string]string{}, YW: map[string]string{}, SpecialI: map[string]string{},
		Retroflex: map[string]bool{}, VFinal: map[string]bool{}, VFinalInitial: map[string]bool{},
	}
}

// DCCheckerData is the per-pair lookup data the WS5 defect-class checkers read (checkers_zh_ru.go). The
// checker DETECTION regexes stay in the pipeline as the pair-scoped algorithm (§12.2); only the LOOKUP
// TABLES live here as data. Parsed from dc-checkers.txt.
type DCCheckerData struct {
	Numeral     map[rune]int   // DC1: small CJK count → value (一→1 … 十→10, incl. 两→2)
	RuHours     map[string]int // DC1: Russian hours-count word → value (один→1 … шесть→6)
	RegisterNeg []string       // DC6: register negative-list lexemes (терем + case forms), authored order
	// Patterns are the DETECTION patterns as DATA (pair-14 data-out): key → verbatim regex or literal probe.
	// The `*_re` keys are compiled by the consumer; the rest are literal strings.Contains probes. A pair that
	// ships no pattern for a key runs that sub-checker inert.
	Patterns map[string]string
	// Ratios are the pair's numeric UNIT RELATIONS as data (nit of generality, pre-run hygiene 25.07): the
	// checker algorithm compares counts, but "one source time-unit equals N target hours" is a fact ABOUT THE
	// PAIR, not about the algorithm — 时辰=2h for zh, and a pair using another unit (刻=¼h) must not need a Go
	// edit to say so. key → integer.
	Ratios map[string]int
	// Messages are the checkers' human-readable DETAIL templates as data (same nit): a detail line naming
	// 时辰/千万/成 is pair SUBSTANCE, and a second pair shipping its own patterns would otherwise inherit a
	// message about Chinese double-hours. `{name}` placeholders are substituted by the consumer; the template
	// is verbatim. Required whenever the corresponding detection data is present (a detector that fires with
	// no way to say what it found is a corrupt pack).
	Messages map[string]string
}

// HeadingRule is the per-pair data for the chapter-title policy: instead of letting the model render a
// chapter heading (which drifted to «Раздел 2» / «Первая глава» / an orphaned « :» across models), the
// chunker detects a source header (Marker + a numeral + a Unit rune), strips it from the model input, and
// the read-models render Template deterministically instead. Data only; parsed from heading.txt.
type HeadingRule struct {
	Marker   string        // the prefix rune(s) that open a numbered heading, e.g. "第"
	Units    map[rune]bool // the section-unit runes accepted right after the numeral (章 节 節 回)
	Template string        // the target rendering; the literal "{n}" is replaced by the parsed Arabic number
}

// Version is the content hash of the pack (packAlgoVersion + a sha256 of the authored file bytes). A pack
// edit changes it, so the pipeline can fold it into the snapshot (a loud --resnapshot on any data edit).
func (p *Pack) Version() string { return p.version }

// srcFiles are the source-morphology files (under configs/langpacks/<src>/), read in this fixed order.
// SCOPE (honest): this manifest is the zh-family NAME-MINER's morphology schema (百家姓 surnames, 甲乙丙 grade
// prefixes, 等/转/阶 title-formants, CJK numerals, …), and validate() forbids an empty table. So "add a pair =
// drop a directory, no recompile" holds for a source in THIS family (a second CJK source drops its files); a
// different source family that wants mining needs new Pack fields + parse + miner channels, not just a dir.
var srcFiles = []string{
	"surnames-single.txt", "surnames-compound.txt", "title-suffix.txt", "ordinal-title.txt",
	"rank-word.txt", "topo-suffix.txt", "grade-prefix.txt", "numeral.txt", "alias-particle.txt",
	"title-formant.txt", "sentence-terminator.txt",
}

// pairFiles are the pair-transliteration files (under configs/langpacks/<pair>/).
var pairFiles = []string{"palladius.txt", "palladius-phonotactics.txt"}

// Load resolves and reads the pack for (sourceLang, targetLang) from root (e.g. "configs/langpacks"): the
// source-morphology files under root/<src>/ and the pair-transliteration files under root/<src>-<tgt>/.
// EVERY declared file must be present and well-formed — a missing/corrupt file fails LOUD (the caller runs
// this at load, before any billing, mirroring the prompt-pack os.Stat loop). The content hash covers all
// files in a fixed order, so it is deterministic and drift-proof.
func Load(root, sourceLang, targetLang string) (*Pack, error) {
	pair := sourceLang + "-" + targetLang
	p := &Pack{Pair: pair, Palladius: newPalladius()}
	h := sha256.New()
	h.Write([]byte(packAlgoVersion))

	read := func(dir, name string) ([]byte, error) {
		path := filepath.Join(root, dir, name)
		b, err := os.ReadFile(path)
		if err != nil {
			return nil, fmt.Errorf("langpack %q: %w (add the file or fix the book's source_lang/target_lang; D39.15: language data is required, never silently empty)", pair, err)
		}
		// Fold the RELATIVE path + bytes so a rename or a moved byte both shift the hash.
		h.Write([]byte("\x00" + dir + "/" + name + "\x00"))
		h.Write(b)
		return b, nil
	}

	// OPTIONAL source manifest declaring which data CHANNELS this source ships. ABSENT → all channels (the
	// original contract; the manifest bytes are not folded, so an existing manifest-less pack is byte-identical
	// and never re-billed). PRESENT → its bytes fold into the hash (a manifest edit is a loud --resnapshot for
	// that pack) and ONLY the declared channels' files are read/required.
	channels, manifestBytes, hasManifest, merr := readManifest(root, sourceLang)
	if merr != nil {
		return nil, fmt.Errorf("langpack %q manifest.txt: %w", pair, merr)
	}
	if hasManifest {
		h.Write([]byte("\x00" + sourceLang + "/manifest.txt\x00"))
		h.Write(manifestBytes)
	}
	p.channels = channels

	if channels[channelMorphology] {
		for _, name := range srcFiles {
			b, err := read(sourceLang, name)
			if err != nil {
				return nil, err
			}
			if err := p.assignSrc(name, b); err != nil {
				return nil, fmt.Errorf("langpack %q %s: %w", pair, name, err)
			}
		}
	}
	if channels[channelTransliteration] {
		for _, name := range pairFiles {
			b, err := read(pair, name)
			if err != nil {
				return nil, err
			}
			if err := p.assignPair(name, b); err != nil {
				return nil, fmt.Errorf("langpack %q %s: %w", pair, name, err)
			}
		}
	}

	// OPTIONAL per-pair / per-source files, in a FIXED order (the order is folded into the content hash, so it
	// is behaviour). Each shares ONE contract (formerly four copy-pasted blocks): ABSENT → the feature is inert
	// and Version() is byte-stable (a pack that does not opt in is never re-billed); PRESENT → the bytes fold
	// into the hash (a loud --resnapshot on any edit) and are parsed into the pack; CORRUPT → fail loud, like a
	// required file. `dir` is the source OR the pair directory (speech-cue is a source fact, the rest pair).
	optional := []struct {
		dir, name string
		apply     func([]byte) error // parse + assign into p
	}{
		// pack-13 title policy.
		{pair, "heading.txt", func(b []byte) error { hr, e := parseHeading(b); p.Heading = hr; return e }},
		// pair-14 §6 DC-checker tables.
		{pair, "dc-checkers.txt", func(b []byte) error { dc, e := parseDCCheckers(b); p.DCCheckers = dc; return e }},
		// pack-19 speech-cue alphabet — a SOURCE fact (说/道 + 「」), so read from the source directory.
		{sourceLang, "speech-cue.txt", func(b []byte) error { sc, e := parseSpeechCue(b); p.SpeechCue = sc; return e }},
		// terminologist sizing (a pair whose value equals the default ships no file — moving the version to
		// change nothing would re-bill every book of the pair).
		{pair, "terminology.txt", func(b []byte) error { ts, e := parseTerminology(b); p.Terminology = ts; return e }},
	}
	for _, o := range optional {
		b, ok, err := readOptional(root, o.dir, o.name)
		if err != nil {
			return nil, fmt.Errorf("langpack %q %s: %w", pair, o.name, err)
		}
		if !ok {
			continue
		}
		h.Write([]byte("\x00" + o.dir + "/" + o.name + "\x00"))
		h.Write(b)
		if perr := o.apply(b); perr != nil {
			return nil, fmt.Errorf("langpack %q %s: %w", pair, o.name, perr)
		}
	}

	// A per-pair GENRE glossary was read here (pack-20 / D39.42 п.1) and was removed by D39.47: a pair-wide
	// file of "the industry rendering" prescribes ONE register to every book of the pair, and the choice
	// between «культивация» and «совершенствование» belongs to the owner's signature on a book's bank —
	// legitimately different from book to book. Nothing replaced it; the terminologist's only anchor is that
	// signed bank. Recorded here because the absence is a decision, not an omission.

	if err := p.validate(); err != nil {
		return nil, fmt.Errorf("langpack %q: %w", pair, err)
	}
	p.version = packAlgoVersion + "-" + hex.EncodeToString(h.Sum(nil))[:12]
	return p, nil
}

// LoadWithOverlay loads the shared pair pack (Load) and then UNIONS a book-scoped OVERLAY on top: a book's
// PRIVATE canon (a clan surname 古月, a sect term) that belongs to ONE book, not the shared pair langpack
// (pair-14 §1 — a book term in the shared pair layer is a leak). overlayRoot has the same layout as root
// (<src>/… + <pair>/…); ONLY the files a book chooses to ship are present, each read OPTIONALLY and UNIONED
// (additive — sets gain members, ordered slices append; an overlay never removes a base entry). The overlay
// bytes fold into Version(), so a book-canon edit is a loud --resnapshot for THAT book while the shared pack
// stays byte-stable. overlayRoot == "" ⇒ identical to Load (same *Pack, same Version).
func LoadWithOverlay(root, src, tgt, overlayRoot string) (*Pack, error) {
	p, err := Load(root, src, tgt)
	if err != nil {
		return nil, err
	}
	if overlayRoot == "" {
		return p, nil
	}
	pair := src + "-" + tgt
	// FAIL LOUD one level UP first (pack-15): an overlay root that does not exist, or that holds anything
	// besides the two expected subdirectories, is an operator mistake — and the fold below would swallow it
	// in silence. langpack_extend is an EXPLICIT opt-in in book.yaml: if it is set, the caller asserts that a
	// private canon exists, so a missing/typo'd path must stop the load, never degrade recall quietly.
	if fi, serr := os.Stat(overlayRoot); serr != nil || !fi.IsDir() {
		return nil, fmt.Errorf("langpack %q overlay root %s is not a readable directory (langpack_extend is an explicit opt-in — a missing or mistyped path would silently drop the book's private canon): %v", pair, overlayRoot, serr)
	}
	rootEnts, rerr := os.ReadDir(overlayRoot)
	if rerr != nil {
		return nil, fmt.Errorf("langpack %q overlay root %s: %w", pair, overlayRoot, rerr)
	}
	for _, e := range rootEnts {
		if e.IsDir() && (e.Name() == src || e.Name() == pair) {
			continue
		}
		return nil, fmt.Errorf("langpack %q overlay root %s: unexpected entry %q — an overlay holds exactly the %q/ and %q/ subdirectories (a data file dropped at the root, or a mistyped pair directory, would be silently ignored)", pair, overlayRoot, e.Name(), src, pair)
	}
	// FAIL LOUD on a misnamed overlay file (review finding, pair-14 scale lens): the fold loop below reads
	// ONLY the manifest names, so a typo (surname-compound.txt) or a non-mergeable file (heading.txt) shipped
	// in an overlay would be SILENTLY ignored — the book's private canon never reaches the miner, recall
	// degrades with no load-time signal. Enumerate the overlay dirs and refuse any unexpected file, keeping
	// the "never silently empty / drift-proof" guarantee the shared loader makes.
	mergeable := map[string]bool{}
	for _, n := range srcFiles {
		mergeable[src+"/"+n] = true
	}
	for _, n := range pairFiles {
		mergeable[pair+"/"+n] = true
	}
	for _, dir := range []string{src, pair} {
		names, derr := overlayDirFiles(filepath.Join(overlayRoot, dir))
		if derr != nil {
			return nil, fmt.Errorf("langpack %q overlay %s: %w", pair, dir, derr)
		}
		for _, n := range names {
			if !mergeable[dir+"/"+n] {
				return nil, fmt.Errorf("langpack %q overlay: unexpected file %s/%s — an overlay merges only the source/pair manifest files (a typo, or a non-mergeable file like heading.txt/dc-checkers.txt, would be silently ignored)", pair, dir, n)
			}
		}
	}
	// Seed a fresh hash with the base version (which already uniquely encodes every base byte), then fold the
	// overlay files in a fixed order — deterministic and drift-proof (edit the overlay → Version() moves).
	h := sha256.New()
	h.Write([]byte(p.version))
	merged := false
	fold := func(dir, name string, pairFile bool) error {
		b, ok, rerr := readOptional(overlayRoot, dir, name)
		if rerr != nil {
			return fmt.Errorf("langpack %q overlay %s/%s: %w", pair, dir, name, rerr)
		}
		if !ok {
			return nil
		}
		h.Write([]byte("\x00" + dir + "/" + name + "\x00"))
		h.Write(b)
		merged = true
		if pairFile {
			return p.mergePair(name, b)
		}
		return p.mergeSrc(name, b)
	}
	for _, name := range srcFiles {
		if err := fold(src, name, false); err != nil {
			return nil, err
		}
	}
	for _, name := range pairFiles {
		if err := fold(pair, name, true); err != nil {
			return nil, err
		}
	}
	if merged {
		p.version = packAlgoVersion + "-x" + hex.EncodeToString(h.Sum(nil))[:12]
	}
	return p, nil
}

// mergeSrc unions an overlay source file into the loaded pack (additive; see LoadWithOverlay). Rune/string
// SETS gain members; ordered slices append (a book's extra title/rank tokens follow the shared ones).
func (p *Pack) mergeSrc(name string, b []byte) error {
	switch name {
	case "surnames-single.txt":
		unionRuneSet(p.SurnamesSingle, runeSet(b))
	case "surnames-compound.txt":
		unionStringSet(p.SurnamesCompound, stringSet(b))
	case "title-suffix.txt":
		p.TitleSuffix = append(p.TitleSuffix, lines(b)...)
	case "ordinal-title.txt":
		p.OrdinalTitle = append(p.OrdinalTitle, lines(b)...)
	case "rank-word.txt":
		p.RankWord = append(p.RankWord, lines(b)...)
	case "topo-suffix.txt":
		unionRuneSet(p.TopoSuffix, runeSet(b))
	case "grade-prefix.txt":
		unionRuneSet(p.GradePrefix, runeSet(b))
	case "numeral.txt":
		unionRuneSet(p.Numeral, runeSet(b))
	case "alias-particle.txt":
		unionRuneSet(p.AliasParticle, runeSet(b))
	case "title-formant.txt":
		unionRuneSet(p.TitleFormant, runeSet(b))
	case "sentence-terminator.txt":
		unionRuneSet(p.SentenceTerminator, runeSet(b))
	default:
		return fmt.Errorf("unknown source file")
	}
	return nil
}

// mergePair unions an overlay pair file into the loaded pack (additive). Both pair files carry Palladius
// categories, parsed generically and unioned into p.Palladius (same path as assignPair).
func (p *Pack) mergePair(name string, b []byte) error {
	return p.assignPair(name, b)
}

func unionRuneSet(dst, src map[rune]bool) {
	for k := range src {
		dst[k] = true
	}
}

func unionStringSet(dst, src map[string]bool) {
	for k := range src {
		dst[k] = true
	}
}

// unionStringMap merges src into dst ADDITIVELY and refuses a COLLISION: a key already present in dst
// (the shared pack) that an overlay also carries would have SILENTLY overridden the base value. Override
// semantics are deliberately NOT ratified (D39.23) — an overlay exists to ADD a book's private canon, and
// a book that needs to change a shared pair mapping is a signal about the shared pack, not something to
// absorb quietly. Revisit if a book ever genuinely needs it; until then the collision is loud and names
// the key and both values. Sets/slices need no such rule: adding a member twice is idempotent.
func unionStringMap(category string, dst, src map[string]string) error {
	for k, v := range src {
		if old, ok := dst[k]; ok {
			return fmt.Errorf("category %q key %q is already defined as %q and the overlay redefines it as %q — an overlay may only ADD (override is not supported: change the shared pair pack instead)", category, k, old, v)
		}
		dst[k] = v
	}
	return nil
}

// validate makes the "never silently empty" contract real: a present-but-empty or comment-only data file
// (a fat-fingered edit once packs are hand-authored at R1) parses to an empty table with no error and would
// silently disable a miner channel — recall degradation with no load-time signal. Every required table must
// be non-empty. The content hash catches an EDIT (a --resnapshot signal), but "edited to empty" is a corrupt
// pack, not an intended change, so it is refused at load, before any consumer, like the missing-file path.
func (p *Pack) validate() error {
	var empty []string
	req := func(name string, n int) {
		if n == 0 {
			empty = append(empty, name)
		}
	}
	// A channel's REQUIRED tables are checked only when the source DECLARES that channel (a manifest-less pack
	// declares all — the original behaviour). A source that ships no name-morphology channel loads with empty
	// miner tables and stays inert, no Go edit.
	if p.channels[channelMorphology] {
		req("surnames-single", len(p.SurnamesSingle))
		req("surnames-compound", len(p.SurnamesCompound))
		req("title-suffix", len(p.TitleSuffix))
		req("ordinal-title", len(p.OrdinalTitle))
		req("rank-word", len(p.RankWord))
		req("topo-suffix", len(p.TopoSuffix))
		req("grade-prefix", len(p.GradePrefix))
		req("numeral", len(p.Numeral))
		req("alias-particle", len(p.AliasParticle))
		req("title-formant", len(p.TitleFormant))
		req("sentence-terminator", len(p.SentenceTerminator))
	}
	if p.channels[channelTransliteration] {
		// The Palladius REQUIRED-category list lives HERE (the consumer), not in the generic parser (addendum).
		req("palladius/initials", len(p.Palladius.Initials))
		req("palladius/finals", len(p.Palladius.Finals))
		req("palladius/yw", len(p.Palladius.YW))
		req("palladius/special_i", len(p.Palladius.SpecialI))
		req("palladius-phonotactics/retroflex", len(p.Palladius.Retroflex))
		req("palladius-phonotactics/vfinal", len(p.Palladius.VFinal))
		req("palladius-phonotactics/vfinal_initial", len(p.Palladius.VFinalInitial))
	}
	if len(empty) > 0 {
		return fmt.Errorf("empty required table(s) %s — a present-but-empty/comment-only data file is a corrupt pack, not a valid one", strings.Join(empty, ", "))
	}
	return nil
}

func (p *Pack) assignSrc(name string, b []byte) error {
	switch name {
	case "surnames-single.txt":
		p.SurnamesSingle = runeSet(b)
	case "surnames-compound.txt":
		p.SurnamesCompound = stringSet(b)
	case "title-suffix.txt":
		p.TitleSuffix = lines(b)
	case "ordinal-title.txt":
		p.OrdinalTitle = lines(b)
	case "rank-word.txt":
		p.RankWord = lines(b)
	case "topo-suffix.txt":
		p.TopoSuffix = runeSet(b)
	case "grade-prefix.txt":
		p.GradePrefix = runeSet(b)
	case "numeral.txt":
		p.Numeral = runeSet(b)
	case "alias-particle.txt":
		p.AliasParticle = runeSet(b)
	case "title-formant.txt":
		p.TitleFormant = runeSet(b)
	case "sentence-terminator.txt":
		p.SentenceTerminator = runeSet(b)
	default:
		return fmt.Errorf("unknown source file")
	}
	return nil
}

// assignPair reads a pair file into p.Palladius. Both pair files (palladius.txt, palladius-phonotactics.txt)
// carry `category<TAB>…` rows, parsed by ONE generic category parser (owner addendum 24.07); the known
// Palladius categories are UNIONED into p.Palladius and an unknown category is simply ignored, NEVER a parse
// error — which categories are REQUIRED is enforced downstream by validate(), not here.
func (p *Pack) assignPair(name string, b []byte) error {
	cats, err := parseCategoryRows(b)
	if err != nil {
		return err
	}
	if merr := p.Palladius.merge(cats); merr != nil {
		return merr
	}
	return nil
}

// overlayDirFiles lists the regular-file names directly in dir (an overlay's <src> or <pair> subdir). A
// MISSING dir is fine (returns nil — a book may overlay only sources or only the pair). Any other read
// error is loud. Nested dirs are ignored (only top-level manifest files are mergeable).
func overlayDirFiles(dir string) ([]string, error) {
	ents, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	var out []string
	for _, e := range ents {
		if !e.IsDir() {
			out = append(out, e.Name())
		}
	}
	return out, nil
}

// readManifest reads the OPTIONAL source manifest.txt (channel<TAB>name lines) into a channel SET. ABSENT →
// every channel (the original all-files-required contract, so a manifest-less pack is byte-identical). PRESENT
// → exactly the declared channels; an unknown channel name is a corrupt pack (fail loud, never a silent drop).
// Returns (channels, rawBytes, present, error); the caller folds rawBytes into the hash only when present.
func readManifest(root, sourceLang string) (map[string]bool, []byte, bool, error) {
	b, ok, err := readOptional(root, sourceLang, "manifest.txt")
	if err != nil {
		return nil, nil, false, err
	}
	ch := map[string]bool{}
	if !ok {
		for _, c := range allChannels {
			ch[c] = true // no manifest → every channel (backward-compatible)
		}
		return ch, nil, false, nil
	}
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
		if t == "" || strings.HasPrefix(t, "#") {
			continue
		}
		f := strings.SplitN(t, "\t", 2)
		if len(f) != 2 || strings.TrimSpace(f[0]) != "channel" {
			return nil, nil, false, fmt.Errorf("manifest line %d: want `channel<TAB>name`, got %q", i+1, t)
		}
		name := strings.TrimSpace(f[1])
		if name != channelMorphology && name != channelTransliteration {
			return nil, nil, false, fmt.Errorf("manifest line %d: unknown channel %q (want %s|%s)", i+1, name, channelMorphology, channelTransliteration)
		}
		ch[name] = true
	}
	if len(ch) == 0 {
		return nil, nil, false, fmt.Errorf("manifest declares no channel — omit the file to ship all channels, or list at least one")
	}
	return ch, b, true, nil
}

// readOptional reads an OPTIONAL pack file. A missing file returns (nil, false, nil) — the feature it
// backs is simply inert — while any OTHER read error (permission, a directory) is a loud failure; a
// present file returns (bytes, true, nil). Used for the pack-13 heading rule, which a pair opts into.
func readOptional(root, dir, name string) ([]byte, bool, error) {
	b, err := os.ReadFile(filepath.Join(root, dir, name))
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		}
		return nil, false, err
	}
	return b, true, nil
}

// parseHeading reads heading.txt into a HeadingRule. Format: `key<TAB>value` per non-comment line, keys
// marker | units | template. `marker` and `template` are REQUIRED; `units` is OPTIONAL — a rule that OMITS
// it matches a marker + number with NO unit rune («Chapter 12», «Глава 3»), while a unit-bearing rule
// («第<N>章») still requires the unit right after the number (the guard against a measure word «第一次»).
// `units` is a rune SET (each rune a member); `template` must contain the literal "{n}" placeholder (else it
// could never render a number). Fail-loud on a missing required key / unknown key / empty value / a template
// without {n} — a malformed rule is a corrupt pack, not an intended silent no-op.
func parseHeading(b []byte) (*HeadingRule, error) {
	hr := &HeadingRule{Units: map[rune]bool{}}
	seen := map[string]bool{}
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimRight(raw, "\r")
		if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
			continue
		}
		f := strings.SplitN(t, "\t", 2)
		if len(f) != 2 || strings.TrimSpace(f[1]) == "" {
			return nil, fmt.Errorf("line %d: want `key<TAB>value` with a non-empty value (%q)", i+1, t)
		}
		key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
		seen[key] = true
		switch key {
		case "marker":
			hr.Marker = val
		case "units":
			for _, r := range val {
				if !isSpace(r) {
					hr.Units[r] = true
				}
			}
		case "template":
			hr.Template = val
		default:
			return nil, fmt.Errorf("line %d: unknown key %q (want marker|units|template)", i+1, key)
		}
	}
	if !seen["marker"] || !seen["template"] {
		return nil, fmt.Errorf("heading rule needs a marker and a template (units are optional)")
	}
	if !strings.Contains(hr.Template, "{n}") {
		return nil, fmt.Errorf("template %q must contain the {n} number placeholder", hr.Template)
	}
	return hr, nil
}

// parseSpeechCue reads speech-cue.txt into a SpeechCue. Format: `key<TAB>value` per non-comment line,
// keys cue | quote_open | quote_close | window. A file that ships cues but no quote marks (or the
// reverse) can never attribute anything, so it is refused rather than loaded inert — the same
// "never silently empty" contract the required tables keep.
func parseSpeechCue(b []byte) (*SpeechCue, error) {
	sc := &SpeechCue{QuoteOpen: map[rune]bool{}, QuoteClose: map[rune]bool{}}
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
		if t == "" || strings.HasPrefix(t, "#") {
			continue
		}
		f := strings.SplitN(t, "\t", 2)
		if len(f) != 2 || strings.TrimSpace(f[1]) == "" {
			return nil, fmt.Errorf("line %d: want `key<TAB>value` with a non-empty value (%q)", i+1, t)
		}
		key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
		switch key {
		case "cue":
			sc.Cues = append(sc.Cues, val)
		case "quote_open", "quote_close":
			r := []rune(val)
			if len(r) != 1 {
				return nil, fmt.Errorf("line %d: %s %q must be a single rune", i+1, key, val)
			}
			if key == "quote_open" {
				sc.QuoteOpen[r[0]] = true
			} else {
				sc.QuoteClose[r[0]] = true
			}
		case "window":
			n, err := strconv.Atoi(val)
			if err != nil || n <= 0 {
				return nil, fmt.Errorf("line %d: window must be a positive integer (%q)", i+1, val)
			}
			sc.Window = n
		default:
			return nil, fmt.Errorf("line %d: unknown key %q (want cue|quote_open|quote_close|window)", i+1, key)
		}
	}
	if len(sc.Cues) == 0 || len(sc.QuoteOpen) == 0 || len(sc.QuoteClose) == 0 {
		return nil, fmt.Errorf("speech-cue needs at least one cue and both quote_open and quote_close (a half table can never attribute anything)")
	}
	if sc.Window == 0 {
		sc.Window = 12 // engine default: a name+cue sits within a dozen runes of the quote in practice
	}
	return sc, nil
}

// parseTerminology reads terminology.txt into a TerminologySizing. Format: `key<TAB>value`, keys
// kwic_per_term | kwic_width, each optional but positive when present. Fail-loud like parseHeading; an
// empty file is refused too, because its bytes move Version() while changing nothing.
func parseTerminology(b []byte) (*TerminologySizing, error) {
	ts := &TerminologySizing{}
	seen := false
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimRight(raw, "\r")
		if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
			continue
		}
		f := strings.SplitN(t, "\t", 2)
		if len(f) != 2 || strings.TrimSpace(f[1]) == "" {
			return nil, fmt.Errorf("line %d: want `key<TAB>value` with a non-empty value (%q)", i+1, t)
		}
		key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
		n, err := strconv.Atoi(val)
		if err != nil || n <= 0 {
			return nil, fmt.Errorf("line %d: %s must be a positive integer (%q)", i+1, key, val)
		}
		switch key {
		case "kwic_per_term":
			ts.KWICPerTerm = n
		case "kwic_width":
			ts.KWICWidth = n
		default:
			return nil, fmt.Errorf("line %d: unknown key %q (want kwic_per_term|kwic_width)", i+1, key)
		}
		seen = true
	}
	if !seen {
		return nil, fmt.Errorf("the file states nothing (want at least one of kwic_per_term|kwic_width); delete it instead — an empty table moves the pack version without changing behaviour")
	}
	return ts, nil
}

// runeSet reads a rune SET: every non-whitespace rune of every non-comment line is a member (order-free).
func runeSet(b []byte) map[rune]bool {
	m := map[rune]bool{}
	for _, ln := range contentLines(b) {
		for _, r := range ln {
			if !isSpace(r) {
				m[r] = true
			}
		}
	}
	return m
}

// stringSet reads a set of whole tokens, one per non-comment line (trimmed).
func stringSet(b []byte) map[string]bool {
	m := map[string]bool{}
	for _, ln := range contentLines(b) {
		if t := strings.TrimSpace(ln); t != "" {
			m[t] = true
		}
	}
	return m
}

// lines reads an ORDERED slice of whole tokens, one per non-comment line (trimmed), in file order.
func lines(b []byte) []string {
	var out []string
	for _, ln := range contentLines(b) {
		if t := strings.TrimSpace(ln); t != "" {
			out = append(out, t)
		}
	}
	return out
}

// parseCategoryRows is the GENERIC Palladius category parser (owner addendum 24.07): it reads a pair file's
// `category<TAB>key[<TAB>value]` rows into category → key → value, WITHOUT a per-category switch and WITHOUT
// treating an unknown category as an error (which categories are required is the CONSUMER's call — validate()).
// A 3-field row (initials/finals/yw/special_i) stores key→cyrillic; a 2-field row (retroflex/vfinal/…) stores
// key→"" (a set member). Scans raw lines so an error names the PHYSICAL file line; the only parse errors are a
// bad field count / an empty key.
func parseCategoryRows(b []byte) (map[string]map[string]string, error) {
	cats := map[string]map[string]string{}
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
		if t == "" || strings.HasPrefix(t, "#") {
			continue
		}
		f := strings.Split(t, "\t")
		if len(f) < 2 || len(f) > 3 || strings.TrimSpace(f[0]) == "" || strings.TrimSpace(f[1]) == "" {
			return nil, fmt.Errorf("line %d: want `category<TAB>key[<TAB>value]`, got %q", i+1, t)
		}
		if cats[f[0]] == nil {
			cats[f[0]] = map[string]string{}
		}
		val := "" // a 2-field row is a set member (value "")
		if len(f) == 3 {
			val = f[2]
		}
		cats[f[0]][f[1]] = val
	}
	return cats, nil
}

// palladiusCategories are the categories this CONSUMER reads. The generic parser stays category-agnostic
// (an unknown category is not a parse error — owner addendum 24.07); the CONSUMER is where a category that
// nothing consumes is caught, because only the consumer knows what "consumed" means.
var palladiusCategories = map[string]bool{
	"initials": true, "finals": true, "yw": true, "special_i": true,
	"retroflex": true, "vfinal": true, "vfinal_initial": true,
}

// merge unions parsed categories into the Palladius table. KNOWN categories populate the typed fields (a
// map category takes key→cyrillic, a set category takes its keys). A category NOTHING consumes is an
// ERROR here (pack-15): a typo (`retroflexx`, `initals`) parses fine and used to vanish — the file looked
// authored, the table stayed empty for that class, and the miner quietly lost a channel. A map-category
// key COLLISION between the base pack and an overlay is loud for the same reason (see unionStringMap).
func (pal *Palladius) merge(cats map[string]map[string]string) error {
	var unknown []string
	for c := range cats {
		if !palladiusCategories[c] {
			unknown = append(unknown, c)
		}
	}
	if len(unknown) > 0 {
		sort.Strings(unknown)
		known := make([]string, 0, len(palladiusCategories))
		for c := range palladiusCategories {
			known = append(known, c)
		}
		sort.Strings(known)
		return fmt.Errorf("category %s is not consumed by anything (a typo would parse fine and silently leave its table empty); known categories: %s",
			strings.Join(unknown, ", "), strings.Join(known, ", "))
	}
	for _, m := range []struct {
		name string
		dst  map[string]string
	}{
		{"initials", pal.Initials}, {"finals", pal.Finals}, {"yw", pal.YW}, {"special_i", pal.SpecialI},
	} {
		if err := unionStringMap(m.name, m.dst, cats[m.name]); err != nil {
			return err
		}
	}
	unionKeysAsSet(pal.Retroflex, cats["retroflex"])
	unionKeysAsSet(pal.VFinal, cats["vfinal"])
	unionKeysAsSet(pal.VFinalInitial, cats["vfinal_initial"])
	return nil
}

// unionKeysAsSet adds the KEYS of a parsed category (a 2-field set) to dst.
func unionKeysAsSet(dst map[string]bool, src map[string]string) {
	for k := range src {
		dst[k] = true
	}
}

// parseDCCheckers reads the DC-checker pair tables from category-keyed lines (checkers_zh_ru.go data):
//
//	cjk_numeral<TAB>rune<TAB>value | ru_hour<TAB>word<TAB>value | register_neg<TAB>lexeme
//	pattern<TAB>key<TAB>value (VERBATIM) | msg<TAB>key<TAB>template (VERBATIM) | dc_ratio<TAB>key<TAB>int
//
// Scans raw lines so an error names the PHYSICAL file line. Fail-loud on a bad field count / non-integer
// value / unknown category (a malformed table is a corrupt pack). RegisterNeg keeps its authored order.
func parseDCCheckers(b []byte) (*DCCheckerData, error) {
	d := &DCCheckerData{
		Numeral: map[rune]int{}, RuHours: map[string]int{},
		Patterns: map[string]string{}, Ratios: map[string]int{}, Messages: map[string]string{},
	}
	for i, raw := range strings.Split(string(b), "\n") {
		// A `pattern` / `msg` line's VALUE is verbatim (a regex may carry trailing metachars, a message its
		// own spacing), so only \r is stripped from the whole line, not the value; other categories tolerate
		// the trimmed form.
		line := strings.TrimRight(raw, "\r")
		t := strings.TrimSpace(line)
		if t == "" || strings.HasPrefix(t, "#") {
			continue
		}
		if strings.HasPrefix(line, "pattern\t") || strings.HasPrefix(line, "msg\t") {
			f := strings.SplitN(line, "\t", 3) // value (f[2]) VERBATIM
			if len(f) != 3 || strings.TrimSpace(f[1]) == "" || f[2] == "" {
				return nil, fmt.Errorf("line %d: %s wants `%s<TAB>key<TAB>value`, got %q", i+1, f[0], f[0], line)
			}
			if f[0] == "msg" {
				d.Messages[f[1]] = f[2]
			} else {
				d.Patterns[f[1]] = f[2]
			}
			continue
		}
		f := strings.Split(t, "\t")
		switch f[0] {
		case "cjk_numeral":
			if len(f) != 3 {
				return nil, fmt.Errorf("line %d: cjk_numeral wants `cjk_numeral<TAB>rune<TAB>value`, got %q", i+1, t)
			}
			r := []rune(f[1])
			if len(r) != 1 {
				return nil, fmt.Errorf("line %d: cjk_numeral key %q must be a single rune", i+1, f[1])
			}
			v, err := strconv.Atoi(strings.TrimSpace(f[2]))
			if err != nil {
				return nil, fmt.Errorf("line %d: cjk_numeral value %q: %w", i+1, f[2], err)
			}
			d.Numeral[r[0]] = v
		case "ru_hour":
			if len(f) != 3 {
				return nil, fmt.Errorf("line %d: ru_hour wants `ru_hour<TAB>word<TAB>value`, got %q", i+1, t)
			}
			v, err := strconv.Atoi(strings.TrimSpace(f[2]))
			if err != nil {
				return nil, fmt.Errorf("line %d: ru_hour value %q: %w", i+1, f[2], err)
			}
			d.RuHours[f[1]] = v
		case "register_neg":
			if len(f) != 2 || strings.TrimSpace(f[1]) == "" {
				return nil, fmt.Errorf("line %d: register_neg wants `register_neg<TAB>lexeme`, got %q", i+1, t)
			}
			d.RegisterNeg = append(d.RegisterNeg, f[1])
		case "dc_ratio":
			if len(f) != 3 {
				return nil, fmt.Errorf("line %d: dc_ratio wants `dc_ratio<TAB>key<TAB>value`, got %q", i+1, t)
			}
			v, err := strconv.Atoi(strings.TrimSpace(f[2]))
			if err != nil {
				return nil, fmt.Errorf("line %d: dc_ratio value %q: %w", i+1, f[2], err)
			}
			d.Ratios[f[1]] = v
		default:
			return nil, fmt.Errorf("line %d: unknown category %q (want cjk_numeral|ru_hour|register_neg|dc_ratio|pattern|msg)", i+1, f[0])
		}
	}
	if len(d.Numeral) == 0 || len(d.RuHours) == 0 || len(d.RegisterNeg) == 0 {
		return nil, fmt.Errorf("dc-checkers needs non-empty cjk_numeral, ru_hour and register_neg sections")
	}
	return d, nil
}

// contentLines splits into lines, dropping '#'-comment and blank lines.
func contentLines(b []byte) []string {
	var out []string
	for _, ln := range strings.Split(string(b), "\n") {
		s := strings.TrimRight(ln, "\r")
		if strings.HasPrefix(strings.TrimSpace(s), "#") || strings.TrimSpace(s) == "" {
			continue
		}
		out = append(out, s)
	}
	return out
}

func isSpace(r rune) bool { return r == ' ' || r == '\t' || r == '\n' || r == '\r' }