textmachine/backend/internal/lang/langpack.go

// Package lang holds the language-specific DATA the translation engine reads, isolated OUT of
// internal/pipeline/ as versioned files under configs/langpacks/ (owner directive D39.15: language data
// must not live as Go constants inside the engine; horizon = hundreds of languages, data-as-files, add a
// pair = drop a directory, no recompile, no pipeline/ edits).
//
// This package carries NO behaviour. The miner / checker ALGORITHMS stay in internal/pipeline (parity-
// locked, owner: "from the engine, DATA leaves; the ALGORITHM stays") and READ these tables. The one-way
// dependency (pipeline → lang, never the reverse) is compile-enforced: nothing here imports pipeline, so
// the data/algorithm boundary is real, not a convention.
//
// A Pack is resolved by (source, target) language and content-hashed at load, mirroring the two in-repo
// precedents: the pair-keyed prompt seam (config.Stage.Prompts + PromptPathFor + PromptSHA256, resolved
// by Book.LangPair(), fail-loud on a missing pair) and the memnorm trad→simp table (a data file whose
// bytes ARE its version via a content hash). Editing a pack file changes its Version() → the pipeline
// folds that into the snapshot → a loud --resnapshot, drift-proof by mechanism, not discipline.
package lang

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

// packAlgoVersion tags the PARSE/layout of a pack (the file manifest + how each file is read). Bump it on
// a schema change (a new file, a format change). The DATA content is versioned separately by hashing the
// files into Version(), so editing a table also invalidates — you cannot forget to bump a version when
// you change the data, because the data's bytes ARE the version (the memnorm.go drift-proofing).
const packAlgoVersion = "langpack-v1"

// Pack is a loaded, versioned language-data pack for one source→target pair. Fields are the DATA the
// pipeline algorithms read; the zero value is unusable (load via Load). Maps are membership sets / lookup
// tables (order-free); slices preserve their authored order.
type Pack struct {
	Pair string // "<src>-<tgt>", e.g. "zh-ru" (== config.Book.LangPair())

	// <src> source morphology (configs/langpacks/<src>/) — the miner's typed-candidate channels.
	SurnamesSingle   map[rune]bool   // 百家姓 single-char surnames
	SurnamesCompound map[string]bool // two-char compound surnames
	TitleSuffix      []string        // title suffixes → title (ordered)
	OrdinalTitle     []string        // ordinal titles 一代/第一… (ordered)
	RankWord         []string        // rank/measure words 等/转… (ordered)
	TopoSuffix       map[rune]bool   // topographic suffix chars → place
	GradePrefix      map[rune]bool   // grade/stem prefix chars 甲乙丙…
	Numeral          map[rune]bool   // CJK numeral chars
	AliasParticle    map[rune]bool   // trailing-particle set marking a boundary fragment

	// <src>-<tgt> pair transliteration (configs/langpacks/<pair>/) — the Palladius (Палладий) table.
	PalladiusInitials map[string]string
	PalladiusFinals   map[string]string
	PalladiusYW       map[string]string
	PalladiusSpecialI map[string]string

	// Heading is the OPTIONAL chapter-heading rule (configs/langpacks/<pair>/heading.txt). nil when the pair
	// carries no heading.txt — the chapter-title feature is then inert (the chunker keeps the source header
	// as-is), so a pair that does not opt in is never re-billed for it. It is DATA only: the detect/strip/
	// render ALGORITHM lives in the chunker (internal/pipeline), which reads this table — the same
	// data/algorithm boundary the miner tables keep.
	Heading *HeadingRule

	version string
}

// HeadingRule is the per-pair data for the chapter-title policy: instead of letting the model render a
// chapter heading (which drifted to «Раздел 2» / «Первая глава» / an orphaned « :» across models), the
// chunker detects a source header (Marker + a numeral + a Unit rune), strips it from the model input, and
// the read-models render Template deterministically instead. Data only; parsed from heading.txt.
type HeadingRule struct {
	Marker   string        // the prefix rune(s) that open a numbered heading, e.g. "第"
	Units    map[rune]bool // the section-unit runes accepted right after the numeral (章 节 節 回)
	Template string        // the target rendering; the literal "{n}" is replaced by the parsed Arabic number
}

// Version is the content hash of the pack (packAlgoVersion + a sha256 of the authored file bytes). A pack
// edit changes it, so the pipeline can fold it into the snapshot (a loud --resnapshot on any data edit).
func (p *Pack) Version() string { return p.version }

// srcFiles are the source-morphology files (under configs/langpacks/<src>/), read in this fixed order.
var srcFiles = []string{
	"surnames-single.txt", "surnames-compound.txt", "title-suffix.txt", "ordinal-title.txt",
	"rank-word.txt", "topo-suffix.txt", "grade-prefix.txt", "numeral.txt", "alias-particle.txt",
}

// pairFiles are the pair-transliteration files (under configs/langpacks/<pair>/).
var pairFiles = []string{"palladius.txt"}

// Load resolves and reads the pack for (sourceLang, targetLang) from root (e.g. "configs/langpacks"): the
// source-morphology files under root/<src>/ and the pair-transliteration files under root/<src>-<tgt>/.
// EVERY declared file must be present and well-formed — a missing/corrupt file fails LOUD (the caller runs
// this at load, before any billing, mirroring the prompt-pack os.Stat loop). The content hash covers all
// files in a fixed order, so it is deterministic and drift-proof.
func Load(root, sourceLang, targetLang string) (*Pack, error) {
	pair := sourceLang + "-" + targetLang
	p := &Pack{Pair: pair}
	h := sha256.New()
	h.Write([]byte(packAlgoVersion))

	read := func(dir, name string) ([]byte, error) {
		path := filepath.Join(root, dir, name)
		b, err := os.ReadFile(path)
		if err != nil {
			return nil, fmt.Errorf("langpack %q: %w (add the file or fix the book's source_lang/target_lang; D39.15: language data is required, never silently empty)", pair, err)
		}
		// Fold the RELATIVE path + bytes so a rename or a moved byte both shift the hash.
		h.Write([]byte("\x00" + dir + "/" + name + "\x00"))
		h.Write(b)
		return b, nil
	}

	for _, name := range srcFiles {
		b, err := read(sourceLang, name)
		if err != nil {
			return nil, err
		}
		if err := p.assignSrc(name, b); err != nil {
			return nil, fmt.Errorf("langpack %q %s: %w", pair, name, err)
		}
	}
	for _, name := range pairFiles {
		b, err := read(pair, name)
		if err != nil {
			return nil, err
		}
		if err := p.assignPair(name, b); err != nil {
			return nil, fmt.Errorf("langpack %q %s: %w", pair, name, err)
		}
	}

	// Optional per-pair heading rule (pack-13 title policy). ABSENT → nil, the title feature is inert and
	// the pack's Version() is byte-stable (a pair that does not opt in is never re-billed); PRESENT → its
	// bytes fold into the content hash (a loud --resnapshot on any edit) and it is parsed; CORRUPT → fail
	// loud, like the required-file path. Distinct from read(): a missing heading.txt is NOT an error.
	if hb, ok, herr := readOptional(root, pair, "heading.txt"); herr != nil {
		return nil, fmt.Errorf("langpack %q heading.txt: %w", pair, herr)
	} else if ok {
		h.Write([]byte("\x00" + pair + "/heading.txt\x00"))
		h.Write(hb)
		hr, perr := parseHeading(hb)
		if perr != nil {
			return nil, fmt.Errorf("langpack %q heading.txt: %w", pair, perr)
		}
		p.Heading = hr
	}

	if err := p.validate(); err != nil {
		return nil, fmt.Errorf("langpack %q: %w", pair, err)
	}
	p.version = packAlgoVersion + "-" + hex.EncodeToString(h.Sum(nil))[:12]
	return p, nil
}

// validate makes the "never silently empty" contract real: a present-but-empty or comment-only data file
// (a fat-fingered edit once packs are hand-authored at R1) parses to an empty table with no error and would
// silently disable a miner channel — recall degradation with no load-time signal. Every required table must
// be non-empty. The content hash catches an EDIT (a --resnapshot signal), but "edited to empty" is a corrupt
// pack, not an intended change, so it is refused at load, before any consumer, like the missing-file path.
func (p *Pack) validate() error {
	var empty []string
	req := func(name string, n int) {
		if n == 0 {
			empty = append(empty, name)
		}
	}
	req("surnames-single", len(p.SurnamesSingle))
	req("surnames-compound", len(p.SurnamesCompound))
	req("title-suffix", len(p.TitleSuffix))
	req("ordinal-title", len(p.OrdinalTitle))
	req("rank-word", len(p.RankWord))
	req("topo-suffix", len(p.TopoSuffix))
	req("grade-prefix", len(p.GradePrefix))
	req("numeral", len(p.Numeral))
	req("alias-particle", len(p.AliasParticle))
	req("palladius/initials", len(p.PalladiusInitials))
	req("palladius/finals", len(p.PalladiusFinals))
	req("palladius/yw", len(p.PalladiusYW))
	req("palladius/special_i", len(p.PalladiusSpecialI))
	if len(empty) > 0 {
		return fmt.Errorf("empty required table(s) %s — a present-but-empty/comment-only data file is a corrupt pack, not a valid one", strings.Join(empty, ", "))
	}
	return nil
}

func (p *Pack) assignSrc(name string, b []byte) error {
	switch name {
	case "surnames-single.txt":
		p.SurnamesSingle = runeSet(b)
	case "surnames-compound.txt":
		p.SurnamesCompound = stringSet(b)
	case "title-suffix.txt":
		p.TitleSuffix = lines(b)
	case "ordinal-title.txt":
		p.OrdinalTitle = lines(b)
	case "rank-word.txt":
		p.RankWord = lines(b)
	case "topo-suffix.txt":
		p.TopoSuffix = runeSet(b)
	case "grade-prefix.txt":
		p.GradePrefix = runeSet(b)
	case "numeral.txt":
		p.Numeral = runeSet(b)
	case "alias-particle.txt":
		p.AliasParticle = runeSet(b)
	default:
		return fmt.Errorf("unknown source file")
	}
	return nil
}

func (p *Pack) assignPair(name string, b []byte) error {
	switch name {
	case "palladius.txt":
		ini, fin, yw, si, err := parsePalladius(b)
		if err != nil {
			return err
		}
		p.PalladiusInitials, p.PalladiusFinals, p.PalladiusYW, p.PalladiusSpecialI = ini, fin, yw, si
	default:
		return fmt.Errorf("unknown pair file")
	}
	return nil
}

// readOptional reads an OPTIONAL pack file. A missing file returns (nil, false, nil) — the feature it
// backs is simply inert — while any OTHER read error (permission, a directory) is a loud failure; a
// present file returns (bytes, true, nil). Used for the pack-13 heading rule, which a pair opts into.
func readOptional(root, dir, name string) ([]byte, bool, error) {
	b, err := os.ReadFile(filepath.Join(root, dir, name))
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		}
		return nil, false, err
	}
	return b, true, nil
}

// parseHeading reads heading.txt into a HeadingRule. Format: `key<TAB>value` per non-comment line, keys
// marker | units | template (all three required). `units` is a rune SET (each rune a member); `template`
// must contain the literal "{n}" placeholder (else it could never render a number). Fail-loud on a missing
// key / unknown key / empty value / a template without {n} — a malformed rule is a corrupt pack, not an
// intended silent no-op (mirrors the required-file "never silently empty" contract).
func parseHeading(b []byte) (*HeadingRule, error) {
	hr := &HeadingRule{Units: map[rune]bool{}}
	seen := map[string]bool{}
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimRight(raw, "\r")
		if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
			continue
		}
		f := strings.SplitN(t, "\t", 2)
		if len(f) != 2 || strings.TrimSpace(f[1]) == "" {
			return nil, fmt.Errorf("line %d: want `key<TAB>value` with a non-empty value (%q)", i+1, t)
		}
		key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
		seen[key] = true
		switch key {
		case "marker":
			hr.Marker = val
		case "units":
			for _, r := range val {
				if !isSpace(r) {
					hr.Units[r] = true
				}
			}
		case "template":
			hr.Template = val
		default:
			return nil, fmt.Errorf("line %d: unknown key %q (want marker|units|template)", i+1, key)
		}
	}
	if !seen["marker"] || !seen["units"] || !seen["template"] || len(hr.Units) == 0 {
		return nil, fmt.Errorf("heading rule needs non-empty marker, units and template")
	}
	if !strings.Contains(hr.Template, "{n}") {
		return nil, fmt.Errorf("template %q must contain the {n} number placeholder", hr.Template)
	}
	return hr, nil
}

// runeSet reads a rune SET: every non-whitespace rune of every non-comment line is a member (order-free).
func runeSet(b []byte) map[rune]bool {
	m := map[rune]bool{}
	for _, ln := range contentLines(b) {
		for _, r := range ln {
			if !isSpace(r) {
				m[r] = true
			}
		}
	}
	return m
}

// stringSet reads a set of whole tokens, one per non-comment line (trimmed).
func stringSet(b []byte) map[string]bool {
	m := map[string]bool{}
	for _, ln := range contentLines(b) {
		if t := strings.TrimSpace(ln); t != "" {
			m[t] = true
		}
	}
	return m
}

// lines reads an ORDERED slice of whole tokens, one per non-comment line (trimmed), in file order.
func lines(b []byte) []string {
	var out []string
	for _, ln := range contentLines(b) {
		if t := strings.TrimSpace(ln); t != "" {
			out = append(out, t)
		}
	}
	return out
}

// parsePalladius reads the `category<TAB>pinyin<TAB>cyrillic` table into the four maps. It scans the raw
// lines directly (not contentLines) so an error names the PHYSICAL file line — the point of the diagnostic
// is to send a human editing the table to the right line.
func parsePalladius(b []byte) (ini, fin, yw, si map[string]string, err error) {
	ini, fin, yw, si = map[string]string{}, map[string]string{}, map[string]string{}, map[string]string{}
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
		if t == "" || strings.HasPrefix(t, "#") {
			continue
		}
		f := strings.Split(t, "\t")
		if len(f) != 3 {
			return nil, nil, nil, nil, fmt.Errorf("line %d: want 3 tab-separated fields, got %d (%q)", i+1, len(f), t)
		}
		switch f[0] {
		case "initials":
			ini[f[1]] = f[2]
		case "finals":
			fin[f[1]] = f[2]
		case "yw":
			yw[f[1]] = f[2]
		case "special_i":
			si[f[1]] = f[2]
		default:
			return nil, nil, nil, nil, fmt.Errorf("line %d: unknown category %q", i+1, f[0])
		}
	}
	return ini, fin, yw, si, nil
}

// contentLines splits into lines, dropping '#'-comment and blank lines.
func contentLines(b []byte) []string {
	var out []string
	for _, ln := range strings.Split(string(b), "\n") {
		s := strings.TrimRight(ln, "\r")
		if strings.HasPrefix(strings.TrimSpace(s), "#") || strings.TrimSpace(s) == "" {
			continue
		}
		out = append(out, s)
	}
	return out
}

func isSpace(r rune) bool { return r == ' ' || r == '\t' || r == '\n' || r == '\r' }