textmachine/backend/internal/lang/langpack.go

// Package lang holds the language-specific DATA the translation engine reads, isolated OUT of
// internal/pipeline/ as versioned files under configs/langpacks/ (owner directive D39.15: language data
// must not live as Go constants inside the engine; horizon = hundreds of languages, data-as-files, add a
// pair = drop a directory, no recompile, no pipeline/ edits).
//
// This package carries NO behaviour. The miner / checker ALGORITHMS stay in internal/pipeline (parity-
// locked, owner: "from the engine, DATA leaves; the ALGORITHM stays") and READ these tables. The one-way
// dependency (pipeline → lang, never the reverse) is compile-enforced: nothing here imports pipeline, so
// the data/algorithm boundary is real, not a convention.
//
// A Pack is resolved by (source, target) language and content-hashed at load, mirroring the two in-repo
// precedents: the pair-keyed prompt seam (config.Stage.Prompts + PromptPathFor + PromptSHA256, resolved
// by Book.LangPair(), fail-loud on a missing pair) and the memnorm trad→simp table (a data file whose
// bytes ARE its version via a content hash). Editing a pack file changes its Version() → the pipeline
// folds that into the snapshot → a loud --resnapshot, drift-proof by mechanism, not discipline.
package lang

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

// packAlgoVersion tags the PARSE/layout of a pack (the file manifest + how each file is read). Bump it on
// a schema change (a new file, a format change). The DATA content is versioned separately by hashing the
// files into Version(), so editing a table also invalidates — you cannot forget to bump a version when
// you change the data, because the data's bytes ARE the version (the memnorm.go drift-proofing).
const packAlgoVersion = "langpack-v1"

// Pack is a loaded, versioned language-data pack for one source→target pair. Fields are the DATA the
// pipeline algorithms read; the zero value is unusable (load via Load). Maps are membership sets / lookup
// tables (order-free); slices preserve their authored order.
type Pack struct {
	Pair string // "<src>-<tgt>", e.g. "zh-ru" (== config.Book.LangPair())

	// <src> source morphology (configs/langpacks/<src>/) — the miner's typed-candidate channels.
	SurnamesSingle   map[rune]bool   // 百家姓 single-char surnames
	SurnamesCompound map[string]bool // two-char compound surnames
	TitleSuffix      []string        // title suffixes → title (ordered)
	OrdinalTitle     []string        // ordinal titles 一代/第一… (ordered)
	RankWord         []string        // rank/measure words 等/转… (ordered)
	TopoSuffix       map[rune]bool   // topographic suffix chars → place
	GradePrefix      map[rune]bool   // grade/stem prefix chars 甲乙丙…
	Numeral          map[rune]bool   // CJK numeral chars
	AliasParticle    map[rune]bool   // trailing-particle set marking a boundary fragment

	// <src>-<tgt> pair transliteration (configs/langpacks/<pair>/) — the Palladius (Палладий) table.
	PalladiusInitials map[string]string
	PalladiusFinals   map[string]string
	PalladiusYW       map[string]string
	PalladiusSpecialI map[string]string

	version string
}

// Version is the content hash of the pack (packAlgoVersion + a sha256 of the authored file bytes). A pack
// edit changes it, so the pipeline can fold it into the snapshot (a loud --resnapshot on any data edit).
func (p *Pack) Version() string { return p.version }

// srcFiles are the source-morphology files (under configs/langpacks/<src>/), read in this fixed order.
var srcFiles = []string{
	"surnames-single.txt", "surnames-compound.txt", "title-suffix.txt", "ordinal-title.txt",
	"rank-word.txt", "topo-suffix.txt", "grade-prefix.txt", "numeral.txt", "alias-particle.txt",
}

// pairFiles are the pair-transliteration files (under configs/langpacks/<pair>/).
var pairFiles = []string{"palladius.txt"}

// Load resolves and reads the pack for (sourceLang, targetLang) from root (e.g. "configs/langpacks"): the
// source-morphology files under root/<src>/ and the pair-transliteration files under root/<src>-<tgt>/.
// EVERY declared file must be present and well-formed — a missing/corrupt file fails LOUD (the caller runs
// this at load, before any billing, mirroring the prompt-pack os.Stat loop). The content hash covers all
// files in a fixed order, so it is deterministic and drift-proof.
func Load(root, sourceLang, targetLang string) (*Pack, error) {
	pair := sourceLang + "-" + targetLang
	p := &Pack{Pair: pair}
	h := sha256.New()
	h.Write([]byte(packAlgoVersion))

	read := func(dir, name string) ([]byte, error) {
		path := filepath.Join(root, dir, name)
		b, err := os.ReadFile(path)
		if err != nil {
			return nil, fmt.Errorf("langpack %q: %w (add the file or fix the book's source_lang/target_lang; D39.15: language data is required, never silently empty)", pair, err)
		}
		// Fold the RELATIVE path + bytes so a rename or a moved byte both shift the hash.
		h.Write([]byte("\x00" + dir + "/" + name + "\x00"))
		h.Write(b)
		return b, nil
	}

	for _, name := range srcFiles {
		b, err := read(sourceLang, name)
		if err != nil {
			return nil, err
		}
		if err := p.assignSrc(name, b); err != nil {
			return nil, fmt.Errorf("langpack %q %s: %w", pair, name, err)
		}
	}
	for _, name := range pairFiles {
		b, err := read(pair, name)
		if err != nil {
			return nil, err
		}
		if err := p.assignPair(name, b); err != nil {
			return nil, fmt.Errorf("langpack %q %s: %w", pair, name, err)
		}
	}

	if err := p.validate(); err != nil {
		return nil, fmt.Errorf("langpack %q: %w", pair, err)
	}
	p.version = packAlgoVersion + "-" + hex.EncodeToString(h.Sum(nil))[:12]
	return p, nil
}

// validate makes the "never silently empty" contract real: a present-but-empty or comment-only data file
// (a fat-fingered edit once packs are hand-authored at R1) parses to an empty table with no error and would
// silently disable a miner channel — recall degradation with no load-time signal. Every required table must
// be non-empty. The content hash catches an EDIT (a --resnapshot signal), but "edited to empty" is a corrupt
// pack, not an intended change, so it is refused at load, before any consumer, like the missing-file path.
func (p *Pack) validate() error {
	var empty []string
	req := func(name string, n int) {
		if n == 0 {
			empty = append(empty, name)
		}
	}
	req("surnames-single", len(p.SurnamesSingle))
	req("surnames-compound", len(p.SurnamesCompound))
	req("title-suffix", len(p.TitleSuffix))
	req("ordinal-title", len(p.OrdinalTitle))
	req("rank-word", len(p.RankWord))
	req("topo-suffix", len(p.TopoSuffix))
	req("grade-prefix", len(p.GradePrefix))
	req("numeral", len(p.Numeral))
	req("alias-particle", len(p.AliasParticle))
	req("palladius/initials", len(p.PalladiusInitials))
	req("palladius/finals", len(p.PalladiusFinals))
	req("palladius/yw", len(p.PalladiusYW))
	req("palladius/special_i", len(p.PalladiusSpecialI))
	if len(empty) > 0 {
		return fmt.Errorf("empty required table(s) %s — a present-but-empty/comment-only data file is a corrupt pack, not a valid one", strings.Join(empty, ", "))
	}
	return nil
}

func (p *Pack) assignSrc(name string, b []byte) error {
	switch name {
	case "surnames-single.txt":
		p.SurnamesSingle = runeSet(b)
	case "surnames-compound.txt":
		p.SurnamesCompound = stringSet(b)
	case "title-suffix.txt":
		p.TitleSuffix = lines(b)
	case "ordinal-title.txt":
		p.OrdinalTitle = lines(b)
	case "rank-word.txt":
		p.RankWord = lines(b)
	case "topo-suffix.txt":
		p.TopoSuffix = runeSet(b)
	case "grade-prefix.txt":
		p.GradePrefix = runeSet(b)
	case "numeral.txt":
		p.Numeral = runeSet(b)
	case "alias-particle.txt":
		p.AliasParticle = runeSet(b)
	default:
		return fmt.Errorf("unknown source file")
	}
	return nil
}

func (p *Pack) assignPair(name string, b []byte) error {
	switch name {
	case "palladius.txt":
		ini, fin, yw, si, err := parsePalladius(b)
		if err != nil {
			return err
		}
		p.PalladiusInitials, p.PalladiusFinals, p.PalladiusYW, p.PalladiusSpecialI = ini, fin, yw, si
	default:
		return fmt.Errorf("unknown pair file")
	}
	return nil
}

// runeSet reads a rune SET: every non-whitespace rune of every non-comment line is a member (order-free).
func runeSet(b []byte) map[rune]bool {
	m := map[rune]bool{}
	for _, ln := range contentLines(b) {
		for _, r := range ln {
			if !isSpace(r) {
				m[r] = true
			}
		}
	}
	return m
}

// stringSet reads a set of whole tokens, one per non-comment line (trimmed).
func stringSet(b []byte) map[string]bool {
	m := map[string]bool{}
	for _, ln := range contentLines(b) {
		if t := strings.TrimSpace(ln); t != "" {
			m[t] = true
		}
	}
	return m
}

// lines reads an ORDERED slice of whole tokens, one per non-comment line (trimmed), in file order.
func lines(b []byte) []string {
	var out []string
	for _, ln := range contentLines(b) {
		if t := strings.TrimSpace(ln); t != "" {
			out = append(out, t)
		}
	}
	return out
}

// parsePalladius reads the `category<TAB>pinyin<TAB>cyrillic` table into the four maps. It scans the raw
// lines directly (not contentLines) so an error names the PHYSICAL file line — the point of the diagnostic
// is to send a human editing the table to the right line.
func parsePalladius(b []byte) (ini, fin, yw, si map[string]string, err error) {
	ini, fin, yw, si = map[string]string{}, map[string]string{}, map[string]string{}, map[string]string{}
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
		if t == "" || strings.HasPrefix(t, "#") {
			continue
		}
		f := strings.Split(t, "\t")
		if len(f) != 3 {
			return nil, nil, nil, nil, fmt.Errorf("line %d: want 3 tab-separated fields, got %d (%q)", i+1, len(f), t)
		}
		switch f[0] {
		case "initials":
			ini[f[1]] = f[2]
		case "finals":
			fin[f[1]] = f[2]
		case "yw":
			yw[f[1]] = f[2]
		case "special_i":
			si[f[1]] = f[2]
		default:
			return nil, nil, nil, nil, fmt.Errorf("line %d: unknown category %q", i+1, f[0])
		}
	}
	return ini, fin, yw, si, nil
}

// contentLines splits into lines, dropping '#'-comment and blank lines.
func contentLines(b []byte) []string {
	var out []string
	for _, ln := range strings.Split(string(b), "\n") {
		s := strings.TrimRight(ln, "\r")
		if strings.HasPrefix(strings.TrimSpace(s), "#") || strings.TrimSpace(s) == "" {
			continue
		}
		out = append(out, s)
	}
	return out
}

func isSpace(r rune) bool { return r == ' ' || r == '\t' || r == '\n' || r == '\r' }