textmachine/backend/internal/config/book.go

package config

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"gopkg.in/yaml.v3"
)

// book.go: translation brief — the book project config (Phase 0 of the plan: language
// pair, genre, audience, 18+, Venuti slider, honorifics, transcription,
// footnotes). brief_hash is part of the TM key and of the snapshot: changing the brief mid-
// book is an explicit command that shows the cost of re-translation (Р6).

// Book is the parsed book.yaml.
type Book struct {
	BookID     string `yaml:"book_id"`
	Title      string `yaml:"title"`
	SourceLang string `yaml:"source_lang"` // zh | ja | en
	TargetLang string `yaml:"target_lang"` // ru
	Genre      string `yaml:"genre"`
	Audience   string `yaml:"audience"`
	// ContentLabels are the book's declared content properties — DATA, not engine knowledge (D39.25):
	// the engine only compares this set against a model's accepts_labels (models.yaml) and against the
	// run config's content_policy registry. No label VALUE is known to Go, so a second label
	// (archaic-register …) is three data edits and zero code. Owner-declared in v1 (there is no
	// automatic screen — the L3 entry policy stays human, D22.7), lower-cased/sorted/deduped on load so
	// a cosmetic reorder is not a different book. Deliberately NOT part of BriefHash: the labels'
	// entire wire effect is the RESOLVED per-stage model, which is folded per-stage and per-wave
	// (D39.26 point 1) — folding the set book-globally would re-pay a paid draft wave when only the
	// editor is re-routed, against the per-stage granularity ratified in D20.2-Q1.
	ContentLabels []string `yaml:"content_labels"`
	// LegacyContentFlag is the RETIRED `adult:` key, declared for exactly two reasons (the
	// Stage.LegacyPrompt discipline): to REJECT `adult: true` with the migration named, and to keep
	// the brief canon's byte layout frozen (see BriefHash). `adult: false` / an absent key stay
	// TOLERATED on purpose — 13 book-shaped files outside git carry the line explicitly, and failing
	// their load would buy nothing (the value false says nothing the empty label set does not).
	LegacyContentFlag bool `yaml:"adult"`
	// Venuti is the foreignization↔domestication slider, 0.0 (domestication)
	// … 1.0 (foreignization).
	Venuti        float64 `yaml:"venuti"`
	Honorifics    string  `yaml:"honorifics"`    // keep | adapt
	Transcription string  `yaml:"transcription"` // e.g. polivanov | palladius
	Footnotes     string  `yaml:"footnotes"`     // none | minimal | rich
	// YoPolicy is the brief's ё-policy for the cheap yofikator gate: auto (flag only inconsistency
	// — the same word both with ё and е) | all-yo | all-e. "" defaults to auto. A brief field: it
	// changes the style-flag verdict, so it is folded into BriefHash (a change re-pins the snapshot).
	YoPolicy string `yaml:"yo_policy"`
	// StyleAllowlist exempts surfaces from the translit-interjection blocklist (a per-project
	// allowlist, 04-unhappy §6): a character actually named «Ара» is not an untranslated filler.
	// Lower-cased on load. Verdict-affecting → also folded into BriefHash.
	StyleAllowlist []string `yaml:"style_allowlist"`
	// RegisterBlocklist is the DC6 out-of-register target lexis for THIS book (D39.79 Q4): fairy-tale /
	// chancery Russian words wrong for the book's genre («терем»…). It lives with the BOOK, not the pair
	// pack, because it is a genre/book property — a different zh→ru book of another genre ships its own (or
	// none). Lower-cased on load; verdict-affecting → folded into BriefHash (omitempty keeps a book without
	// one byte-identical). Empty = the DC6 register checker runs inert («inert without data»).
	RegisterBlocklist []string `yaml:"register_blocklist"`

	// Wiring: paths are resolved relative to the book.yaml location.
	Pipeline   string `yaml:"pipeline"`
	ModelsFile string `yaml:"models"`
	SourceFile string `yaml:"source_file"` // Phase 0: one file = one chunk
	// Encoding of the txt source: auto|utf8|gb18030 ("" defaults to auto). Real zh .txt are often
	// GB18030 (the 蛊真人 acceptance book is), which the auto-detect handles; declare it explicitly
	// to skip detection. Ignored for epub (its documents carry their own charset). See ingest.go.
	Encoding  string `yaml:"encoding"`
	ProjectDB string `yaml:"project_db"` // default: <book_id>.db next to book.yaml
	// GlossarySeed is the optional path to the manual glossary seed YAML (memory v2,
	// step 4). Its curated approved/draft terms + the classified ruby readings are the
	// deterministic inputs the glossary is REPLACED from each run; editing it changes the
	// materialized approved set → a loud --resnapshot (F1). Empty = ruby-seed only (or an
	// empty glossary → the injection is inert, a safe no-op).
	GlossarySeed string `yaml:"glossary_seed"`
	// LangpackRoot is the optional root of the language-data packs (configs/langpacks/, D39.15/16): the
	// miner reads configs/langpacks/<src>/ (source morphology) + configs/langpacks/<src>-<tgt>/ (Palladius)
	// from here. Resolved relative to book.yaml. Empty ⇒ no pack (the miner is inert / W1.5 auto-continues);
	// set ⇒ the runner loads the pack in W0, failing LOUD when the pair's catalog dir exists but a file is
	// missing/corrupt, and folds pack.Version() into the snapshot so a pack edit is a loud --resnapshot (R1).
	LangpackRoot string `yaml:"langpack_root"`
	// LangpackExtend is the optional root of a BOOK-SCOPED langpack overlay (pair-14 §1): a book's PRIVATE
	// canon — a clan surname (古月 in 蛊真人 is a book clan, not a 百家姓 family), a sect term — that must reach
	// the miner FOR THIS BOOK without polluting the shared pair langpack. Same layout as langpack_root; only
	// the files a book ships are present, each UNIONED onto the shared pack (additive) with its bytes folded
	// into pack.Version() (a book-canon edit is a loud --resnapshot for this book only). Resolved relative to
	// book.yaml. Empty ⇒ no overlay (the shared pack is used as-is).
	LangpackExtend string `yaml:"langpack_extend"`
	// MinedDelta is the path of the owner-curated mined-term delta YAML (the sign artifact, R1). Its
	// terms are loaded as Source:"mined" (NOT Source:"seed"), so they fold into the ENRICHED bank version
	// but NOT the base — adding them moves only the edit-wave snapshot (the pay-once invariant). Same
	// seedTerm schema as glossary_seed.
	//
	// DERIVED, never read from the file: `<book_id><MinedDeltaSuffix>` beside book.yaml, a pure function
	// of the config's location (D39.156 п.3). The path used to be declarable, and that is what made the
	// cross-book race reachable: two books pointing their keys at ONE file were not mutually excluded by
	// the per-project lock, and a demonstrated race lost one side's approvals with exit 0. A convention
	// has no second spelling, so the race is unreachable by construction. An absent file is «nobody has
	// decided anything yet», the same standing an absent project database has before the first run.
	MinedDelta string `yaml:"-"`
	// MinedRejects is the path of the owner's mined-term REJECT list YAML (R1-FL-B). It lists the src
	// surfaces the owner reviewed and DECLINED, so the emission stops re-proposing them. Unlike
	// MinedDelta it is a PROPOSAL filter only: rejects NEVER enter the bank content, so they are
	// deliberately NOT folded into the snapshot. Derived by the same convention as MinedDelta.
	MinedRejects string `yaml:"-"`
	// RetiredMinedDelta / RetiredMinedRejects catch the two RETIRED path keys, the `adult:` discipline:
	// a config written for the declarable era must fail naming the cure, not decode into a book whose
	// decisions silently land on the conventional path instead of the declared one.
	RetiredMinedDelta   string  `yaml:"mined_delta"`
	RetiredMinedRejects string  `yaml:"mined_rejects"`
	Ceilings            BookCap `yaml:"ceilings"`
	// RebillConsentUSD overrides the ratified default threshold above which a run must be given
	// explicit consent (`--accept-rebill[=usd]`) before it re-pays for work already billed —
	// `min($0.50, 5% × ProjectedBookUSD)`, D20.2-Q2. 0 (absent) = the default formula. WIRING, not
	// brief: it is deliberately absent from BriefHash and from the snapshot, so setting it never
	// re-pays anything by itself.
	RebillConsentUSD float64 `yaml:"rebill_consent_usd"`

	// Dir is the directory book.yaml was loaded from — the BOOK's directory, which is also where the
	// run-event journal lives (D39.106 §2: `events.jsonl` in the book's directory, which is the
	// platform's workdir for this run). Derived, never read from the file: `yaml:"-"` keeps the strict
	// decoder from ever accepting a `dir:` key that would let a config claim a directory it is not in.
	// It is NOT ProjectDB's directory — `project_db` may point anywhere, and the seam is anchored to
	// the config the platform passed, not to a path inside it.
	Dir string `yaml:"-"`
}

// MinedDeltaSuffix / MinedRejectsSuffix name the two owner-decision files the engine defaults into the
// book's directory when book.yaml declares no path for them. They are CONSTANTS and not a format string
// because `tmctl bank-apply` writes exactly these files and the platform must never derive either name —
// the engine owns the paths of its own schema (17-seam-inbound-law п.1).
const (
	MinedDeltaSuffix   = ".mined-delta.yaml"
	MinedRejectsSuffix = ".mined-rejects.yaml"
)

// BookCap are the ledger admission limits (Р7: the $ ceiling per book/day).
type BookCap struct {
	BookUSD float64 `yaml:"book_usd"`
	DayUSD  float64 `yaml:"day_usd"`
}

// LoadBook reads and validates book.yaml, resolving relative paths against
// its directory.
func LoadBook(path string) (*Book, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("config: read %s: %w", path, err)
	}
	var b Book
	// STRICT decode (KnownFields), sanctioned for pack-16 after a stand audit found ZERO unknown keys in any
	// book-shaped yaml in the repo or on the stand: yaml.v3 silently DROPS an unknown field, so a typo reads
	// as "not set" and the run looks normal — `langpack_extend` misspelled means the book's private canon is
	// quietly absent, `glossary_seed` misspelled means the seed never loads. The pipeline loader has been
	// strict since pack-15; these are the remaining halves of the same silent-substitution class.
	dec := yaml.NewDecoder(bytes.NewReader(raw))
	dec.KnownFields(true)
	if err := dec.Decode(&b); err != nil {
		return nil, fmt.Errorf("config: parse %s: %w", path, err)
	}
	dir := filepath.Dir(path)
	b.Dir = dir
	resolve := func(p string) string {
		if p == "" {
			return p
		}
		if filepath.IsAbs(p) {
			// Clean, not verbatim: the pairwise same-path guard below compares STRINGS, and an uncleaned
			// absolute path (`<dir>/./shared.yaml`) is a second spelling of a guarded path — the guard was
			// bypassed with exactly that and the original data-loss blocker reproduced whole. Join cleans
			// on its own; this keeps both branches on one rule.
			return filepath.Clean(p)
		}
		return filepath.Join(dir, p)
	}
	b.Pipeline = resolve(b.Pipeline)
	b.ModelsFile = resolve(b.ModelsFile)
	b.SourceFile = resolve(b.SourceFile)
	b.GlossarySeed = resolve(b.GlossarySeed)
	b.LangpackRoot = resolve(b.LangpackRoot)
	b.LangpackExtend = resolve(b.LangpackExtend)
	if b.ProjectDB == "" {
		b.ProjectDB = filepath.Join(dir, b.BookID+".db")
	} else {
		b.ProjectDB = resolve(b.ProjectDB)
	}
	// The two decision files live by the SAME convention project_db defaults to — the book's own
	// directory, named after book_id so two books sharing a directory cannot collide. Deliberately NOT
	// hung off project_db like the engine's read-out sidecars (.bank.json, .auto-bank.yaml): `project_db`
	// may point anywhere, and these two are the USER's decisions, which have to travel with the book in a
	// backup or an export of its directory. book_id is validated below, so an empty one can only reach
	// here on a config that is already being refused.
	b.MinedDelta = filepath.Join(dir, b.BookID+MinedDeltaSuffix)
	b.MinedRejects = filepath.Join(dir, b.BookID+MinedRejectsSuffix)

	var problems []string
	bad := func(format string, a ...any) { problems = append(problems, fmt.Sprintf(format, a...)) }
	// The engine's four owned files must be four DIFFERENT files. The decision paths cannot collide with
	// each other any more (both are derived, and the suffixes differ), but the two DECLARABLE keys can
	// still be pointed at them: `project_db` on the conventional delta path means the decisions door
	// writes YAML over the book's sqlite store, `glossary_seed` on it means the seed and the delta are
	// one file read under two schemas.
	//
	// Compared as the engine RESOLVED them (Cleaned absolute, or joined to the book's directory), which
	// is what makes `./x.yaml`, `x.yaml` and `<dir>/./x.yaml` one answer — the acceptance walked through
	// the uncleaned spelling and reproduced the original data-loss blocker whole.
	owned := []struct{ key, path string }{
		{"project_db", b.ProjectDB}, {"glossary_seed", b.GlossarySeed},
		{"mined_delta", b.MinedDelta}, {"mined_rejects", b.MinedRejects},
	}
	for i := range owned {
		for j := 0; j < i; j++ {
			if owned[i].path == "" || owned[i].path != owned[j].path {
				continue
			}
			bad("%s and %s both name %s — the engine reads and writes these as different documents, and one would silently overwrite the other",
				owned[j].key, owned[i].key, owned[i].path)
		}
	}
	if b.BookID == "" {
		bad("book_id is required")
	} else if strings.ContainsAny(b.BookID, `/\`) || b.BookID == "." || b.BookID == ".." {
		// book_id is a FILE NAME, not a label: four of the engine's files are named after it — the project
		// database and the three sidecars beside it, two of which now carry the OWNER's decisions. A value
		// with a separator in it walks that whole set out of the book's directory, and `../x` puts a user's
		// decisions somewhere nobody backs up with the book. One path element, and not the two that mean a
		// directory.
		bad("book_id %q is used as a FILE NAME (%s.db and the decision files beside it), so it has to be a single path element: no «/» or «\\», and not «.» or «..»", b.BookID, b.BookID)
	}
	if b.SourceLang == "" || b.TargetLang == "" {
		bad("source_lang and target_lang are required")
	}
	if b.Venuti < 0 || b.Venuti > 1 {
		bad("venuti must be in [0,1], got %v", b.Venuti)
	}
	if b.Pipeline == "" {
		bad("pipeline config path is required")
	}
	if b.ModelsFile == "" {
		bad("models config path is required")
	}
	if b.SourceFile == "" {
		bad("source_file is required")
	} else if _, err := os.Stat(b.SourceFile); err != nil {
		// Deliberately NOT a verdict about the TEXT: a path that is missing, a permission or an I/O error
		// is a fact about the configuration or the host, and the class that blames the user's text is the
		// one an automated caller acts on by deleting their upload (PD-196, pipeline/refusal.go).
		bad("source_file %s is not readable: %v", b.SourceFile, err)
	}
	if b.GlossarySeed != "" {
		if _, err := os.Stat(b.GlossarySeed); err != nil {
			bad("glossary_seed %s is not readable: %v", b.GlossarySeed, err)
		}
	}
	// The retired declarable era of the two decision paths (see RetiredMinedDelta). An empty declared
	// value is tolerated like `adult: false`: it asked for the convention, which is what it gets.
	if b.RetiredMinedDelta != "" || b.RetiredMinedRejects != "" {
		bad("`mined_delta:`/`mined_rejects:` are retired — the engine owns these paths by convention (%s%s and %s%s beside book.yaml, D39.156 п.3), and a declarable path is what made the cross-book write race reachable; delete the key(s)",
			b.BookID, MinedDeltaSuffix, b.BookID, MinedRejectsSuffix)
	}
	if b.Encoding == "" {
		b.Encoding = "auto"
	}
	switch strings.ToLower(b.Encoding) {
	case "auto", "utf8", "utf-8", "gb18030", "gbk", "gb2312":
	default:
		bad("encoding %q is not supported (use auto|utf8|gb18030)", b.Encoding)
	}
	// The retired `adult:` key: `true` is a MIGRATION error naming the replacement, never a silently
	// honoured routing request (it used to be the whole 18+ mechanism, so a stale `true` would read as
	// "this book is routed" while nothing routes it). `false`/absent is tolerated (see the field).
	if b.LegacyContentFlag {
		bad("`adult: true` is retired — declare the book's content properties as data instead: `content_labels: [<label>]`, with a matching `content_policy` entry in the run config and `accepts_labels` on the providers allowed to receive it (D39.25/D39.26). `adult: false` may stay or be dropped")
	}
	// Content labels are a SET: trimmed, lower-cased, deduped and sorted, so a cosmetic reorder or a
	// case difference is the same book (the StyleAllowlist discipline below). An empty entry is a typo,
	// not "no label" — the empty list already says that.
	labels := make([]string, 0, len(b.ContentLabels))
	seenLabel := map[string]bool{}
	for i, l := range b.ContentLabels {
		l = strings.ToLower(strings.TrimSpace(l))
		if l == "" {
			bad("content_labels[%d] is empty — drop the entry (an empty list means \"no labels\")", i)
			continue
		}
		if seenLabel[l] {
			continue
		}
		seenLabel[l] = true
		labels = append(labels, l)
	}
	sort.Strings(labels)
	b.ContentLabels = labels
	if b.YoPolicy == "" {
		b.YoPolicy = "auto"
	}
	switch b.YoPolicy {
	case "auto", "all-yo", "all-e":
	default:
		bad("yo_policy %q is not supported (use auto|all-yo|all-e)", b.YoPolicy)
	}
	for i, s := range b.StyleAllowlist {
		b.StyleAllowlist[i] = strings.ToLower(strings.TrimSpace(s))
	}
	// Canonicalize (sort) the allowlist so a cosmetic reorder in book.yaml does not change BriefHash
	// → the snapshot → force an unnecessary re-pin (self-review: it is a set, consumed order-
	// independently in cheapGateConfig).
	sort.Strings(b.StyleAllowlist)
	// Same discipline for the register blocklist (D39.79 Q4): lower-fold + sort so a reorder is BriefHash-neutral.
	for i, s := range b.RegisterBlocklist {
		b.RegisterBlocklist[i] = strings.ToLower(strings.TrimSpace(s))
	}
	sort.Strings(b.RegisterBlocklist)
	if b.Ceilings.BookUSD <= 0 && b.Ceilings.DayUSD <= 0 {
		bad("ceilings: at least one of book_usd/day_usd must be set (a ledger with no ceiling is forbidden, Р7)")
	}
	if b.RebillConsentUSD < 0 {
		bad("rebill_consent_usd must be ≥ 0 (0 = the ratified default min($0.50, 5%%×projected book cost)), got %v", b.RebillConsentUSD)
	}
	if len(problems) > 0 {
		return nil, fmt.Errorf("config %s:\n  - %s", path, strings.Join(problems, "\n  - "))
	}
	return &b, nil
}

// BriefHash is the canonical hash of the SEMANTIC brief fields — the ones
// whose change legitimately invalidates the TM (Р6: brief_hash is part of the
// TM key). Wiring fields (paths, ceilings, db) deliberately excluded: moving
// the project into another directory must not re-translate the book. JSON with
// a fixed field order → a deterministic byte render.
func (b *Book) BriefHash() string {
	// Title is part of the brief: it is a semantic field (used by prompts via
	// {{title}} and lands in the render → request-hash), not a wiring identifier.
	// Without it, editing the title would silently invalidate checkpoints, bypassing
	// the snapshot gate (review finding).
	// FROZEN LAYOUT (D39.26 point 1). The 7th pair is the retired `adult:` key's slot, rendered as a
	// CONSTANT false. It is dead as a semantic and alive as bytes: dropping it (or replacing it with a
	// content_labels field) moves brief_hash → both wave snapshotIDs → every RequestHash → the whole
	// book is re-billed on the next resume, including the stand books the pack-16 remnant measurement
	// rides ($0 draft resume, D39.20). Verified by derivation: today c2021b1b…cab0df (= the golden's
	// recorded brief_hash), without the slot 58883ff5…b39822. Content labels are deliberately absent
	// from this canon — their wire fate is the resolved per-stage model (stageSnap), not the brief.
	const retiredContentSlot = false
	canon := struct {
		BookID         string   `json:"book_id"`
		Title          string   `json:"title"`
		SourceLang     string   `json:"source_lang"`
		TargetLang     string   `json:"target_lang"`
		Genre          string   `json:"genre"`
		Audience       string   `json:"audience"`
		RetiredSlot    bool     `json:"adult"`
		Venuti         float64  `json:"venuti"`
		Honorifics     string   `json:"honorifics"`
		Transcription  string   `json:"transcription"`
		Footnotes      string   `json:"footnotes"`
		YoPolicy       string   `json:"yo_policy"`
		StyleAllowlist []string `json:"style_allowlist"`
		// RegisterBlocklist is verdict-affecting (DC6), so a change must re-pin. omitempty keeps a book WITHOUT
		// one (the golden fixture, every pre-D39.79 book) byte-identical → the FROZEN LAYOUT above is preserved.
		RegisterBlocklist []string `json:"register_blocklist,omitempty"`
	}{b.BookID, b.Title, b.SourceLang, b.TargetLang, b.Genre, b.Audience, retiredContentSlot, b.Venuti, b.Honorifics, b.Transcription, b.Footnotes, b.YoPolicy, b.StyleAllowlist, b.RegisterBlocklist}
	data, err := json.Marshal(canon)
	if err != nil {
		// A struct of scalars cannot fail to marshal; keep the signature clean.
		panic(fmt.Sprintf("config: brief hash marshal: %v", err))
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:])
}

// LangPair returns "ja-ru"-style pair key (the coverage-gate threshold keys).
func (b *Book) LangPair() string {
	return b.SourceLang + "-" + b.TargetLang
}