322 lines
17 KiB
Go
322 lines
17 KiB
Go
package config
|
||
|
||
import (
|
||
"bytes"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
|
||
"gopkg.in/yaml.v3"
|
||
)
|
||
|
||
// book.go: translation brief — the book project config (Phase 0 of the plan: language
|
||
// pair, genre, audience, 18+, Venuti slider, honorifics, transcription,
|
||
// footnotes). brief_hash is part of the TM key and of the snapshot: changing the brief mid-
|
||
// book is an explicit command that shows the cost of re-translation (Р6).
|
||
|
||
// Book is the parsed book.yaml.
|
||
type Book struct {
|
||
BookID string `yaml:"book_id"`
|
||
Title string `yaml:"title"`
|
||
SourceLang string `yaml:"source_lang"` // zh | ja | en
|
||
TargetLang string `yaml:"target_lang"` // ru
|
||
Genre string `yaml:"genre"`
|
||
Audience string `yaml:"audience"`
|
||
// ContentLabels are the book's declared content properties — DATA, not engine knowledge (D39.25):
|
||
// the engine only compares this set against a model's accepts_labels (models.yaml) and against the
|
||
// run config's content_policy registry. No label VALUE is known to Go, so a second label
|
||
// (archaic-register …) is three data edits and zero code. Owner-declared in v1 (there is no
|
||
// automatic screen — the L3 entry policy stays human, D22.7), lower-cased/sorted/deduped on load so
|
||
// a cosmetic reorder is not a different book. Deliberately NOT part of BriefHash: the labels'
|
||
// entire wire effect is the RESOLVED per-stage model, which is folded per-stage and per-wave
|
||
// (D39.26 point 1) — folding the set book-globally would re-pay a paid draft wave when only the
|
||
// editor is re-routed, against the per-stage granularity ratified in D20.2-Q1.
|
||
ContentLabels []string `yaml:"content_labels"`
|
||
// LegacyContentFlag is the RETIRED `adult:` key, declared for exactly two reasons (the
|
||
// Stage.LegacyPrompt discipline): to REJECT `adult: true` with the migration named, and to keep
|
||
// the brief canon's byte layout frozen (see BriefHash). `adult: false` / an absent key stay
|
||
// TOLERATED on purpose — 13 book-shaped files outside git carry the line explicitly, and failing
|
||
// their load would buy nothing (the value false says nothing the empty label set does not).
|
||
LegacyContentFlag bool `yaml:"adult"`
|
||
// Venuti is the foreignization↔domestication slider, 0.0 (domestication)
|
||
// … 1.0 (foreignization).
|
||
Venuti float64 `yaml:"venuti"`
|
||
Honorifics string `yaml:"honorifics"` // keep | adapt
|
||
Transcription string `yaml:"transcription"` // e.g. polivanov | palladius
|
||
Footnotes string `yaml:"footnotes"` // none | minimal | rich
|
||
// YoPolicy is the brief's ё-policy for the cheap yofikator gate: auto (flag only inconsistency
|
||
// — the same word both with ё and е) | all-yo | all-e. "" defaults to auto. A brief field: it
|
||
// changes the style-flag verdict, so it is folded into BriefHash (a change re-pins the snapshot).
|
||
YoPolicy string `yaml:"yo_policy"`
|
||
// StyleAllowlist exempts surfaces from the translit-interjection blocklist (a per-project
|
||
// allowlist, 04-unhappy §6): a character actually named «Ара» is not an untranslated filler.
|
||
// Lower-cased on load. Verdict-affecting → also folded into BriefHash.
|
||
StyleAllowlist []string `yaml:"style_allowlist"`
|
||
// RegisterBlocklist is the DC6 out-of-register target lexis for THIS book (D39.79 Q4): fairy-tale /
|
||
// chancery Russian words wrong for the book's genre («терем»…). It lives with the BOOK, not the pair
|
||
// pack, because it is a genre/book property — a different zh→ru book of another genre ships its own (or
|
||
// none). Lower-cased on load; verdict-affecting → folded into BriefHash (omitempty keeps a book without
|
||
// one byte-identical). Empty = the DC6 register checker runs inert («inert without data»).
|
||
RegisterBlocklist []string `yaml:"register_blocklist"`
|
||
|
||
// Wiring: paths are resolved relative to the book.yaml location.
|
||
Pipeline string `yaml:"pipeline"`
|
||
ModelsFile string `yaml:"models"`
|
||
SourceFile string `yaml:"source_file"` // Phase 0: one file = one chunk
|
||
// Encoding of the txt source: auto|utf8|gb18030 ("" defaults to auto). Real zh .txt are often
|
||
// GB18030 (the 蛊真人 acceptance book is), which the auto-detect handles; declare it explicitly
|
||
// to skip detection. Ignored for epub (its documents carry their own charset). See ingest.go.
|
||
Encoding string `yaml:"encoding"`
|
||
ProjectDB string `yaml:"project_db"` // default: <book_id>.db next to book.yaml
|
||
// GlossarySeed is the optional path to the manual glossary seed YAML (memory v2,
|
||
// step 4). Its curated approved/draft terms + the classified ruby readings are the
|
||
// deterministic inputs the glossary is REPLACED from each run; editing it changes the
|
||
// materialized approved set → a loud --resnapshot (F1). Empty = ruby-seed only (or an
|
||
// empty glossary → the injection is inert, a safe no-op).
|
||
GlossarySeed string `yaml:"glossary_seed"`
|
||
// LangpackRoot is the optional root of the language-data packs (configs/langpacks/, D39.15/16): the
|
||
// miner reads configs/langpacks/<src>/ (source morphology) + configs/langpacks/<src>-<tgt>/ (Palladius)
|
||
// from here. Resolved relative to book.yaml. Empty ⇒ no pack (the miner is inert / W1.5 auto-continues);
|
||
// set ⇒ the runner loads the pack in W0, failing LOUD when the pair's catalog dir exists but a file is
|
||
// missing/corrupt, and folds pack.Version() into the snapshot so a pack edit is a loud --resnapshot (R1).
|
||
LangpackRoot string `yaml:"langpack_root"`
|
||
// LangpackExtend is the optional root of a BOOK-SCOPED langpack overlay (pair-14 §1): a book's PRIVATE
|
||
// canon — a clan surname (古月 in 蛊真人 is a book clan, not a 百家姓 family), a sect term — that must reach
|
||
// the miner FOR THIS BOOK without polluting the shared pair langpack. Same layout as langpack_root; only
|
||
// the files a book ships are present, each UNIONED onto the shared pack (additive) with its bytes folded
|
||
// into pack.Version() (a book-canon edit is a loud --resnapshot for this book only). Resolved relative to
|
||
// book.yaml. Empty ⇒ no overlay (the shared pack is used as-is).
|
||
LangpackExtend string `yaml:"langpack_extend"`
|
||
// MinedDelta is the optional path to the owner-curated mined-term delta YAML (the W1.5 sign artifact,
|
||
// R1). Its terms are loaded as Source:"mined" (NOT Source:"seed"), so they fold into the ENRICHED bank
|
||
// version but NOT the base — adding them moves only snapshot_W2 (the pay-once invariant). Same seedTerm
|
||
// schema as glossary_seed; resolved relative to book.yaml. Empty = no mined terms.
|
||
MinedDelta string `yaml:"mined_delta"`
|
||
// MinedRejects is the optional path to the owner's mined-term REJECT list YAML (R1-FL-B). It lists the
|
||
// src surfaces the owner reviewed and DECLINED, so the bank-mining stop stops re-proposing them every
|
||
// run (else a declined term re-fires the stop forever — a livelock). Unlike MinedDelta it is a PROPOSAL
|
||
// filter only: rejects NEVER enter the bank content, so they are deliberately NOT folded into the
|
||
// snapshot. Resolved relative to book.yaml. Empty = no rejects.
|
||
MinedRejects string `yaml:"mined_rejects"`
|
||
Ceilings BookCap `yaml:"ceilings"`
|
||
// RebillConsentUSD overrides the ratified default threshold above which a run must be given
|
||
// explicit consent (`--accept-rebill[=usd]`) before it re-pays for work already billed —
|
||
// `min($0.50, 5% × ProjectedBookUSD)`, D20.2-Q2. 0 (absent) = the default formula. WIRING, not
|
||
// brief: it is deliberately absent from BriefHash and from the snapshot, so setting it never
|
||
// re-pays anything by itself.
|
||
RebillConsentUSD float64 `yaml:"rebill_consent_usd"`
|
||
|
||
// Dir is the directory book.yaml was loaded from — the BOOK's directory, which is also where the
|
||
// run-event journal lives (D39.106 §2: `events.jsonl` in the book's directory, which is the
|
||
// platform's workdir for this run). Derived, never read from the file: `yaml:"-"` keeps the strict
|
||
// decoder from ever accepting a `dir:` key that would let a config claim a directory it is not in.
|
||
// It is NOT ProjectDB's directory — `project_db` may point anywhere, and the seam is anchored to
|
||
// the config the platform passed, not to a path inside it.
|
||
Dir string `yaml:"-"`
|
||
}
|
||
|
||
// BookCap are the ledger admission limits (Р7: the $ ceiling per book/day).
|
||
type BookCap struct {
|
||
BookUSD float64 `yaml:"book_usd"`
|
||
DayUSD float64 `yaml:"day_usd"`
|
||
}
|
||
|
||
// LoadBook reads and validates book.yaml, resolving relative paths against
|
||
// its directory.
|
||
func LoadBook(path string) (*Book, error) {
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("config: read %s: %w", path, err)
|
||
}
|
||
var b Book
|
||
// STRICT decode (KnownFields), sanctioned for pack-16 after a stand audit found ZERO unknown keys in any
|
||
// book-shaped yaml in the repo or on the stand: yaml.v3 silently DROPS an unknown field, so a typo reads
|
||
// as "not set" and the run looks normal — `langpack_extend` misspelled means the book's private canon is
|
||
// quietly absent, `glossary_seed` misspelled means the seed never loads. The pipeline loader has been
|
||
// strict since pack-15; these are the remaining halves of the same silent-substitution class.
|
||
dec := yaml.NewDecoder(bytes.NewReader(raw))
|
||
dec.KnownFields(true)
|
||
if err := dec.Decode(&b); err != nil {
|
||
return nil, fmt.Errorf("config: parse %s: %w", path, err)
|
||
}
|
||
dir := filepath.Dir(path)
|
||
b.Dir = dir
|
||
resolve := func(p string) string {
|
||
if p == "" || filepath.IsAbs(p) {
|
||
return p
|
||
}
|
||
return filepath.Join(dir, p)
|
||
}
|
||
b.Pipeline = resolve(b.Pipeline)
|
||
b.ModelsFile = resolve(b.ModelsFile)
|
||
b.SourceFile = resolve(b.SourceFile)
|
||
b.GlossarySeed = resolve(b.GlossarySeed)
|
||
b.LangpackRoot = resolve(b.LangpackRoot)
|
||
b.LangpackExtend = resolve(b.LangpackExtend)
|
||
b.MinedDelta = resolve(b.MinedDelta)
|
||
b.MinedRejects = resolve(b.MinedRejects)
|
||
if b.ProjectDB == "" {
|
||
b.ProjectDB = filepath.Join(dir, b.BookID+".db")
|
||
} else {
|
||
b.ProjectDB = resolve(b.ProjectDB)
|
||
}
|
||
|
||
var problems []string
|
||
bad := func(format string, a ...any) { problems = append(problems, fmt.Sprintf(format, a...)) }
|
||
if b.BookID == "" {
|
||
bad("book_id is required")
|
||
}
|
||
if b.SourceLang == "" || b.TargetLang == "" {
|
||
bad("source_lang and target_lang are required")
|
||
}
|
||
if b.Venuti < 0 || b.Venuti > 1 {
|
||
bad("venuti must be in [0,1], got %v", b.Venuti)
|
||
}
|
||
if b.Pipeline == "" {
|
||
bad("pipeline config path is required")
|
||
}
|
||
if b.ModelsFile == "" {
|
||
bad("models config path is required")
|
||
}
|
||
if b.SourceFile == "" {
|
||
bad("source_file is required")
|
||
} else if _, err := os.Stat(b.SourceFile); err != nil {
|
||
// Deliberately NOT a verdict about the TEXT: a path that is missing, a permission or an I/O error
|
||
// is a fact about the configuration or the host, and the class that blames the user's text is the
|
||
// one an automated caller acts on by deleting their upload (PD-196, pipeline/refusal.go).
|
||
bad("source_file %s is not readable: %v", b.SourceFile, err)
|
||
}
|
||
if b.GlossarySeed != "" {
|
||
if _, err := os.Stat(b.GlossarySeed); err != nil {
|
||
bad("glossary_seed %s is not readable: %v", b.GlossarySeed, err)
|
||
}
|
||
}
|
||
if b.MinedDelta != "" {
|
||
if _, err := os.Stat(b.MinedDelta); err != nil {
|
||
bad("mined_delta %s is not readable: %v", b.MinedDelta, err)
|
||
}
|
||
}
|
||
if b.MinedRejects != "" {
|
||
if _, err := os.Stat(b.MinedRejects); err != nil {
|
||
bad("mined_rejects %s is not readable: %v", b.MinedRejects, err)
|
||
}
|
||
}
|
||
if b.Encoding == "" {
|
||
b.Encoding = "auto"
|
||
}
|
||
switch strings.ToLower(b.Encoding) {
|
||
case "auto", "utf8", "utf-8", "gb18030", "gbk", "gb2312":
|
||
default:
|
||
bad("encoding %q is not supported (use auto|utf8|gb18030)", b.Encoding)
|
||
}
|
||
// The retired `adult:` key: `true` is a MIGRATION error naming the replacement, never a silently
|
||
// honoured routing request (it used to be the whole 18+ mechanism, so a stale `true` would read as
|
||
// "this book is routed" while nothing routes it). `false`/absent is tolerated (see the field).
|
||
if b.LegacyContentFlag {
|
||
bad("`adult: true` is retired — declare the book's content properties as data instead: `content_labels: [<label>]`, with a matching `content_policy` entry in the run config and `accepts_labels` on the providers allowed to receive it (D39.25/D39.26). `adult: false` may stay or be dropped")
|
||
}
|
||
// Content labels are a SET: trimmed, lower-cased, deduped and sorted, so a cosmetic reorder or a
|
||
// case difference is the same book (the StyleAllowlist discipline below). An empty entry is a typo,
|
||
// not "no label" — the empty list already says that.
|
||
labels := make([]string, 0, len(b.ContentLabels))
|
||
seenLabel := map[string]bool{}
|
||
for i, l := range b.ContentLabels {
|
||
l = strings.ToLower(strings.TrimSpace(l))
|
||
if l == "" {
|
||
bad("content_labels[%d] is empty — drop the entry (an empty list means \"no labels\")", i)
|
||
continue
|
||
}
|
||
if seenLabel[l] {
|
||
continue
|
||
}
|
||
seenLabel[l] = true
|
||
labels = append(labels, l)
|
||
}
|
||
sort.Strings(labels)
|
||
b.ContentLabels = labels
|
||
if b.YoPolicy == "" {
|
||
b.YoPolicy = "auto"
|
||
}
|
||
switch b.YoPolicy {
|
||
case "auto", "all-yo", "all-e":
|
||
default:
|
||
bad("yo_policy %q is not supported (use auto|all-yo|all-e)", b.YoPolicy)
|
||
}
|
||
for i, s := range b.StyleAllowlist {
|
||
b.StyleAllowlist[i] = strings.ToLower(strings.TrimSpace(s))
|
||
}
|
||
// Canonicalize (sort) the allowlist so a cosmetic reorder in book.yaml does not change BriefHash
|
||
// → the snapshot → force an unnecessary re-pin (self-review: it is a set, consumed order-
|
||
// independently in cheapGateConfig).
|
||
sort.Strings(b.StyleAllowlist)
|
||
// Same discipline for the register blocklist (D39.79 Q4): lower-fold + sort so a reorder is BriefHash-neutral.
|
||
for i, s := range b.RegisterBlocklist {
|
||
b.RegisterBlocklist[i] = strings.ToLower(strings.TrimSpace(s))
|
||
}
|
||
sort.Strings(b.RegisterBlocklist)
|
||
if b.Ceilings.BookUSD <= 0 && b.Ceilings.DayUSD <= 0 {
|
||
bad("ceilings: at least one of book_usd/day_usd must be set (a ledger with no ceiling is forbidden, Р7)")
|
||
}
|
||
if b.RebillConsentUSD < 0 {
|
||
bad("rebill_consent_usd must be ≥ 0 (0 = the ratified default min($0.50, 5%%×projected book cost)), got %v", b.RebillConsentUSD)
|
||
}
|
||
if len(problems) > 0 {
|
||
return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||
}
|
||
return &b, nil
|
||
}
|
||
|
||
// BriefHash is the canonical hash of the SEMANTIC brief fields — the ones
|
||
// whose change legitimately invalidates the TM (Р6: brief_hash is part of the
|
||
// TM key). Wiring fields (paths, ceilings, db) deliberately excluded: moving
|
||
// the project into another directory must not re-translate the book. JSON with
|
||
// a fixed field order → a deterministic byte render.
|
||
func (b *Book) BriefHash() string {
|
||
// Title is part of the brief: it is a semantic field (used by prompts via
|
||
// {{title}} and lands in the render → request-hash), not a wiring identifier.
|
||
// Without it, editing the title would silently invalidate checkpoints, bypassing
|
||
// the snapshot gate (review finding).
|
||
// FROZEN LAYOUT (D39.26 point 1). The 7th pair is the retired `adult:` key's slot, rendered as a
|
||
// CONSTANT false. It is dead as a semantic and alive as bytes: dropping it (or replacing it with a
|
||
// content_labels field) moves brief_hash → both wave snapshotIDs → every RequestHash → the whole
|
||
// book is re-billed on the next resume, including the stand books the pack-16 remnant measurement
|
||
// rides ($0 draft resume, D39.20). Verified by derivation: today c2021b1b…cab0df (= the golden's
|
||
// recorded brief_hash), without the slot 58883ff5…b39822. Content labels are deliberately absent
|
||
// from this canon — their wire fate is the resolved per-stage model (stageSnap), not the brief.
|
||
const retiredContentSlot = false
|
||
canon := struct {
|
||
BookID string `json:"book_id"`
|
||
Title string `json:"title"`
|
||
SourceLang string `json:"source_lang"`
|
||
TargetLang string `json:"target_lang"`
|
||
Genre string `json:"genre"`
|
||
Audience string `json:"audience"`
|
||
RetiredSlot bool `json:"adult"`
|
||
Venuti float64 `json:"venuti"`
|
||
Honorifics string `json:"honorifics"`
|
||
Transcription string `json:"transcription"`
|
||
Footnotes string `json:"footnotes"`
|
||
YoPolicy string `json:"yo_policy"`
|
||
StyleAllowlist []string `json:"style_allowlist"`
|
||
// RegisterBlocklist is verdict-affecting (DC6), so a change must re-pin. omitempty keeps a book WITHOUT
|
||
// one (the golden fixture, every pre-D39.79 book) byte-identical → the FROZEN LAYOUT above is preserved.
|
||
RegisterBlocklist []string `json:"register_blocklist,omitempty"`
|
||
}{b.BookID, b.Title, b.SourceLang, b.TargetLang, b.Genre, b.Audience, retiredContentSlot, b.Venuti, b.Honorifics, b.Transcription, b.Footnotes, b.YoPolicy, b.StyleAllowlist, b.RegisterBlocklist}
|
||
data, err := json.Marshal(canon)
|
||
if err != nil {
|
||
// A struct of scalars cannot fail to marshal; keep the signature clean.
|
||
panic(fmt.Sprintf("config: brief hash marshal: %v", err))
|
||
}
|
||
sum := sha256.Sum256(data)
|
||
return hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
// LangPair returns "ja-ru"-style pair key (the coverage-gate threshold keys).
|
||
func (b *Book) LangPair() string {
|
||
return b.SourceLang + "-" + b.TargetLang
|
||
}
|