textmachine/backend/internal/terminology/terminology.go

// Package terminology is the deterministic core of the TERMINOLOGIST role (D39.42 п.1): it merges the two
// bank-candidate channels into one list, gathers the source contexts each candidate needs to be translated
// at all, ranks the renderings the drafts already produced, and turns the model's reply back into
// consolidated terms.
//
// The split of duty is the point. The miner answers WHICH source surfaces belong in the bank (offline,
// source-side, high recall); the banknote answers WHAT some chunk's translator called them (draft-side,
// sparse — the two channels named overlapping entities in only 3% of the mini-run by construction). Neither
// answers "what should this book call it". That question needs the WHOLE book's contexts at once, which is
// exactly what exists at the wave boundary and nowhere else.
//
// Everything here is PURE and deterministic: no clock, no randomness, no map-order iteration, no I/O, no
// network. The pair data (Palladius conformance) arrives as values, so the package holds no pair-specific
// literal and works for a pair that is not in the repo yet.
package terminology

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"unicode"
)

// Origin records WHICH channel surfaced a candidate — the provenance the owner reads at sign time and the
// only thing that distinguishes "the detector found it in the source" from "a translator invented it".
type Origin string

const (
	OriginMined    Origin = "mined"    // the offline WHICH detector saw it in the source
	OriginBanknote Origin = "banknote" // only a draft's banknote proposed it (the reverse section)
	OriginBoth     Origin = "both"     // both channels named the same entity
	// OriginAlias: a draft named a surface the miner holds as an ALIAS of another cluster. It is kept as a
	// candidate of its own so the terminologist is asked to translate it, but it is NOT emitted into the
	// delta — whether the cluster is right is a clustering question, and that decision is not this pack's.
	OriginAlias Origin = "alias"
)

// Chunk is one normalized source chunk — the same view the miner takes (the candidate space and the KWIC
// space must be the same space, or a term is contextless exactly when it is hardest to translate).
type Chunk struct {
	Chapter  int
	ChunkIdx int
	NSource  string
}

// Mined is one candidate from the offline WHICH detector.
type Mined struct {
	Key      string // normalized source key (the join key)
	Src      string // the surface as the detector holds it
	Type     string // name|place|title|term
	Freq     int
	SinceCh  int
	Aliases  []string // normalized co-surfaces of the same entity
	Evidence []string
}

// Proposal is one draft-side rendering of a surface, already folded per (key, dst) by the caller.
type Proposal struct {
	Dst    string
	Type   string
	Chunks int // how many draft chunks proposed exactly this rendering
}

// Observed is the draft-side view of one source surface (the banknote channel, folded per key).
type Observed struct {
	Key       string
	Src       string
	Type      string
	Proposals []Proposal
}

// Neighbour is an already-APPROVED bank row — the anchor the §C2-3 "agreement with approved neighbours of
// the series" factor scores against (the 元-series precedent: a new 元-term rendered like its signed
// siblings is likelier right than one that is not).
type Neighbour struct {
	Src string
	Dst string
}

// Variant is one observed rendering with its consolidation score.
type Variant struct {
	Dst     string
	Chunks  int
	Score   float64
	Signals []string // which factors fired, in fixed order — the score's audit trail
	// Via names the surface that actually proposed this rendering when it is NOT the candidate's own —
	// i.e. an ALIAS of the cluster. Without it a mis-clustered alias silently re-labels its rendering: the
	// polygon's 葛家 cluster holding 族长 makes «глава клана» read as a rendering OF the family name, and the
	// terminologist would consolidate a surname to a title. Provenance, not a decision.
	Via string
}

// Candidate is one merged bank candidate: the union of what both channels know about one entity.
type Candidate struct {
	Key      string
	Src      string
	Type     string
	Freq     int // occurrences in the source; 0 when only the draft side saw it
	SinceCh  int
	Aliases  []string
	Evidence []string
	Origin   Origin
	Related  []string // mined keys this surface CONTAINS or is contained by (never merged — see Merge)
	Variants []Variant
	KWIC     []string
}

// Merge is the two-way $0 join of the WHICH and WHAT channels, with the alias/containment resolver that
// decides — deterministically — whether a draft-side surface IS a mined candidate or merely touches one.
//
//   - EXACT key match → MERGE. Same entity, so the draft's rendering is a rendering OF it.
//   - registered ALIAS of a mined entity → the rendering merges into the cluster (so the §C2-3 ranking sees
//     all of the entity's evidence) AND the surface ALSO stands as a candidate of its own, Origin `alias`,
//     pointing at the cluster. The miner's clustering can be wrong — measured on 50 chapters: a surname
//     cluster holding a clan title — and a plain merge is where such a term DISAPPEARS: absent from the
//     delta (the cluster represents it) and absent from the input (it folded away). The merged rendering
//     carries Via so a mis-clustered alias cannot silently re-label a surname to a title.
//   - CONTAINMENT (方源 inside 古月方源) → a candidate of its OWN, in the reverse section, with the mined key
//     recorded in Related. This is deliberately NOT a merge: «Гуюэ Фан Юань» is not a rendering of 方源, and
//     folding it in would put a wrong variant on a term the owner is about to sign. Containment is also
//     exactly where the miner's emission is silent — a cluster touching a seed surface is suppressed as an
//     alias-of-existing — so the reverse section is where the 3% coverage gap actually lives.
//   - otherwise → a candidate of its own, Origin banknote.
//
// Output order is by Key (never map order). Every mined candidate appears exactly once; every observed key
// either merges into one or appears once in the reverse section.
func Merge(mined []Mined, observed []Observed) []Candidate {
	byKey := make(map[string]*Candidate, len(mined))
	order := make([]string, 0, len(mined)+len(observed))
	// aliasOwner maps every alias surface to its owning mined key, so an aliased proposal merges.
	aliasOwner := map[string]string{}
	for _, m := range mined {
		if m.Key == "" || byKey[m.Key] != nil {
			continue
		}
		c := &Candidate{
			Key: m.Key, Src: m.Src, Type: m.Type, Freq: m.Freq, SinceCh: m.SinceCh,
			Aliases: append([]string(nil), m.Aliases...), Evidence: append([]string(nil), m.Evidence...),
			Origin: OriginMined,
		}
		byKey[m.Key] = c
		order = append(order, m.Key)
		for _, a := range m.Aliases {
			if a != "" && aliasOwner[a] == "" && byKey[a] == nil {
				aliasOwner[a] = m.Key
			}
		}
	}
	minedKeys := make([]string, 0, len(byKey))
	for k := range byKey {
		minedKeys = append(minedKeys, k)
	}
	sort.Strings(minedKeys) // deterministic containment scan

	for _, o := range observed {
		if o.Key == "" {
			continue
		}
		owner := o.Key
		if byKey[owner] == nil {
			owner = aliasOwner[o.Key]
		}
		if owner != "" && byKey[owner] != nil { // exact or alias → merge
			c := byKey[owner]
			if c.Origin == OriginMined {
				c.Origin = OriginBoth
			}
			vs := variantsOf(o.Proposals)
			if owner != o.Key { // merged through an ALIAS, not on its own key
				for i := range vs {
					vs[i].Via = o.Src
				}
			}
			c.Variants = append(c.Variants, vs...)
			if c.Type == "" {
				c.Type = o.Type
			}
			if owner != o.Key && byKey[o.Key] == nil {
				// DO NOT LOSE THE SURFACE (polygon package seven, G-series). The miner's alias cluster may be
				// wrong — measured: a surname cluster swallowing a clan title — and when it is, merging is the
				// only place the term can disappear: it is not in the miner's delta (the cluster represents it)
				// and it is not a candidate here (it folded into the owner). The terminologist then never sees
				// a term the drafts named seventeen times. So the surface ALSO stands on its own, carrying its
				// own renderings and a pointer to the cluster that claims it. Origin `alias` keeps it out of
				// the emitted delta: proposing it for signature would BE the clustering fix, which is a
				// separate decision.
				a := &Candidate{Key: o.Key, Src: o.Src, Type: o.Type, Origin: OriginAlias,
					Related: []string{owner}, Variants: variantsOf(o.Proposals)}
				byKey[a.Key] = a
				order = append(order, a.Key)
			}
			continue
		}
		c := &Candidate{
			Key: o.Key, Src: o.Src, Type: o.Type, Origin: OriginBanknote,
			Related: containmentRelations(o.Key, minedKeys), Variants: variantsOf(o.Proposals),
		}
		if byKey[c.Key] != nil {
			continue // a duplicate observed key (the caller folded per key, but never assume it)
		}
		byKey[c.Key] = c
		order = append(order, c.Key)
	}

	out := make([]Candidate, 0, len(order))
	for _, k := range order {
		c := *byKey[k]
		c.Variants = foldVariants(c.Variants)
		out = append(out, c)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
	return out
}

// foldVariants sums the chunk counts of IDENTICAL renderings. It matters on the alias path: 方源 and its
// alias 方小子 arrive as two Observed rows, and both may carry the same rendering — appended raw that is
// two variants of one word, which lies twice. Spread() would report 2 (the disagreement column the owner
// reads to decide whether a term is contested) for a term nobody disagreed about, and the §C2-3 frequency
// factor would score the consensus rendering on HALF its evidence, which can hand the win to a genuine
// competitor. Deterministic: first-seen order is preserved, so the fold introduces no new ordering.
func foldVariants(vs []Variant) []Variant {
	if len(vs) < 2 {
		return vs
	}
	at := make(map[string]int, len(vs))
	out := make([]Variant, 0, len(vs))
	for _, v := range vs {
		if i, seen := at[v.Dst]; seen {
			out[i].Chunks += v.Chunks
			continue
		}
		at[v.Dst] = len(out)
		out = append(out, v)
	}
	return out
}

// containmentRelations returns the mined keys that strictly contain, or are strictly contained by, key —
// sorted, so the record is deterministic. Bounded to the first few: the relation is context for a human
// and a model, not an index.
func containmentRelations(key string, minedKeys []string) []string {
	var out []string
	for _, mk := range minedKeys {
		if mk == key {
			continue
		}
		if strings.Contains(mk, key) || strings.Contains(key, mk) {
			out = append(out, mk)
		}
		if len(out) == 4 {
			break
		}
	}
	return out
}

func variantsOf(ps []Proposal) []Variant {
	out := make([]Variant, 0, len(ps))
	for _, p := range ps {
		if strings.TrimSpace(p.Dst) == "" {
			continue
		}
		out = append(out, Variant{Dst: p.Dst, Chunks: p.Chunks})
	}
	return out
}

// AttachKWIC fills every candidate's KWIC with up to maxPer contexts of ±width runes around its
// occurrences in the normalized source, scanned in (chapter, chunk, offset) order. A candidate with no
// occurrence keeps an empty list — honestly empty rather than padded, because "the term is not in the
// source I was given" is itself the signal (it means the draft invented it).
func AttachKWIC(cands []Candidate, chunks []Chunk, maxPer, width int) []Candidate {
	if maxPer <= 0 || width <= 0 {
		return cands
	}
	ordered := append([]Chunk(nil), chunks...)
	sort.SliceStable(ordered, func(i, j int) bool {
		if ordered[i].Chapter != ordered[j].Chapter {
			return ordered[i].Chapter < ordered[j].Chapter
		}
		return ordered[i].ChunkIdx < ordered[j].ChunkIdx
	})
	for i := range cands {
		cands[i].KWIC = kwicFor(cands[i].Key, ordered, maxPer, width)
		if cands[i].Freq == 0 {
			// A draft-side-only candidate has no detector frequency, and the KWIC list is CAPPED — using
			// its length as the count would print "freq: 3" for a term that occurs forty times, in the very
			// column the owner reads to judge whether a term matters. Count the occurrences properly.
			cands[i].Freq = countOccurrences(cands[i].Key, ordered)
		}
	}
	return cands
}

// countOccurrences counts non-overlapping occurrences of key across the chunks (uncapped, unlike KWIC).
func countOccurrences(key string, chunks []Chunk) int {
	if key == "" {
		return 0
	}
	n := 0
	for _, ch := range chunks {
		n += strings.Count(ch.NSource, key)
	}
	return n
}

func kwicFor(key string, chunks []Chunk, maxPer, width int) []string {
	if key == "" {
		return nil
	}
	var out []string
	for _, ch := range chunks {
		from := 0
		for len(out) < maxPer {
			i := strings.Index(ch.NSource[from:], key)
			if i < 0 {
				break
			}
			at := from + i
			out = append(out, window(ch.NSource, at, at+len(key), width))
			from = at + len(key)
		}
		if len(out) >= maxPer {
			break
		}
	}
	return out
}

// window returns the rune-bounded ±width context around [start,end) of s, never splitting a rune.
func window(s string, start, end, width int) string {
	rs := []rune(s)
	// Convert byte offsets to rune offsets by counting runes in the prefixes (the strings are chunk-sized).
	rStart := len([]rune(s[:start]))
	rEnd := len([]rune(s[:end]))
	lo := rStart - width
	if lo < 0 {
		lo = 0
	}
	hi := rEnd + width
	if hi > len(rs) {
		hi = len(rs)
	}
	return strings.TrimSpace(string(rs[lo:hi]))
}

// ScoreOpts carries the pair DATA the §C2-3 consolidation formula reads. Every factor is injected, so the
// package holds no pair knowledge: a pair with no transliteration table simply passes a nil Conformance
// and that factor goes neutral.
type ScoreOpts struct {
	// Conformance scores a rendering's fidelity to the pair's transliteration convention for a NAME or
	// PLACE, in [0,1]. nil → the factor is neutral (1.0) for every variant.
	Conformance func(dst, typ string) float64
	// Neighbours are the already-approved bank rows the "agreement with signed siblings" factor consults.
	// This is the ONLY consistency anchor of the formula (D39.47 removed the pair/genre one): a signed row
	// is a FACT about this book, whereas a pair-wide glossary would have prescribed one register to every
	// book of the pair — and the choice between «культивация» and «совершенствование» is legitimately the
	// owner's, per book.
	Neighbours []Neighbour
}

// Consolidation factor bounds. They are multiplicative and all ≥ minFactor, so no single signal can zero a
// variant out: the formula RANKS evidence for a human and a model, it does not adjudicate alone. Plain
// majority is explicitly not the rule (§C2-3: «побеждает вариант, а НЕ первое вхождение»; D39.42: «формула
// §C2 п.3, не мажоритарность») — frequency is one factor among four.
// The frequency factor's dynamic range is deliberately bounded at 2× (floor 0.5), and the conformance
// factor can match it. That is the arithmetic behind «не мажоритарность»: N chunks agreeing are N
// independent guesses made from N PARTIAL views of the book — not N confirmations — whereas conformity to
// the pair's transliteration table is a constraint from the brief, and agreement with a SIGNED sibling is
// a fact about the book. So evidence can outrank count, and with no evidence at all count still decides.
const (
	freqFloor      = 0.5 // a rendering proposed once keeps half the frequency factor
	conformBonus   = 1.0 // full transliteration conformance can overturn a maximal frequency gap
	neighbourBonus = 0.5
	lemmaPenalty   = 0.4 // a visibly mangled form (trailing hyphen, stray bracket) is heavily demoted
	// stemMinPrefix is the ABSOLUTE floor of the common-prefix test below: fewer shared characters than
	// this is never a lexeme, whatever the word lengths.
	stemMinPrefix = 3
	// stemSuffixSlack is how many trailing characters two forms of one lexeme may differ in. The test is
	// RELATIVE (shared prefix ≥ shorter length − slack) because a fixed threshold gets short words wrong in
	// both directions, which the live probe demonstrated: at a flat 5 «море» and «моря» — one signed noun
	// and its own genitive — did not count as the same word, and the canon check false-flagged a rendering
	// that in fact carried the signature. Language-neutral: it matches less, never wrongly more, for a
	// target that does not inflect by suffix.
	stemSuffixSlack = 2
)

// ScoreVariants applies the §C2-3 consolidation formula to a candidate's observed renderings and sorts
// them best-first. It is a RANKING, not a verdict: the winner is evidence handed to the terminologist (and
// to the owner at the stop), and a term whose top two variants are close is precisely the disagreement a
// canon exists to close. Deterministic — ties break on the rendering itself, never on map order.
func ScoreVariants(c *Candidate, opts ScoreOpts) {
	if len(c.Variants) == 0 {
		return
	}
	maxChunks := 0
	for _, v := range c.Variants {
		if v.Chunks > maxChunks {
			maxChunks = v.Chunks
		}
	}
	if maxChunks == 0 {
		maxChunks = 1
	}
	for i := range c.Variants {
		v := &c.Variants[i]
		v.Signals = nil
		// (a) frequency across ALL chunks — not "the first occurrence wins" (the anti-LTCR-poisoning rule).
		share := freqFloor + (1-freqFloor)*float64(v.Chunks)/float64(maxChunks)
		score := share
		if v.Chunks == maxChunks && maxChunks > 1 {
			v.Signals = append(v.Signals, "freq")
		}
		// (b) transliteration conformance, for the types where a convention exists at all.
		if opts.Conformance != nil {
			if cf := opts.Conformance(v.Dst, c.Type); cf > 0 {
				score *= 1 + conformBonus*clamp01(cf)
				if cf >= 0.5 {
					v.Signals = append(v.Signals, "conform")
				}
			}
		}
		// (c) agreement with already-APPROVED siblings of the same source series. Matched on KEY, not on the
		// raw surface: the caller normalises both tables with text.NormalizeSourceKey, so comparing the raw
		// surface makes both evidence factors go silently dead for exactly the surfaces normalisation exists
		// to reconcile (a traditional-form 修煉者 against a simplified table).
		if neighbourAgrees(c.Key, v.Dst, opts.Neighbours) {
			score *= 1 + neighbourBonus
			v.Signals = append(v.Signals, "neighbour")
		}
		// (d) lemma completeness — a PROXY. The ratified factor is morphological (a complete case lemma),
		// and the morphology analyser was dropped with default B; what is checkable without it is that the
		// form is not visibly mangled. Named as a proxy rather than dressed up as the real thing.
		if !wellFormedLemma(v.Dst) {
			score *= lemmaPenalty
			v.Signals = append(v.Signals, "malformed")
		}
		v.Score = score
	}
	sort.SliceStable(c.Variants, func(i, j int) bool {
		if c.Variants[i].Score != c.Variants[j].Score {
			return c.Variants[i].Score > c.Variants[j].Score
		}
		if c.Variants[i].Chunks != c.Variants[j].Chunks {
			return c.Variants[i].Chunks > c.Variants[j].Chunks
		}
		return c.Variants[i].Dst < c.Variants[j].Dst
	})
}

func clamp01(f float64) float64 {
	if f < 0 {
		return 0
	}
	if f > 1 {
		return 1
	}
	return f
}

// neighbourAgrees reports whether dst shares a whole word with the approved rendering of a neighbour whose
// SOURCE shares a character with src — the "same series" heuristic §C2-3 names (a 元-row rendered like its
// signed 元-siblings). Deterministic, order-free (any match suffices).
func neighbourAgrees(src, dst string, ns []Neighbour) bool {
	if src == "" || dst == "" {
		return false
	}
	want := wordSet(dst)
	if len(want) == 0 {
		return false
	}
	for _, n := range ns {
		if n.Src == src || sharedRunes(src, n.Src) == 0 {
			continue
		}
		if lexemeOverlap(want, wordSet(n.Dst)) {
			return true
		}
	}
	return false
}

// lexemeOverlap reports whether any word of a is the same LEXEME as any word of b (see stemMinPrefix).
// Both sets are iterated through sorted keys so the result never depends on map order.
func lexemeOverlap(a, b map[string]bool) bool {
	for _, x := range sortedKeys(a) {
		for _, y := range sortedKeys(b) {
			if sameLexeme(x, y) {
				return true
			}
		}
	}
	return false
}

// lexemeSubset reports whether EVERY word of need has a same-lexeme counterpart in have. Sorted iteration
// on both sides, so the answer never depends on map order.
func lexemeSubset(need, have map[string]bool) bool {
	if len(need) == 0 {
		return true
	}
	haveKeys := sortedKeys(have)
	for _, w := range sortedKeys(need) {
		found := false
		for _, h := range haveKeys {
			if sameLexeme(w, h) {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}
	return true
}

func sortedKeys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// sameLexeme is the morphology stand-in: identical forms, one a prefix of the other, or a shared prefix
// long enough relative to the SHORTER form (see stemSuffixSlack) and never shorter than stemMinPrefix.
func sameLexeme(a, b string) bool {
	if a == b {
		return true
	}
	ar, br := []rune(a), []rune(b)
	n := 0
	for n < len(ar) && n < len(br) && ar[n] == br[n] {
		n++
	}
	shorter := len(ar)
	if len(br) < shorter {
		shorter = len(br)
	}
	// One form being a full prefix of the other counts as one lexeme ONLY once the shared stem is itself
	// long enough to be a stem. Without the floor the rule fires on any short word that happens to open a
	// longer one — «гу» (蛊) against «Гуюэ» (the clan), «ад» against «адрес» — and the neighbour
	// factor would then award its bonus to a rendering with no relation to the anchor at all.
	if (n == len(ar) || n == len(br)) && shorter >= stemMinPrefix {
		return true
	}
	need := shorter - stemSuffixSlack
	if need < stemMinPrefix {
		need = stemMinPrefix
	}
	return n >= need
}

// wordSet splits a rendering into lower-cased word tokens (letters and digits only), dropping
// single-CHARACTER words — Russian prepositions and conjunctions («и», «в») carry no terminological weight
// and, in the canon check, requiring them would false-flag on a dropped preposition. The length test is in
// RUNES, not bytes: on bytes a Cyrillic «и» is 2 bytes and would survive a >1 test while a Latin "a" would
// not, which is a per-script difference nothing here intends.
func wordSet(s string) map[string]bool {
	out := map[string]bool{}
	for _, f := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
	}) {
		if len([]rune(f)) > 1 {
			out[f] = true
		}
	}
	return out
}

// wellFormedLemma is the visible-mangling check standing in for the morphological factor: a rendering must
// be non-empty, must not end mid-word on a hyphen, and must not carry stray structural characters.
func wellFormedLemma(dst string) bool {
	t := strings.TrimSpace(dst)
	if len([]rune(t)) < 2 {
		return false
	}
	if strings.HasSuffix(t, "-") || strings.HasPrefix(t, "-") {
		return false
	}
	if strings.ContainsAny(t, "[]{}<>|\t\n") {
		return false
	}
	return true
}

// Best returns the top-scored rendering, or "" when nothing was observed.
func (c Candidate) Best() string {
	if len(c.Variants) == 0 {
		return ""
	}
	return c.Variants[0].Dst
}

// Spread is the disagreement signal: how many DISTINCT renderings the drafts produced for this surface.
// One term coming back three ways is the drift a canon closes — the reason to sign it at all.
func (c Candidate) Spread() int { return len(c.Variants) }

// --- the wire: request table and reply parser ---------------------------------------------------------

// NoDst is the engine sentinel the terminologist returns for a term it cannot render confidently. A
// bracketed engine token (the ⟦TM-BANK-v1⟧/⟦TM-NOCHANGE⟧ idiom) rather than a natural-language phrase:
// wording would be pair data leaking into Go, and any phrase would eventually collide with a real
// rendering. A declined term emits status:auto (inert) instead of a guess — §C2-7's other mode.
const NoDst = "⟦TM-NO-DST⟧"

// fieldSplit is the tolerant reply delimiter (a tab, a run of ≥2 spaces, or a padded pipe) — the same
// tolerance the banknote parser applies, for the same reason: models substitute spaces for tabs.
var fieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)

// splitFields splits one reply line into its non-empty trimmed fields.
func splitFields(line string) []string {
	var parts []string
	for _, p := range fieldSplit.Split(strings.TrimSpace(line), -1) {
		if p = strings.TrimSpace(p); p != "" {
			parts = append(parts, p)
		}
	}
	return parts
}

// RenderBatch serializes one batch of candidates into the block the terminologist reads. The layout is
// engine-neutral (ASCII field names + data): every word of instruction lives in the pair's authored
// prompt, so a new pair needs no Go edit. Deterministic — candidates are already key-ordered and nothing
// here iterates a map.
func RenderBatch(cands []Candidate) string {
	var b strings.Builder
	for _, c := range cands {
		fmt.Fprintf(&b, "### %s\n", c.Src)
		fmt.Fprintf(&b, "key: %s\ntype: %s\norigin: %s\nfreq: %d\nsince_ch: %d\n", c.Key, typeOr(c.Type), c.Origin, c.Freq, c.SinceCh)
		if len(c.Aliases) > 0 {
			fmt.Fprintf(&b, "aliases: %s\n", strings.Join(c.Aliases, ", "))
		}
		if len(c.Related) > 0 {
			fmt.Fprintf(&b, "related: %s\n", strings.Join(c.Related, ", "))
		}
		if len(c.Evidence) > 0 {
			fmt.Fprintf(&b, "evidence: %s\n", strings.Join(c.Evidence, ", "))
		}
		if len(c.Variants) > 0 {
			var vs []string
			for _, v := range c.Variants {
				if v.Via != "" {
					vs = append(vs, fmt.Sprintf("%s ×%d (proposed for %s)", v.Dst, v.Chunks, v.Via))
					continue
				}
				vs = append(vs, fmt.Sprintf("%s ×%d", v.Dst, v.Chunks))
			}
			fmt.Fprintf(&b, "drafts: %s\n", strings.Join(vs, " | "))
		}
		for _, k := range c.KWIC {
			fmt.Fprintf(&b, "ctx: %s\n", k)
		}
		b.WriteString("\n")
	}
	return strings.TrimRight(b.String(), "\n")
}

func typeOr(t string) string {
	if t == "" {
		return "term"
	}
	return t
}

// CanonMarker labels the anchor block. A bracketed engine token (the ⟦TM-BANK-v1⟧ idiom) rather than a
// natural-language heading: a heading would be pair language living in Go, and the prompt is where every
// word the model reads about it belongs. CANON is the book's own SIGNED rows and is law.
//
// It is the ONLY anchor. A second block carrying pair/genre conventions shipped with pack-20 and was
// removed by D39.47: the market has no single register to anchor to («культивация» against
// «совершенствование» is a school, not a fact), so prescribing one to every book of the pair would have
// overridden the only authority that exists here — the owner's signature on THIS book's bank.
const CanonMarker = "⟦TM-CANON⟧"

// RenderCanonAnchor serializes the signed rows the terminologist reads before it decides. Empty → "" and
// the caller sends no anchor message at all, so a book with no signed rows takes a byte-identical path to
// before the anchor existed.
func RenderCanonAnchor(canon [][2]string) string {
	return renderPairs(CanonMarker, canon)
}

func renderPairs(marker string, pairs [][2]string) string {
	if len(pairs) == 0 {
		return ""
	}
	var b strings.Builder
	b.WriteString(marker)
	b.WriteString("\n")
	for _, p := range pairs {
		fmt.Fprintf(&b, "%s\t%s\n", p[0], p[1])
	}
	return strings.TrimRight(b.String(), "\n")
}

// CanonFor selects the APPROVED bank rows related to something in this batch — the book's already-signed
// law, which the consolidation must not contradict.
//
// It exists because the live probe (26.07) caught the consolidation doing exactly that: for 元海空窍 the
// DRAFT proposed «апертура моря истинной ци», canon-consistent with the signed 元海 → «море истинной ци»
// and 空窍 → «апертура», and the terminologist — which had never been shown either row — overrode it with
// «апертура Первозданного моря». A step whose whole purpose is one consistent canon must not be the step
// that breaks it, and the §C2-3 factor "agreement with approved siblings" cannot do that job alone: it can
// only RANK renderings the drafts happened to produce, and in that probe 34 of 35 candidates carried a
// single variant, so there was nothing to rank.
//
// RELATION, strongest first: one source contains the other (元海 inside 元海空窍 — the compositional case,
// and the one that actually failed), then the number of source characters the two share (元火 ~ 元水 — the
// morpheme series of a dense script; in an alphabetic pair this is weaker, which is why it ranks below
// containment and why the list is capped rather than exhaustive). Deterministic: ties break on the source.
func CanonFor(batch []Candidate, ns []Neighbour, max int) [][2]string {
	if len(batch) == 0 || len(ns) == 0 || max <= 0 {
		return nil
	}
	type scored struct {
		src, dst string
		weight   int // 2 = containment, 1 = shared characters
		shared   int
	}
	var picked []scored
	for _, n := range ns {
		if n.Src == "" || n.Dst == "" {
			continue
		}
		best := scored{src: n.Src, dst: n.Dst}
		for _, c := range batch {
			if c.Key == "" || c.Key == n.Src {
				continue // a candidate that IS the signed row has nothing to learn from it
			}
			w, sh := 0, sharedRunes(c.Key, n.Src)
			switch {
			case strings.Contains(c.Key, n.Src) || strings.Contains(n.Src, c.Key):
				w = 2
			case sh > 0:
				w = 1
			}
			if w > best.weight || (w == best.weight && sh > best.shared) {
				best.weight, best.shared = w, sh
			}
		}
		if best.weight > 0 {
			picked = append(picked, best)
		}
	}
	sort.Slice(picked, func(i, j int) bool {
		if picked[i].weight != picked[j].weight {
			return picked[i].weight > picked[j].weight
		}
		if picked[i].shared != picked[j].shared {
			return picked[i].shared > picked[j].shared
		}
		return picked[i].src < picked[j].src
	})
	if len(picked) > max {
		picked = picked[:max]
	}
	out := make([][2]string, 0, len(picked))
	for _, p := range picked {
		out = append(out, [2]string{p.src, p.dst})
	}
	return out
}

// CanonConflict is one consolidated rendering that contradicts a row the owner already signed.
type CanonConflict struct {
	Src      string // the candidate's source surface
	Dst      string // what the role consolidated it to
	CanonSrc string // the SIGNED source contained in it
	CanonDst string // the signed rendering its own dst fails to carry
}

// CanonConflicts is the deterministic half of canon consistency, and it exists because the anchor is not
// one. The live probe of 26.07 showed both halves: with the signed rows on the wire the role fixed 元海空窍
// («апертура моря истинной ци», carrying both signed elements), and in the same reply it still rendered
// 一代族长 as «Первый глава рода» while 族长 is signed «глава клана». A prompt-level anchor is a nudge; a
// paid step that can contradict the owner's signature with NO signal is the silent-degradation class this
// codebase refuses to ship blind (the post-check's whole rationale, mempostcheck.go).
//
// So this REPORTS, and only reports: for every consolidated rendering whose SOURCE strictly contains a
// signed source, the rendering must carry a word of that signed rendering (lexeme-wise, so Russian case
// endings do not false-flag). It never rewrites a rendering and never fails a run — forcing a dictionary
// form into a compound is exactly the post-REPLACE mistake E2 forbids. Deterministic: candidates are
// key-ordered, neighbours are sorted by the caller, and nothing here iterates a map for output.
func CanonConflicts(cands []Candidate, consolidated map[string]string, ns []Neighbour) []CanonConflict {
	if len(consolidated) == 0 || len(ns) == 0 {
		return nil
	}
	var out []CanonConflict
	for _, c := range cands {
		dst := consolidated[c.Key]
		if dst == "" || c.Key == "" {
			continue
		}
		have := wordSet(dst)
		for _, n := range ns {
			if n.Src == "" || n.Dst == "" || n.Src == c.Key || !strings.Contains(c.Key, n.Src) {
				continue
			}
			// EVERY significant word of the signed rendering must survive, not merely one: «Первый глава
			// рода» shares «глава» with the signed «глава клана» and is still the contradiction — the
			// disagreement lives precisely in the word that was dropped.
			if lexemeSubset(wordSet(n.Dst), have) {
				continue
			}
			out = append(out, CanonConflict{Src: c.Src, Dst: dst, CanonSrc: n.Src, CanonDst: n.Dst})
		}
	}
	return out
}

// sharedRunes counts the DISTINCT runes two sources have in common.
func sharedRunes(a, b string) int {
	if a == "" || b == "" {
		return 0
	}
	set := map[rune]bool{}
	for _, r := range a {
		set[r] = true
	}
	seen := map[rune]bool{}
	n := 0
	for _, r := range b {
		if set[r] && !seen[r] {
			seen[r] = true
			n++
		}
	}
	return n
}

// ReplyStats is what one reply cost in unusable lines. Bad is the TOTAL refused; OffLanguage is the
// subset refused by the answer-language screen, kept apart because the two mean different things to an
// operator — a broken format against a run producing canon in the wrong language.
type ReplyStats struct {
	Bad                int
	OffLanguage        int
	OffLanguageSamples []string // first few refused lines verbatim, so the warning names them
}

const offLanguageSampleCap = 8

// ParseReply turns the terminologist's reply into key → consolidated rendering. Lines are
// `<src><TAB><dst>`; the src is matched through `normalize` against the batch's EXPECTED keys, so a line
// for a term that was not asked about is ignored (a model inventing rows must not inject terms into the
// bank) and a rendering written for a differently-spelled src still lands. NoDst yields an explicit empty
// rendering — the two-mode emission's "no dst" branch, which is a decision, not a failure.
//
// `target` is the target language's script (nil → the answer-language check is inert; see OffLanguage).
//
// Returns the accepted map plus the tally of unusable lines, which the caller logs: silence about a reply
// the parser could not read is how a paid call turns into an empty bank with no signal.
func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable) (map[string]string, ReplyStats) {
	want := make(map[string]bool, len(expected))
	for _, k := range expected {
		want[k] = true
	}
	out := map[string]string{}
	var st ReplyStats
	for _, ln := range strings.Split(reply, "\n") {
		if strings.TrimSpace(ln) == "" || strings.HasPrefix(strings.TrimSpace(ln), "#") {
			continue
		}
		f := splitFields(ln)
		if len(f) < 2 {
			st.Bad++
			continue
		}
		key := normalize(f[0])
		if !want[key] {
			st.Bad++
			continue
		}
		if _, dup := out[key]; dup {
			continue // first answer wins; a model repeating itself is not a second opinion
		}
		// Everything after the FIRST field is the rendering, re-joined on single spaces. The splitter is
		// deliberately tolerant (a tab, a run of ≥2 spaces, a padded pipe) because models substitute one for
		// another — but that tolerance also splits INSIDE a rendering the moment a model writes «Фан  Юань»
		// with a stray double space, and taking f[1] alone would then bank the half-name «Фан» silently:
		// it passes wellFormedLemma, so no counter would ever mention it. The role's reply is specified as
		// exactly two fields, so there is no third column to lose by re-joining.
		dst := strings.TrimSpace(strings.Join(f[1:], " "))
		if dst == NoDst {
			out[key] = ""
			continue
		}
		if !wellFormedLemma(dst) {
			st.Bad++
			continue
		}
		// A rendering that IS the source is not a rendering. Models answer this under pressure (a term they
		// cannot translate echoed back), and nothing downstream would catch it: it passes wellFormedLemma,
		// it is stamped status:draft, and it reaches the editor as «方源 ⟨проверить⟩» — an untranslated
		// surface presented as this book's canon. Counted as a bad line, so a reply full of echoes is loud.
		if normalize(dst) == key {
			st.Bad++
			continue
		}
		// Nor is a rendering written in another script — the same argument as the echo check above, at the
		// one place a reply becomes bank content. Past here it is indistinguishable from a correct answer.
		if OffLanguage(dst, target) {
			st.Bad++
			st.OffLanguage++
			if len(st.OffLanguageSamples) < offLanguageSampleCap {
				st.OffLanguageSamples = append(st.OffLanguageSamples, f[0]+" → "+dst)
			}
			continue
		}
		out[key] = dst
	}
	return out, st
}

// Batch splits candidates into groups whose rendered size stays under maxRunes. A series (a key present in
// seriesID; pass nil to disable the channel) is kept WHOLE in one batch so the terminologist picks one
// generic head for it (§1); a series or a single candidate larger than the budget still gets its own batch
// rather than being split or silently dropped (the caller WARNs on such an over-cap unit — see BatchRunes —
// because the whole unit still bills against one output cap). The batches are then ordered by descending
// source frequency, so a budget ceiling drops the least-frequent terms instead of the lexicographic tail
// (feed_cap, §7).
//
// The ordering is DETERMINISTIC — a stable sort on a pure function of each batch's content — so the same
// input reproduces the same batch order every run and the $0 checkpoint resume holds. It is NOT, however,
// independent of that ordering: the consumer addresses batch i as ChunkIdx i (terminologist.go), which folds
// into the request hash, so a batch's POSITION is part of its address. If a later run's frequencies reorder
// the batches their ordinals shift and the affected batches re-pay — the accepted price of value-ordering,
// and no worse than the content edit that would already re-pay them.
func Batch(cands []Candidate, maxRunes int, seriesID map[string]int) [][]Candidate {
	if len(cands) == 0 {
		return nil
	}
	if maxRunes <= 0 {
		return [][]Candidate{cands}
	}
	ordered := orderBySeries(cands, seriesID)
	var out [][]Candidate
	var cur []Candidate
	size := 0
	flush := func() {
		if len(cur) > 0 {
			out = append(out, cur)
			cur, size = nil, 0
		}
	}
	for i := 0; i < len(ordered); {
		j := i + 1 // extend over a whole series block; a singleton is a unit of one
		if id := seriesID[ordered[i].Key]; id != 0 {
			for j < len(ordered) && seriesID[ordered[j].Key] == id {
				j++
			}
		}
		unit := ordered[i:j]
		if n := unitRunes(unit); len(cur) > 0 && size+n > maxRunes {
			flush()
			cur, size = append(cur, unit...), n
		} else {
			cur, size = append(cur, unit...), size+n
		}
		i = j
	}
	flush()
	sort.SliceStable(out, func(a, b int) bool { return batchFreq(out[a]) > batchFreq(out[b]) })
	return out
}

// BatchRunes reports the rendered size RenderBatch produces for a whole batch — the same measure Batch packs
// against. The terminologist uses it to WARN when a co-batched series or a lone candidate exceeds the budget:
// §1 keeps such a unit WHOLE rather than split it, so it is a real over-cap call the operator must see coming
// (the cap-8000 output mine, PROBES §1), not a silent overrun.
func BatchRunes(batch []Candidate) int { return unitRunes(batch) }

// unitRunes is the rendered size the accountant charges a unit: each member's single-block render plus the
// blank line RenderBatch puts between blocks (+2). Without the +2 a 200-term book overruns its budget by
// 2×(N−1) runes. Summing per member matches the original per-candidate arithmetic exactly.
func unitRunes(unit []Candidate) int {
	n := 0
	for _, c := range unit {
		n += len([]rune(RenderBatch([]Candidate{c}))) + 2
	}
	return n
}

func batchFreq(b []Candidate) int {
	n := 0
	for _, c := range b {
		n += c.Freq
	}
	return n
}