textmachine/backend/internal/terminology/terminology.go

// Package terminology is the deterministic core of the TERMINOLOGIST role (D39.42 п.1): it merges the two
// bank-candidate channels into one list, gathers the source contexts each candidate needs to be translated
// at all, ranks the renderings the drafts already produced, and turns the model's reply back into
// consolidated terms.
//
// The split of duty is the point. The miner answers WHICH source surfaces belong in the bank (offline,
// source-side, high recall); the banknote answers WHAT some chunk's translator called them (draft-side,
// sparse — the two channels named overlapping entities in only 3% of the mini-run by construction). Neither
// answers "what should this book call it". That question needs the WHOLE book's contexts at once, which is
// exactly what exists at the wave boundary and nowhere else.
//
// Everything here is PURE and deterministic: no clock, no randomness, no map-order iteration, no I/O, no
// network. The pair data (Palladius conformance) arrives as values, so the package holds no pair-specific
// literal and works for a pair that is not in the repo yet.
package terminology

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"unicode"
)

// Origin records WHICH channel surfaced a candidate — the provenance the owner reads at sign time and the
// only thing that distinguishes "the detector found it in the source" from "a translator invented it".
type Origin string

const (
	OriginMined    Origin = "mined"    // the offline WHICH detector saw it in the source
	OriginBanknote Origin = "banknote" // only a draft's banknote proposed it (the reverse section)
	OriginBoth     Origin = "both"     // both channels named the same entity
	// OriginAlias: a draft named a surface the miner holds as an ALIAS of another cluster. It is kept as a
	// candidate of its own so the terminologist is asked to translate it, but it is NOT emitted into the
	// delta — whether the cluster is right is a clustering question, and that decision is not this pack's.
	OriginAlias Origin = "alias"
)

// Chunk is one normalized source chunk — the same view the miner takes (the candidate space and the KWIC
// space must be the same space, or a term is contextless exactly when it is hardest to translate).
type Chunk struct {
	Chapter  int
	ChunkIdx int
	NSource  string
}

// Mined is one candidate from the offline WHICH detector.
type Mined struct {
	Key      string // normalized source key (the join key)
	Src      string // the surface as the detector holds it
	Type     string // name|place|title|term
	Freq     int
	SinceCh  int
	Aliases  []string // normalized co-surfaces of the same entity
	Evidence []string
}

// Proposal is one draft-side rendering of a surface, already folded per (key, dst) by the caller.
type Proposal struct {
	Dst    string
	Type   string
	Chunks int // how many draft chunks proposed exactly this rendering
}

// Observed is the draft-side view of one source surface (the banknote channel, folded per key).
type Observed struct {
	Key       string
	Src       string
	Type      string
	Proposals []Proposal
}

// Neighbour is an already-APPROVED bank row — the anchor the §C2-3 "agreement with approved neighbours of
// the series" factor scores against (the 元-series precedent: a new 元-term rendered like its signed
// siblings is likelier right than one that is not).
type Neighbour struct {
	Src string
	Dst string
}

// Variant is one observed rendering with its consolidation score.
type Variant struct {
	Dst    string
	Chunks int
	// Forms is how many DISTINCT raw renderings folded into this one (see foldVariants). 1 for an unfolded
	// variant; it is what keeps Spread() meaning "how many ways did the drafts write it" once the vote is
	// counted per CONVENTION rather than per byte string.
	Forms   int
	Score   float64
	Signals []string // which factors fired, in fixed order — the score's audit trail
	// Via names the surface that actually proposed this rendering when it is NOT the candidate's own —
	// i.e. an ALIAS of the cluster. Without it a mis-clustered alias silently re-labels its rendering: the
	// polygon's 葛家 cluster holding 族长 makes «глава клана» read as a rendering OF the family name, and the
	// terminologist would consolidate a surname to a title. Provenance, not a decision.
	Via string
}

// Candidate is one merged bank candidate: the union of what both channels know about one entity.
type Candidate struct {
	Key      string
	Src      string
	Type     string
	Freq     int // occurrences in the source; 0 when only the draft side saw it
	SinceCh  int
	Aliases  []string
	Evidence []string
	Origin   Origin
	Related  []string // mined keys this surface CONTAINS or is contained by (never merged — see Merge)
	Variants []Variant
	KWIC     []string
}

// Merge is the two-way $0 join of the WHICH and WHAT channels, with the alias/containment resolver that
// decides — deterministically — whether a draft-side surface IS a mined candidate or merely touches one.
//
//   - EXACT key match → MERGE. Same entity, so the draft's rendering is a rendering OF it.
//   - registered ALIAS of a mined entity → the rendering merges into the cluster (so the §C2-3 ranking sees
//     all of the entity's evidence) AND the surface ALSO stands as a candidate of its own, Origin `alias`,
//     pointing at the cluster. The miner's clustering can be wrong — measured on 50 chapters: a surname
//     cluster holding a clan title — and a plain merge is where such a term DISAPPEARS: absent from the
//     delta (the cluster represents it) and absent from the input (it folded away). The merged rendering
//     carries Via so a mis-clustered alias cannot silently re-label a surname to a title.
//   - CONTAINMENT (方源 inside 古月方源) → a candidate of its OWN, in the reverse section, with the mined key
//     recorded in Related. This is deliberately NOT a merge: «Гуюэ Фан Юань» is not a rendering of 方源, and
//     folding it in would put a wrong variant on a term the owner is about to sign. Containment is also
//     exactly where the miner's emission is silent — a cluster touching a seed surface is suppressed as an
//     alias-of-existing — so the reverse section is where the 3% coverage gap actually lives.
//   - otherwise → a candidate of its own, Origin banknote.
//
// Output order is by Key (never map order). Every mined candidate appears exactly once; every observed key
// either merges into one or appears once in the reverse section.
//
// foldDst is the TARGET-form normalizer the vote is counted under (see foldVariants); nil folds only
// byte-identical renderings, which is what this did before the fix-pack. It is injected rather than
// imported so the package keeps no target knowledge — the ё/case/whitespace folds a Russian target needs
// are facts about that target, not about this algorithm.
func Merge(mined []Mined, observed []Observed, foldDst func(string) string) []Candidate {
	byKey := make(map[string]*Candidate, len(mined))
	order := make([]string, 0, len(mined)+len(observed))
	// aliasOwner maps every alias surface to its owning mined key, so an aliased proposal merges.
	aliasOwner := map[string]string{}
	for _, m := range mined {
		if m.Key == "" || byKey[m.Key] != nil {
			continue
		}
		c := &Candidate{
			Key: m.Key, Src: m.Src, Type: m.Type, Freq: m.Freq, SinceCh: m.SinceCh,
			Aliases: append([]string(nil), m.Aliases...), Evidence: append([]string(nil), m.Evidence...),
			Origin: OriginMined,
		}
		byKey[m.Key] = c
		order = append(order, m.Key)
		for _, a := range m.Aliases {
			if a != "" && aliasOwner[a] == "" && byKey[a] == nil {
				aliasOwner[a] = m.Key
			}
		}
	}
	minedKeys := make([]string, 0, len(byKey))
	for k := range byKey {
		minedKeys = append(minedKeys, k)
	}
	sort.Strings(minedKeys) // deterministic containment scan

	for _, o := range observed {
		if o.Key == "" {
			continue
		}
		owner := o.Key
		if byKey[owner] == nil {
			owner = aliasOwner[o.Key]
		}
		if owner != "" && byKey[owner] != nil { // exact or alias → merge
			c := byKey[owner]
			if c.Origin == OriginMined {
				c.Origin = OriginBoth
			}
			vs := variantsOf(o.Proposals)
			if owner != o.Key { // merged through an ALIAS, not on its own key
				for i := range vs {
					vs[i].Via = o.Src
				}
			}
			c.Variants = append(c.Variants, vs...)
			if c.Type == "" {
				c.Type = o.Type
			}
			if owner != o.Key && byKey[o.Key] == nil {
				// DO NOT LOSE THE SURFACE (polygon package seven, G-series). The miner's alias cluster may be
				// wrong — measured: a surname cluster swallowing a clan title — and when it is, merging is the
				// only place the term can disappear: it is not in the miner's delta (the cluster represents it)
				// and it is not a candidate here (it folded into the owner). The terminologist then never sees
				// a term the drafts named seventeen times. So the surface ALSO stands on its own, carrying its
				// own renderings and a pointer to the cluster that claims it. Origin `alias` keeps it out of
				// the emitted delta: proposing it for signature would BE the clustering fix, which is a
				// separate decision.
				a := &Candidate{Key: o.Key, Src: o.Src, Type: o.Type, Origin: OriginAlias,
					Related: []string{owner}, Variants: variantsOf(o.Proposals)}
				byKey[a.Key] = a
				order = append(order, a.Key)
			}
			continue
		}
		c := &Candidate{
			Key: o.Key, Src: o.Src, Type: o.Type, Origin: OriginBanknote,
			Related: containmentRelations(o.Key, minedKeys), Variants: variantsOf(o.Proposals),
		}
		if byKey[c.Key] != nil {
			continue // a duplicate observed key (the caller folded per key, but never assume it)
		}
		byKey[c.Key] = c
		order = append(order, c.Key)
	}

	out := make([]Candidate, 0, len(order))
	for _, k := range order {
		c := *byKey[k]
		c.Variants = foldVariants(c.Variants, foldDst)
		out = append(out, c)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
	return out
}

// foldVariants sums the chunk counts of renderings that are the SAME CONVENTION and shows the raw form.
//
// It matters on the alias path: 方源 and its alias 方小子 arrive as two Observed rows, and both may carry the
// same rendering — appended raw that is two variants of one word, which lies twice. It matters just as much
// on orthography (fix-pack §G5, research/24 §A5): «Море истинной ци» and «море истинной ци» are one
// convention written two ways, and folding only byte-identical strings splits the §C2-3 frequency factor
// across them — the consensus rendering is then scored on half its evidence and a genuine competitor can
// take the win. So the VOTE is counted under foldDst (the target-form normalizer), while the form the owner
// and the model see stays the RAW one the drafts actually wrote.
//
// Two counts survive the fold and mean different things: Spread() (Σ Forms) is how many ways the drafts
// wrote the term, Conventions() is how many of those are genuinely different decisions.
//
// Via is aggregated honestly rather than by first-wins. A rendering that ALSO arrived on the candidate's own
// key is direct — labelling it «proposed for <alias>» because the alias happened to be seen first would
// present a consensus as a mis-clustered alias's guess. Only when EVERY contribution came through aliases
// is the provenance kept, and then it names all of them.
//
// Deterministic: first-seen order of the folded classes is preserved, and the representative is the raw form
// with the most chunks, ties going to the one seen first.
func foldVariants(vs []Variant, foldDst func(string) string) []Variant {
	if len(vs) == 0 {
		return vs
	}
	if foldDst == nil {
		foldDst = func(s string) string { return s }
	}
	type form struct {
		dst    string
		chunks int
	}
	type class struct {
		forms   []form
		byForm  map[string]int
		chunks  int
		direct  bool
		vias    []string
		seenVia map[string]bool
	}
	at := make(map[string]int, len(vs))
	classes := make([]*class, 0, len(vs))
	for _, v := range vs {
		k := foldDst(v.Dst)
		i, seen := at[k]
		if !seen {
			i = len(classes)
			at[k] = i
			classes = append(classes, &class{byForm: map[string]int{}, seenVia: map[string]bool{}})
		}
		cl := classes[i]
		cl.chunks += v.Chunks
		if j, had := cl.byForm[v.Dst]; had {
			cl.forms[j].chunks += v.Chunks
		} else {
			cl.byForm[v.Dst] = len(cl.forms)
			cl.forms = append(cl.forms, form{dst: v.Dst, chunks: v.Chunks})
		}
		if v.Via == "" {
			cl.direct = true
		} else if !cl.seenVia[v.Via] {
			cl.seenVia[v.Via] = true
			cl.vias = append(cl.vias, v.Via)
		}
	}
	out := make([]Variant, 0, len(classes))
	for _, cl := range classes {
		best := 0
		for j := 1; j < len(cl.forms); j++ {
			if cl.forms[j].chunks > cl.forms[best].chunks {
				best = j
			}
		}
		v := Variant{Dst: cl.forms[best].dst, Chunks: cl.chunks, Forms: len(cl.forms)}
		if !cl.direct {
			v.Via = strings.Join(cl.vias, ", ")
		}
		out = append(out, v)
	}
	return out
}

// containmentRelations returns the mined keys that strictly contain, or are strictly contained by, key —
// sorted, so the record is deterministic. Bounded to the first few: the relation is context for a human
// and a model, not an index.
func containmentRelations(key string, minedKeys []string) []string {
	var out []string
	for _, mk := range minedKeys {
		if mk == key {
			continue
		}
		if strings.Contains(mk, key) || strings.Contains(key, mk) {
			out = append(out, mk)
		}
		if len(out) == 4 {
			break
		}
	}
	return out
}

func variantsOf(ps []Proposal) []Variant {
	out := make([]Variant, 0, len(ps))
	for _, p := range ps {
		if strings.TrimSpace(p.Dst) == "" {
			continue
		}
		out = append(out, Variant{Dst: p.Dst, Chunks: p.Chunks})
	}
	return out
}

// AttachKWIC fills every candidate's KWIC with up to maxPer contexts of ±width runes around its
// occurrences in the normalized source, scanned in (chapter, chunk, offset) order. A candidate with no
// occurrence keeps an empty list — honestly empty rather than padded, because "the term is not in the
// source I was given" is itself the signal (it means the draft invented it).
func AttachKWIC(cands []Candidate, chunks []Chunk, maxPer, width int) []Candidate {
	if maxPer <= 0 || width <= 0 {
		return cands
	}
	ordered := append([]Chunk(nil), chunks...)
	sort.SliceStable(ordered, func(i, j int) bool {
		if ordered[i].Chapter != ordered[j].Chapter {
			return ordered[i].Chapter < ordered[j].Chapter
		}
		return ordered[i].ChunkIdx < ordered[j].ChunkIdx
	})
	for i := range cands {
		cands[i].KWIC = kwicFor(cands[i].Key, ordered, maxPer, width)
		if cands[i].Freq == 0 {
			// A draft-side-only candidate has no detector frequency, and the KWIC list is CAPPED — using
			// its length as the count would print "freq: 3" for a term that occurs forty times, in the very
			// column the owner reads to judge whether a term matters. Count the occurrences properly.
			cands[i].Freq = countOccurrences(cands[i].Key, ordered)
		}
	}
	return cands
}

// countOccurrences counts non-overlapping occurrences of key across the chunks (uncapped, unlike KWIC).
func countOccurrences(key string, chunks []Chunk) int {
	if key == "" {
		return 0
	}
	n := 0
	for _, ch := range chunks {
		n += strings.Count(ch.NSource, key)
	}
	return n
}

func kwicFor(key string, chunks []Chunk, maxPer, width int) []string {
	if key == "" {
		return nil
	}
	var out []string
	for _, ch := range chunks {
		from := 0
		for len(out) < maxPer {
			i := strings.Index(ch.NSource[from:], key)
			if i < 0 {
				break
			}
			at := from + i
			out = append(out, window(ch.NSource, at, at+len(key), width))
			from = at + len(key)
		}
		if len(out) >= maxPer {
			break
		}
	}
	return out
}

// window returns the rune-bounded ±width context around [start,end) of s, never splitting a rune.
func window(s string, start, end, width int) string {
	rs := []rune(s)
	// Convert byte offsets to rune offsets by counting runes in the prefixes (the strings are chunk-sized).
	rStart := len([]rune(s[:start]))
	rEnd := len([]rune(s[:end]))
	lo := rStart - width
	if lo < 0 {
		lo = 0
	}
	hi := rEnd + width
	if hi > len(rs) {
		hi = len(rs)
	}
	return strings.TrimSpace(string(rs[lo:hi]))
}

// ScoreOpts carries the pair DATA the §C2-3 consolidation formula reads. Every factor is injected, so the
// package holds no pair knowledge: a pair with no transliteration table simply passes a nil Conformance
// and that factor goes neutral.
type ScoreOpts struct {
	// Conformance scores a rendering's fidelity to the pair's transliteration convention for a NAME or
	// PLACE, in [0,1]. nil → the factor is neutral (1.0) for every variant.
	Conformance func(dst, typ string) float64
	// Neighbours are the already-approved bank rows the "agreement with signed siblings" factor consults.
	// This is the ONLY consistency anchor of the formula (D39.47 removed the pair/genre one): a signed row
	// is a FACT about this book, whereas a pair-wide glossary would have prescribed one register to every
	// book of the pair — and the choice between «культивация» and «совершенствование» is legitimately the
	// owner's, per book.
	Neighbours []Neighbour
}

// Consolidation factor bounds. They are multiplicative and all ≥ minFactor, so no single signal can zero a
// variant out: the formula RANKS evidence for a human and a model, it does not adjudicate alone. Plain
// majority is explicitly not the rule (§C2-3: «побеждает вариант, а НЕ первое вхождение»; D39.42: «формула
// §C2 п.3, не мажоритарность») — frequency is one factor among four.
// The frequency factor's dynamic range is deliberately bounded at 2× (floor 0.5), and the conformance
// factor can match it. That is the arithmetic behind «не мажоритарность»: N chunks agreeing are N
// independent guesses made from N PARTIAL views of the book — not N confirmations — whereas conformity to
// the pair's transliteration table is a constraint from the brief, and agreement with a SIGNED sibling is
// a fact about the book. So evidence can outrank count, and with no evidence at all count still decides.
const (
	freqFloor      = 0.5 // a rendering proposed once keeps half the frequency factor
	conformBonus   = 1.0 // full transliteration conformance can overturn a maximal frequency gap
	neighbourBonus = 0.5
	lemmaPenalty   = 0.4 // a visibly mangled form (trailing hyphen, stray bracket) is heavily demoted
	// stemMinPrefix is the ABSOLUTE floor of the common-prefix test below: fewer shared characters than
	// this is never a lexeme, whatever the word lengths.
	stemMinPrefix = 3
	// stemSuffixSlack is how many trailing characters two forms of one lexeme may differ in. The test is
	// RELATIVE (shared prefix ≥ shorter length − slack) because a fixed threshold gets short words wrong in
	// both directions, which the live probe demonstrated: at a flat 5 «море» and «моря» — one signed noun
	// and its own genitive — did not count as the same word, and the canon check false-flagged a rendering
	// that in fact carried the signature. Language-neutral: it matches less, never wrongly more, for a
	// target that does not inflect by suffix.
	stemSuffixSlack = 2
)

// ScoreVariants applies the §C2-3 consolidation formula to a candidate's observed renderings and sorts
// them best-first. It is a RANKING, not a verdict: the winner is evidence handed to the terminologist (and
// to the owner at the stop), and a term whose top two variants are close is precisely the disagreement a
// canon exists to close. Deterministic — ties break on the rendering itself, never on map order.
func ScoreVariants(c *Candidate, opts ScoreOpts) {
	if len(c.Variants) == 0 {
		return
	}
	maxChunks := 0
	for _, v := range c.Variants {
		if v.Chunks > maxChunks {
			maxChunks = v.Chunks
		}
	}
	if maxChunks == 0 {
		maxChunks = 1
	}
	for i := range c.Variants {
		v := &c.Variants[i]
		v.Signals = nil
		// (a) frequency across ALL chunks — not "the first occurrence wins" (the anti-LTCR-poisoning rule).
		share := freqFloor + (1-freqFloor)*float64(v.Chunks)/float64(maxChunks)
		score := share
		if v.Chunks == maxChunks && maxChunks > 1 {
			v.Signals = append(v.Signals, "freq")
		}
		// (b) transliteration conformance, for the types where a convention exists at all.
		if opts.Conformance != nil {
			if cf := opts.Conformance(v.Dst, c.Type); cf > 0 {
				score *= 1 + conformBonus*clamp01(cf)
				if cf >= 0.5 {
					v.Signals = append(v.Signals, "conform")
				}
			}
		}
		// (c) agreement with already-APPROVED siblings of the same source series. Matched on KEY, not on the
		// raw surface: the caller normalises both tables with text.NormalizeSourceKey, so comparing the raw
		// surface makes both evidence factors go silently dead for exactly the surfaces normalisation exists
		// to reconcile (a traditional-form 修煉者 against a simplified table).
		if neighbourAgrees(c.Key, v.Dst, opts.Neighbours) {
			score *= 1 + neighbourBonus
			v.Signals = append(v.Signals, "neighbour")
		}
		// (d) lemma completeness — a PROXY. The ratified factor is morphological (a complete case lemma),
		// and the morphology analyser was dropped with default B; what is checkable without it is that the
		// form is not visibly mangled. Named as a proxy rather than dressed up as the real thing.
		if !wellFormedLemma(v.Dst) {
			score *= lemmaPenalty
			v.Signals = append(v.Signals, "malformed")
		}
		v.Score = score
	}
	sort.SliceStable(c.Variants, func(i, j int) bool {
		if c.Variants[i].Score != c.Variants[j].Score {
			return c.Variants[i].Score > c.Variants[j].Score
		}
		if c.Variants[i].Chunks != c.Variants[j].Chunks {
			return c.Variants[i].Chunks > c.Variants[j].Chunks
		}
		return c.Variants[i].Dst < c.Variants[j].Dst
	})
}

func clamp01(f float64) float64 {
	if f < 0 {
		return 0
	}
	if f > 1 {
		return 1
	}
	return f
}

// neighbourAgrees reports whether dst shares a whole word with the approved rendering of a neighbour whose
// SOURCE shares a character with src — the "same series" heuristic §C2-3 names (a 元-row rendered like its
// signed 元-siblings). Deterministic, order-free (any match suffices).
func neighbourAgrees(src, dst string, ns []Neighbour) bool {
	if src == "" || dst == "" {
		return false
	}
	want := wordSet(dst)
	if len(want) == 0 {
		return false
	}
	for _, n := range ns {
		if n.Src == src || sharedRunes(src, n.Src) == 0 {
			continue
		}
		if lexemeOverlap(want, wordSet(n.Dst)) {
			return true
		}
	}
	return false
}

// lexemeOverlap reports whether any word of a is the same LEXEME as any word of b (see stemMinPrefix).
// Both sets are iterated through sorted keys so the result never depends on map order.
func lexemeOverlap(a, b map[string]bool) bool {
	for _, x := range sortedKeys(a) {
		for _, y := range sortedKeys(b) {
			if sameLexeme(x, y) {
				return true
			}
		}
	}
	return false
}

// lexemeSubset reports whether EVERY word of need has a same-lexeme counterpart in have. Sorted iteration
// on both sides, so the answer never depends on map order.
func lexemeSubset(need, have map[string]bool) bool {
	if len(need) == 0 {
		return true
	}
	haveKeys := sortedKeys(have)
	for _, w := range sortedKeys(need) {
		found := false
		for _, h := range haveKeys {
			if sameLexeme(w, h) {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}
	return true
}

func sortedKeys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// sameLexeme is the morphology stand-in: identical forms, one a prefix of the other, or a shared prefix
// long enough relative to the SHORTER form (see stemSuffixSlack) and never shorter than stemMinPrefix.
func sameLexeme(a, b string) bool {
	if a == b {
		return true
	}
	ar, br := []rune(a), []rune(b)
	n := 0
	for n < len(ar) && n < len(br) && ar[n] == br[n] {
		n++
	}
	shorter := len(ar)
	if len(br) < shorter {
		shorter = len(br)
	}
	// One form being a full prefix of the other counts as one lexeme ONLY once the shared stem is itself
	// long enough to be a stem. Without the floor the rule fires on any short word that happens to open a
	// longer one — «гу» (蛊) against «Гуюэ» (the clan), «ад» against «адрес» — and the neighbour
	// factor would then award its bonus to a rendering with no relation to the anchor at all.
	if (n == len(ar) || n == len(br)) && shorter >= stemMinPrefix {
		return true
	}
	need := shorter - stemSuffixSlack
	if need < stemMinPrefix {
		need = stemMinPrefix
	}
	return n >= need
}

// wordSet splits a rendering into lower-cased word tokens (letters and digits only), dropping
// single-CHARACTER words — Russian prepositions and conjunctions («и», «в») carry no terminological weight
// and, in the canon check, requiring them would false-flag on a dropped preposition. The length test is in
// RUNES, not bytes: on bytes a Cyrillic «и» is 2 bytes and would survive a >1 test while a Latin "a" would
// not, which is a per-script difference nothing here intends.
func wordSet(s string) map[string]bool {
	out := map[string]bool{}
	for _, f := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
	}) {
		if len([]rune(f)) > 1 {
			out[f] = true
		}
	}
	return out
}

// wellFormedLemma is the visible-mangling check standing in for the morphological factor: a rendering must
// be non-empty, must not end mid-word on a hyphen, and must not carry stray structural characters.
func wellFormedLemma(dst string) bool {
	t := strings.TrimSpace(dst)
	if len([]rune(t)) < 2 {
		return false
	}
	if strings.HasSuffix(t, "-") || strings.HasPrefix(t, "-") {
		return false
	}
	if strings.ContainsAny(t, "[]{}<>|\t\n") {
		return false
	}
	return true
}

// Best returns the top-scored rendering, or "" when nothing was observed.
func (c Candidate) Best() string {
	if len(c.Variants) == 0 {
		return ""
	}
	return c.Variants[0].Dst
}

// Spread is the disagreement signal: how many DISTINCT renderings the drafts produced for this surface.
// One term coming back three ways is the drift a canon closes — the reason to sign it at all. It counts RAW
// forms (Forms per folded class), so the fix-pack's convention fold did not silently shrink the column the
// owner reads to decide whether a term is contested. A variant built outside Merge carries no Forms and
// counts as one.
func (c Candidate) Spread() int {
	n := 0
	for _, v := range c.Variants {
		if v.Forms > 1 {
			n += v.Forms
			continue
		}
		n++
	}
	return n
}

// Conventions is the disagreement that actually matters: how many distinct DECISIONS the drafts made, after
// renderings that differ only in target form (case, ё, spacing) are folded together. Spread 3 with
// Conventions 1 is one canon written three ways — a normalization nit; Spread 3 with Conventions 3 is a real
// contest, and the two must not read the same at the stop.
func (c Candidate) Conventions() int { return len(c.Variants) }

// --- the wire: request table and reply parser ---------------------------------------------------------

// NoDst is the engine sentinel the terminologist returns for a term it cannot render confidently. A
// bracketed engine token (the ⟦TM-BANK-v1⟧/⟦TM-NOCHANGE⟧ idiom) rather than a natural-language phrase:
// wording would be pair data leaking into Go, and any phrase would eventually collide with a real
// rendering. A declined term emits status:auto (inert) instead of a guess — §C2-7's other mode.
const NoDst = "⟦TM-NO-DST⟧"

// fieldSplit is the tolerant reply delimiter (a tab, a run of ≥2 spaces, or a padded pipe) — the same
// tolerance the banknote parser applies, for the same reason: models substitute spaces for tabs.
var fieldSplit = regexp.MustCompile(`\t| {2,}|\s*\|\s*`)

// columnSplit is the UNAMBIGUOUS half of that tolerance — a tab or a pipe is a delimiter the model chose and
// cannot occur inside a rendering, so a line carrying one has declared its own columns. spaceRun folds the
// stray spacing inside such a column. Both exist because guessing where the columns are, on a line that says
// where they are, is how a fix for one mis-split rendering corrupts the lines that were never mis-split.
var (
	columnSplit = regexp.MustCompile(`\t|\s*\|\s*`)
	spaceRun    = regexp.MustCompile(` {2,}`)
)

// replyColumns splits one reply line into src, rendering and the CONFIDENCE column, and says whether a
// third column was present but unreadable as a confidence.
//
// A tab/pipe line is positional: field three IS the confidence column the pair's prompt declared, so a value
// that is not a bare 0…100 is a malformed CONFIDENCE — counted, and never glued onto the rendering. That
// glue is the failure this shape exists to prevent: «наставник» + «95%» silently became the rendering
// «наставник 95%», passed every downstream screen (it is well-formed, it is in the target script, it is not
// an echo) and would enter the bank as this book's canon with every counter reading clean.
//
// A line delimited only by a run of ≥2 spaces is ambiguous — that tolerance is exactly what splits «Фан␣␣
// Юань» — so there the rendering is re-joined and only a trailing bare number is taken as a confidence.
func replyColumns(line string) (src, dst string, conf int, badConf, ok bool) {
	conf = -1
	if cols := splitOn(columnSplit, line); len(cols) >= 2 {
		src, dst = cols[0], strings.TrimSpace(spaceRun.ReplaceAllString(cols[1], " "))
		if len(cols) >= 3 {
			if v, good := confidenceField(cols[2]); good {
				conf = v
			} else {
				badConf = true
			}
		}
		return src, dst, conf, badConf, dst != ""
	}
	f := splitOn(fieldSplit, line)
	if len(f) < 2 {
		return "", "", -1, false, false
	}
	if len(f) >= 3 {
		if v, good := confidenceField(f[len(f)-1]); good {
			conf, f = v, f[:len(f)-1]
		}
	}
	return f[0], strings.TrimSpace(strings.Join(f[1:], " ")), conf, false, true
}

// splitOn splits a line on re, dropping empty and whitespace-only fields.
func splitOn(re *regexp.Regexp, line string) []string {
	var out []string
	for _, p := range re.Split(strings.TrimSpace(line), -1) {
		if p = strings.TrimSpace(p); p != "" {
			out = append(out, p)
		}
	}
	return out
}

// splitFields splits one reply line into its non-empty trimmed fields.
func splitFields(line string) []string {
	var parts []string
	for _, p := range fieldSplit.Split(strings.TrimSpace(line), -1) {
		if p = strings.TrimSpace(p); p != "" {
			parts = append(parts, p)
		}
	}
	return parts
}

// RenderBatch serializes one batch of candidates into the block the terminologist reads. The layout is
// engine-neutral (ASCII field names + data): every word of instruction lives in the pair's authored
// prompt, so a new pair needs no Go edit. Deterministic — candidates are already key-ordered and nothing
// here iterates a map.
func RenderBatch(cands []Candidate) string {
	var b strings.Builder
	for _, c := range cands {
		fmt.Fprintf(&b, "### %s\n", c.Src)
		fmt.Fprintf(&b, "key: %s\ntype: %s\norigin: %s\nfreq: %d\nsince_ch: %d\n", c.Key, typeOr(c.Type), c.Origin, c.Freq, c.SinceCh)
		if len(c.Aliases) > 0 {
			fmt.Fprintf(&b, "aliases: %s\n", strings.Join(c.Aliases, ", "))
		}
		if len(c.Related) > 0 {
			fmt.Fprintf(&b, "related: %s\n", strings.Join(c.Related, ", "))
		}
		if len(c.Evidence) > 0 {
			fmt.Fprintf(&b, "evidence: %s\n", strings.Join(c.Evidence, ", "))
		}
		if len(c.Variants) > 0 {
			var vs []string
			for _, v := range c.Variants {
				if v.Via != "" {
					vs = append(vs, fmt.Sprintf("%s ×%d (proposed for %s)", v.Dst, v.Chunks, v.Via))
					continue
				}
				vs = append(vs, fmt.Sprintf("%s ×%d", v.Dst, v.Chunks))
			}
			fmt.Fprintf(&b, "drafts: %s\n", strings.Join(vs, " | "))
		}
		for _, k := range c.KWIC {
			fmt.Fprintf(&b, "ctx: %s\n", k)
		}
		b.WriteString("\n")
	}
	return strings.TrimRight(b.String(), "\n")
}

func typeOr(t string) string {
	if t == "" {
		return "term"
	}
	return t
}

// CanonMarker labels the anchor block. A bracketed engine token (the ⟦TM-BANK-v1⟧ idiom) rather than a
// natural-language heading: a heading would be pair language living in Go, and the prompt is where every
// word the model reads about it belongs. CANON is the book's own SIGNED rows and is law.
//
// It is the ONLY anchor. A second block carrying pair/genre conventions shipped with pack-20 and was
// removed by D39.47: the market has no single register to anchor to («культивация» against
// «совершенствование» is a school, not a fact), so prescribing one to every book of the pair would have
// overridden the only authority that exists here — the owner's signature on THIS book's bank.
const CanonMarker = "⟦TM-CANON⟧"

// RenderCanonAnchor serializes the signed rows the terminologist reads before it decides. Empty → "" and
// the caller sends no anchor message at all, so a book with no signed rows takes a byte-identical path to
// before the anchor existed.
func RenderCanonAnchor(canon [][2]string) string {
	return renderPairs(CanonMarker, canon)
}

func renderPairs(marker string, pairs [][2]string) string {
	if len(pairs) == 0 {
		return ""
	}
	var b strings.Builder
	b.WriteString(marker)
	b.WriteString("\n")
	for _, p := range pairs {
		fmt.Fprintf(&b, "%s\t%s\n", p[0], p[1])
	}
	return strings.TrimRight(b.String(), "\n")
}

// CanonFor selects the APPROVED bank rows related to something in this batch — the book's already-signed
// law, which the consolidation must not contradict.
//
// It exists because the live probe (26.07) caught the consolidation doing exactly that: for 元海空窍 the
// DRAFT proposed «апертура моря истинной ци», canon-consistent with the signed 元海 → «море истинной ци»
// and 空窍 → «апертура», and the terminologist — which had never been shown either row — overrode it with
// «апертура Первозданного моря». A step whose whole purpose is one consistent canon must not be the step
// that breaks it, and the §C2-3 factor "agreement with approved siblings" cannot do that job alone: it can
// only RANK renderings the drafts happened to produce, and in that probe 34 of 35 candidates carried a
// single variant, so there was nothing to rank.
//
// RELATION, strongest first: one source contains the other (元海 inside 元海空窍 — the compositional case,
// and the one that actually failed), then the number of source characters the two share (元火 ~ 元水 — the
// morpheme series of a dense script; in an alphabetic pair this is weaker, which is why it ranks below
// containment and why the list is capped rather than exhaustive). Deterministic: ties break on the source.
func CanonFor(batch []Candidate, ns []Neighbour, max int) [][2]string {
	if len(batch) == 0 || len(ns) == 0 || max <= 0 {
		return nil
	}
	type scored struct {
		src, dst string
		weight   int // 2 = containment, 1 = shared characters
		shared   int
	}
	var picked []scored
	for _, n := range ns {
		if n.Src == "" || n.Dst == "" {
			continue
		}
		best := scored{src: n.Src, dst: n.Dst}
		for _, c := range batch {
			if c.Key == "" || c.Key == n.Src {
				continue // a candidate that IS the signed row has nothing to learn from it
			}
			w, sh := 0, sharedRunes(c.Key, n.Src)
			switch {
			case strings.Contains(c.Key, n.Src) || strings.Contains(n.Src, c.Key):
				w = 2
			case sh > 0:
				w = 1
			}
			if w > best.weight || (w == best.weight && sh > best.shared) {
				best.weight, best.shared = w, sh
			}
		}
		if best.weight > 0 {
			picked = append(picked, best)
		}
	}
	sort.Slice(picked, func(i, j int) bool {
		if picked[i].weight != picked[j].weight {
			return picked[i].weight > picked[j].weight
		}
		if picked[i].shared != picked[j].shared {
			return picked[i].shared > picked[j].shared
		}
		return picked[i].src < picked[j].src
	})
	if len(picked) > max {
		picked = picked[:max]
	}
	out := make([][2]string, 0, len(picked))
	for _, p := range picked {
		out = append(out, [2]string{p.src, p.dst})
	}
	return out
}

// CanonConflict is one consolidated rendering that contradicts a row the owner already signed.
type CanonConflict struct {
	Src      string // the candidate's source surface
	Dst      string // what the role consolidated it to
	CanonSrc string // the SIGNED source contained in it
	CanonDst string // the signed rendering its own dst fails to carry
}

// CanonConflicts is the deterministic half of canon consistency, and it exists because the anchor is not
// one. The live probe of 26.07 showed both halves: with the signed rows on the wire the role fixed 元海空窍
// («апертура моря истинной ци», carrying both signed elements), and in the same reply it still rendered
// 一代族长 as «Первый глава рода» while 族长 is signed «глава клана». A prompt-level anchor is a nudge; a
// paid step that can contradict the owner's signature with NO signal is the silent-degradation class this
// codebase refuses to ship blind (the post-check's whole rationale, mempostcheck.go).
//
// So this REPORTS, and only reports: for every consolidated rendering whose SOURCE strictly contains a
// signed source, the rendering must carry a word of that signed rendering (lexeme-wise, so Russian case
// endings do not false-flag). It never rewrites a rendering and never fails a run — forcing a dictionary
// form into a compound is exactly the post-REPLACE mistake E2 forbids. Deterministic: candidates are
// key-ordered, neighbours are sorted by the caller, and nothing here iterates a map for output.
func CanonConflicts(cands []Candidate, consolidated map[string]string, ns []Neighbour) []CanonConflict {
	if len(consolidated) == 0 || len(ns) == 0 {
		return nil
	}
	var out []CanonConflict
	for _, c := range cands {
		dst := consolidated[c.Key]
		if dst == "" || c.Key == "" {
			continue
		}
		have := wordSet(dst)
		for _, n := range ns {
			if n.Src == "" || n.Dst == "" || n.Src == c.Key || !strings.Contains(c.Key, n.Src) {
				continue
			}
			// EVERY significant word of the signed rendering must survive, not merely one: «Первый глава
			// рода» shares «глава» with the signed «глава клана» and is still the contradiction — the
			// disagreement lives precisely in the word that was dropped.
			if lexemeSubset(wordSet(n.Dst), have) {
				continue
			}
			out = append(out, CanonConflict{Src: c.Src, Dst: dst, CanonSrc: n.Src, CanonDst: n.Dst})
		}
	}
	return out
}

// ConsolidationConflict is one consolidated rendering that contradicts ANOTHER consolidation of the SAME
// run — the compositional rule broken inside one reply rather than against a signed row.
type ConsolidationConflict struct {
	// Key is the candidate KEY the finding belongs to — the normalized surface the check compared on, and
	// the only string a consumer may match it back by. Src is the raw one and the two differ for every
	// traditional or katakana spelling, so matching on Src loses exactly the candidates the fold exists
	// for. Carried rather than re-derived: a consumer deriving it again would be the second place the key
	// is chosen, which is how this pair drifted apart unnoticed for three packs.
	Key     string
	Src     string // the containing candidate's source surface, as a human reads it
	Dst     string // what the role consolidated it to
	PartSrc string // the source surface contained in it, also consolidated this run
	PartDst string // the rendering its own dst fails to carry
}

// PartLabel is the one rendering of the CONTRADICTED part every human-facing surface uses — the sheet
// column and the sidecar table — so the finding cannot be phrased two ways by two callers. Same
// discipline as membank.BankKeyConflict.BankRowLabel on the other half of the sheet.
func (c ConsolidationConflict) PartLabel() string {
	return fmt.Sprintf("%s→%q", c.PartSrc, c.PartDst)
}

// ConsolidationConflictMessages renders the findings for the run log, where the reader needs BOTH sides:
// which rendering dropped which. Returned as parts by ConsolidationConflicts and assembled here for the
// same reason the bank side does it (membank.ConflictMessages) — a caller free to re-phrase would state
// one finding two ways.
func ConsolidationConflictMessages(cs []ConsolidationConflict) []string {
	out := make([]string, 0, len(cs))
	for _, c := range cs {
		out = append(out, fmt.Sprintf("%s→%q drops %s", c.Src, c.Dst, c.PartLabel()))
	}
	return out
}

// ConsolidationConflicts is CanonConflicts' thin brother, and it exists because the canon check can only see
// what the owner already signed. On a live bank that is the minority: 18 of 149 contradictions the same run
// produced were between its OWN consolidations (research/24 §A4) — 元海空窍 rendered without the 元海 this
// very reply had just fixed — and nothing looked at them, because the role runs once and never reads itself.
// A step whose entire purpose is ONE consistent canon must not be the step that quietly breaks it.
//
// Same test as the canon side (containment + lexemeSubset, so Russian case endings do not false-flag), same
// discipline: it REPORTS, never rewrites and never fails a run. $0 — it is arithmetic over a map that
// already exists. Deterministic: candidates are key-ordered on both sides and nothing here iterates a map.
func ConsolidationConflicts(cands []Candidate, consolidated map[string]string) []ConsolidationConflict {
	if len(consolidated) < 2 {
		return nil
	}
	var out []ConsolidationConflict
	for _, c := range cands {
		dst := consolidated[c.Key]
		if dst == "" || c.Key == "" {
			continue
		}
		have := wordSet(dst)
		for _, p := range cands {
			pd := consolidated[p.Key]
			if pd == "" || p.Key == "" || p.Key == c.Key || !strings.Contains(c.Key, p.Key) {
				continue
			}
			if lexemeSubset(wordSet(pd), have) {
				continue
			}
			out = append(out, ConsolidationConflict{Key: c.Key, Src: c.Src, Dst: dst, PartSrc: p.Src, PartDst: pd})
		}
	}
	return out
}

// sharedRunes counts the DISTINCT runes two sources have in common.
func sharedRunes(a, b string) int {
	if a == "" || b == "" {
		return 0
	}
	set := map[rune]bool{}
	for _, r := range a {
		set[r] = true
	}
	seen := map[rune]bool{}
	n := 0
	for _, r := range b {
		if set[r] && !seen[r] {
			seen[r] = true
			n++
		}
	}
	return n
}

// ReplyStats is what one reply cost in unusable lines. Bad is the TOTAL refused; OffLanguage is the
// subset refused by the answer-language screen, kept apart because the two mean different things to an
// operator — a broken format against a run producing canon in the wrong language.
type ReplyStats struct {
	Bad                int
	OffLanguage        int
	OffLanguageSamples []string // first few refused lines verbatim, so the warning names them
	// BadConfidence counts lines whose declared CONFIDENCE column was not a bare 0…100 («95%», «0.9»,
	// «высокая»). The rendering is kept — it is fine — but the value is dropped rather than glued onto it,
	// and the count is what tells an operator the prompt's third column is not landing.
	BadConfidence int
	// BadGender / NoGender are the CLASSIFIER's third column, and they are two counters because they are
	// two different failures. BadGender is a value outside the closed vocabulary — the class is kept, the
	// gender is dropped rather than written, because an unknown value would reach the bank and the seed
	// loader refuses the whole file over one (memseed.GenderVocabViolations). NoGender is a line that
	// answered the class and omitted the column entirely: legitimate-looking, and the only signal that a
	// model has quietly stopped answering the question the pair's prompt asks.
	BadGender int
	NoGender  int
	// DeclinedByPhrase counts the replies that declined a term IN WORDS rather than with the engine
	// sentinel — «не термин» where ⟦TM-NO-DST⟧ was asked for. They join the sentinel's bucket (an explicit
	// empty rendering), and the count is kept apart because the two are one DECISION arriving in two
	// shapes: a rising number here is the pair's prompt losing its format, not the model losing its nerve.
	DeclinedByPhrase int
	// NoLetters counts renderings with NO LETTER IN ANY SCRIPT that got this far — «90», «——», «№1». A
	// ONE-rune answer («…», «-») never reaches this counter: wellFormedLemma refuses it a check earlier on
	// length, so it is a bad line without being a letterless one. They are
	// refused as bad lines, so the term reads as UNANSWERED downstream, which is what it is: the role was
	// asked and produced nothing a bank can hold. Separate from OffLanguage by construction — that screen
	// judges WHICH script the letters are in and is silent when there are none (see OffLanguage).
	NoLetters int
	// NoLettersSamples are the first few such lines verbatim: a count alone cannot tell a budget-truncated
	// reply from a model answering in numbers, and the population of this class on bought material is zero
	// so far (research/35 §2.0, row Б-5: 0 of 69 and 0 of 66) — the first live one is worth reading.
	NoLettersSamples []string
}

const offLanguageSampleCap = 8

// ParseReply turns the terminologist's reply into key → consolidated rendering. Lines are
// `<src><TAB><dst>`; the src is matched through `normalize` against the batch's EXPECTED keys, so a line
// for a term that was not asked about is ignored (a model inventing rows must not inject terms into the
// bank) and a rendering written for a differently-spelled src still lands. NoDst yields an explicit empty
// rendering — the two-mode emission's "no dst" branch, which is a decision, not a failure.
//
// `target` is the target language's script (nil → the answer-language check is inert; see OffLanguage).
//
// `declined` recognises a decline written in WORDS — the target-language half of the sentinel, carried as
// data (lang.DeclinePhrasesFor) and nil-inert here, which is also its shipping state. A prose decline lands
// in the SAME bucket as the sentinel rather than in the bank: what the role said is «I will not render
// this», and banking those words makes the editor obey them as this book's canon.
//
// Returns the accepted map, the role's own stated CONFIDENCE per key (absent when the reply carried none),
// and the tally of unusable lines, which the caller logs: silence about a reply the parser could not read is
// how a paid call turns into an empty bank with no signal.
func ParseReply(reply string, expected []string, normalize func(string) string, target *unicode.RangeTable, declined func(string) bool) (map[string]string, map[string]int, ReplyStats) {
	want := make(map[string]bool, len(expected))
	for _, k := range expected {
		want[k] = true
	}
	out := map[string]string{}
	conf := map[string]int{}
	var st ReplyStats
	for _, ln := range strings.Split(reply, "\n") {
		if strings.TrimSpace(ln) == "" || strings.HasPrefix(strings.TrimSpace(ln), "#") {
			continue
		}
		rawSrc, dst, c, badConf, ok := replyColumns(ln)
		if !ok {
			st.Bad++
			continue
		}
		key := normalize(rawSrc)
		if !want[key] {
			st.Bad++
			continue
		}
		if _, dup := out[key]; dup {
			continue // first answer wins; a model repeating itself is not a second opinion
		}
		if badConf {
			// The rendering is usable; the confidence column is not. Counted rather than refused — and never
			// folded into the rendering, which is the whole point.
			st.BadConfidence++
		}
		// Everything after the FIRST field is the rendering, re-joined on single spaces. The splitter is
		// deliberately tolerant (a tab, a run of ≥2 spaces, a padded pipe) because models substitute one for
		// another — but that tolerance also splits INSIDE a rendering the moment a model writes «Фан  Юань»
		// with a stray double space, and taking f[1] alone would then bank the half-name «Фан» silently:
		// it passes wellFormedLemma, so no counter would ever mention it. The role's reply is specified as
		// exactly two fields, so there is no third column to lose by re-joining — see replyColumns for how the
		// third (confidence) column and a mis-split rendering are told apart.
		if isNoDst(dst) {
			out[key] = ""
			if c >= 0 {
				conf[key] = c
			}
			continue
		}
		// The SAME decision in the other shape. Without this branch «не термин» passes every check below —
		// it is well formed, it is not an echo, it is in the target's own script — and is banked as the
		// book's canon, reaching the editor as law byte-identically to a rendering the owner signed (that
		// is measured, not feared: K1 executed 家族 → «не термин» and 族长 → «90», research/35 §П3).
		// Inert until a phrase is authored, and the shipping registry is empty.
		if declined != nil && declined(dst) {
			out[key] = ""
			st.DeclinedByPhrase++
			if c >= 0 {
				conf[key] = c
			}
			continue
		}
		if !wellFormedLemma(dst) {
			st.Bad++
			continue
		}
		// A rendering with NO LETTER AT ALL is not a rendering: «90» is two runes, carries no bracket and no
		// dash, so wellFormedLemma passes it, and the answer-language screen below is silent by construction
		// on a field with no letters to judge («a rendering with no letters at all is not judged here»). Past
		// this point it would be stamped status:draft and injected as canon.
		//
		// ⚠ DIGITS ARE NOT THE TEST — their ABSENCE of letters is. Numbers are legitimate inside a name or a
		// series («Отряд 731»), and refusing a rendering for containing one would cost the bank real rows.
		// Direction of error, declared: a target whose rendering of some term is genuinely letterless (a bare
		// symbol) loses that one proposal, which the owner can still sign by hand; the opposite mistake puts
		// «90» into a book's canon for the whole book.
		if !hasLetter(dst) {
			st.Bad++
			st.NoLetters++
			if len(st.NoLettersSamples) < offLanguageSampleCap {
				st.NoLettersSamples = append(st.NoLettersSamples, rawSrc+" → "+dst)
			}
			continue
		}
		// A rendering that IS the source is not a rendering. Models answer this under pressure (a term they
		// cannot translate echoed back), and nothing downstream would catch it: it passes wellFormedLemma,
		// it is stamped status:draft, and it reaches the editor as «方源 → 方源» inside the law block — an
		// untranslated surface presented as this book's canon, and since D39.104 п.2 nothing on the wire
		// marks it apart from a signed row. Counted as a bad line, so a reply full of echoes is loud.
		if normalize(dst) == key {
			st.Bad++
			continue
		}
		// Nor is a rendering written in another script — the same argument as the echo check above, at the
		// one place a reply becomes bank content. Past here it is indistinguishable from a correct answer.
		if OffLanguage(dst, target) {
			st.Bad++
			st.OffLanguage++
			if len(st.OffLanguageSamples) < offLanguageSampleCap {
				st.OffLanguageSamples = append(st.OffLanguageSamples, rawSrc+" → "+dst)
			}
			continue
		}
		out[key] = dst
		if c >= 0 {
			conf[key] = c
		}
	}
	return out, conf, st
}

// hasLetter reports whether a string carries at least one letter in ANY script. The weakest question that
// separates «the role answered something» from «the role answered nothing a bank can hold», and it is
// script-blind on purpose: which script the letters belong to is a different screen with a different
// verdict (OffLanguage), and merging the two would make one counter answer two questions.
func hasLetter(s string) bool {
	for _, r := range s {
		if unicode.IsLetter(r) {
			return true
		}
	}
	return false
}

// confidenceField reads the role's stated confidence off a reply FIELD: 1–3 digits in 0…100 and nothing
// else. Out of that range it is not a confidence and stays part of the rendering.
//
// ⚠ What this number may and may not do is RATIFIED (D39.102): it orders the review list «least sure
// first» — an ordinal INSIDE one model's reply — and nothing more. It is not a weight, not a threshold, not
// comparable between models, and never decides which rendering wins; the measured AUC (0.77–0.85) supports
// exactly the ordering claim and nothing stronger.
func confidenceField(s string) (int, bool) {
	s = strings.TrimSpace(s)
	if len(s) == 0 || len(s) > 3 {
		return 0, false
	}
	n := 0
	for _, r := range s {
		if r < '0' || r > '9' {
			return 0, false
		}
		n = n*10 + int(r-'0')
	}
	if n > 100 {
		return 0, false
	}
	return n, true
}

// isNoDst recognises the decline sentinel, including the forms a model mangles it into — a trailing period,
// surrounding quotes. Narrow on purpose: the bracketed engine token must still OPEN the rendering and only
// punctuation may follow, so no natural rendering can match. Without it a decline written «⟦TM-NO-DST⟧.»
// fails wellFormedLemma-then-bad-line and the term is counted as a PARSE failure rather than as the
// decision §C2-7 says it is.
func isNoDst(dst string) bool {
	t := strings.Trim(strings.TrimSpace(dst), `"'«»“”`)
	if !strings.HasPrefix(t, NoDst) {
		return false
	}
	return strings.Trim(strings.TrimSpace(t[len(NoDst):]), `.,;:!?"'«»“”`) == ""
}

// Batch splits candidates into groups whose rendered size stays under maxRunes. A batch UNIT — a series or
// a family, i.e. a key present in unitID; pass nil to disable both channels — is kept WHOLE in one batch so
// the terminologist picks one generic head or one shared root for it (§1, §G1); a unit or a single
// candidate larger than the budget still gets its own batch
// rather than being split or silently dropped (the caller WARNs on such an over-cap unit — see BatchRunes —
// because the whole unit still bills against one output cap). The batches are then ordered by descending
// source frequency, so a budget ceiling drops the least-frequent terms instead of the lexicographic tail
// (feed_cap, §7).
//
// The ordering is DETERMINISTIC — a stable sort on a pure function of each batch's content — so the same
// input reproduces the same batch order every run and the $0 checkpoint resume holds. It is NOT, however,
// independent of that ordering: the consumer addresses batch i as ChunkIdx i (terminologist.go), which folds
// into the request hash, so a batch's POSITION is part of its address. If a later run's frequencies reorder
// the batches their ordinals shift and the affected batches re-pay — the accepted price of value-ordering,
// and no worse than the content edit that would already re-pay them.
func Batch(cands []Candidate, maxRunes int, unitID map[string]int) [][]Candidate {
	if len(cands) == 0 {
		return nil
	}
	if maxRunes <= 0 {
		return [][]Candidate{cands}
	}
	ordered := orderByUnit(cands, unitID)
	var out [][]Candidate
	var cur []Candidate
	size := 0
	flush := func() {
		if len(cur) > 0 {
			out = append(out, cur)
			cur, size = nil, 0
		}
	}
	for i := 0; i < len(ordered); {
		j := i + 1 // extend over a whole unit block; a singleton is a unit of one
		if id := unitID[ordered[i].Key]; id != 0 {
			for j < len(ordered) && unitID[ordered[j].Key] == id {
				j++
			}
		}
		unit := ordered[i:j]
		if n := unitRunes(unit); len(cur) > 0 && size+n > maxRunes {
			flush()
			cur, size = append(cur, unit...), n
		} else {
			cur, size = append(cur, unit...), size+n
		}
		i = j
	}
	flush()
	sort.SliceStable(out, func(a, b int) bool { return batchFreq(out[a]) > batchFreq(out[b]) })
	return out
}

// BatchRunes reports the rendered size RenderBatch produces for a whole batch — the same measure Batch packs
// against. The terminologist uses it to WARN when a co-batched unit or a lone candidate exceeds the budget:
// §1 keeps such a unit WHOLE rather than split it, so it is a real over-cap call the operator must see coming
// (the cap-8000 output mine, PROBES §1), not a silent overrun.
func BatchRunes(batch []Candidate) int { return unitRunes(batch) }

// unitRunes is the rendered size the accountant charges a unit: each member's single-block render plus the
// blank line RenderBatch puts between blocks (+2). Without the +2 a 200-term book overruns its budget by
// 2×(N−1) runes. Summing per member matches the original per-candidate arithmetic exactly.
func unitRunes(unit []Candidate) int {
	n := 0
	for _, c := range unit {
		n += len([]rune(RenderBatch([]Candidate{c}))) + 2
	}
	return n
}

func batchFreq(b []Candidate) int {
	n := 0
	for _, c := range b {
		n += c.Freq
	}
	return n
}