textmachine/backend/internal/membank/memory.go

// Package membank is the book's deterministic memory bank: the frozen glossary materialized into a
// matcher, the per-chunk retrieval that decides WHICH records a model sees, the injection renderers
// that turn them into wire text, and the post-check that verifies the rendering came back.
//
// The hot path is deterministic by construction — an exact multi-pattern match (Aho-Corasick) over the
// normalized chunk, no vectors, no FTS, no LLM, no clock — because the injected block is folded into
// request_hash: the same chunk under the same bank must produce the same bytes on resume, or a paid
// checkpoint is silently invalidated. Bank content is versioned (Version/BaseVersion) so a glossary
// edit is a loud --resnapshot rather than a silent change of what the model was told.
//
// It owns the seed side of that contract too (loading, validating and colliding-checking the seed
// YAML), and holds no run state: it takes rows and text, returns a selection.
package membank

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"maps"
	"slices"
	"sort"
	"strconv"
	"strings"
	"unicode"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/store"
	"textmachine/backend/internal/text"
)

// memory.go: the DETERMINISTIC hot path of the memory bank v2 (registry §Hot-path
// Contract / research/13 Q3). $0, no LLM, no embeddings, no FTS5 — a multi-pattern
// exact match (Aho-Corasick) of glossary keys/aliases over the NORMALIZED chunk, with
// the single-key ban (A3), longest-match whole-entity replacement, the spoiler
// hard-reject window (C1), sticky scene-inertia (A5), a priority token budget with a
// logged eviction (F2), and the three-way injection disposition (A2). Pure and
// deterministic (no map-order output, no time/rand): the injected block is a pure
// function of (frozen bank, normalized chunk, chapter, sticky, budget), and it enters
// request_hash via the rendered messages — so a resumed chunk reproduces it for free.
//
// The registry's principle drives every choice here: "better nothing than garbage" and
// PRECISION over recall. A wrong injection (a homograph / short alias firing on the
// wrong sense) is the MAIN source of silent degradation (2510.00829) — worse than an
// empty match, which is a safe fallback to the model's base behaviour. So the matcher
// errs toward NOT firing, and the post-check (memcheck) makes what did fire observable.

// matchVersion versions the matcher + injection-selection ALGORITHM (min-key
// ban, longest-match containment, spoiler gate, sticky, budget priority order). Folded
// into memoryVersion() → a change is a loud --resnapshot (it shifts the injected bytes
// → the wire → a resumed checkpoint's content). Sibling of text.NormVersion().
// v3 (D16.2/D16.3): sticky now INHERITS the original match's disposition instead of
// recomputing it status-based (a collision-prone AMBIGUOUS carry no longer silently
// upgrades to CONFIRMED), and a spaced-phonetic (Latin/Cyrillic) source key is validated
// against a word boundary (approved "rose" no longer fires inside "roseanne").
// v4 (D39 layer 4): the longest-match suppressor is now DISPOSITION-GATED (suppressContained) —
// a longer but LOWER-trust key (draft/ambiguous) no longer deletes a nested HIGHER-trust key
// (approved/confirmed), closing the term-drift code root the D38.5 seed-promote only patched by
// data (L3-suppressor-disposition-blind). A refused suppression is recorded loudly (trustGated).
// The injection SELECTION changes for a draft-longer-over-approved-shorter chunk (the approved term
// now survives), so a loud --resnapshot + golden re-capture (invariant №8).
const matchVersion = "memmatch-v4-perlang-minkey+collision-ambiguous+longest-trustgated+spoiler+sticky-inherit-disp+phonetic-srcboundary+budget+postcheck-declaware"

// minKeyLenHan / minKeyLenPhonetic are the per-language single-key floors (A3, registry
// "min_key_len per-language"). An ideographic (Han) key is semantically distinct at 2 chars
// (empirically precision-1.0 on the zh retrieval_bench homograph traps). A PURELY PHONETIC
// key (kana/latin/cyrillic, no Han anchor) is collision-prone at 2 — short phonetic
// sequences appear INSIDE ordinary words (リン in リンゴ/apple, AI in RAID) — so it needs ≥3.
// external-review major #3: the MIN=2 measurement was on zh-Han and does NOT transfer to
// ja-kana; 3 is a conservative default pending a ja-kana precision measurement (an E1-
// protocol extension). A per-entry allow_short overrides both (rare, guarded).
const (
	minKeyLenHan      = 2
	minKeyLenPhonetic = 3
)

// minKeyLenFor is the floor for a normalized key: any Han ideograph anchors it (2), an
// all-phonetic key needs 3.
func minKeyLenFor(normKey string) int {
	if runesAnyHan(normKey) {
		return minKeyLenHan
	}
	return minKeyLenPhonetic
}

// collisionProneKey reports whether a key is short AND purely phonetic (no Han anchor),
// so it is likely to fire inside an unrelated word — the A2 "surface collision" case,
// injected as AMBIGUOUS (unverified) rather than authoritatively CONFIRMED.
func collisionProneKey(normKey string) bool {
	return !runesAnyHan(normKey) && text.SignificantLen(normKey) <= minKeyLenPhonetic
}

// UnionSticky merges the recent chunks' exact-matched ids into one sticky_prev, carrying
// each id's DISPOSITION (D16.2). When an id fired in several recent chunks, the MOST RECENT
// firing wins (win is ordered oldest→newest), so a re-established CONFIRMED match overrides
// an older collision-prone AMBIGUOUS one — and vice-versa, never silently upgrading.
func UnionSticky(win []map[string]InjectionDisposition) map[string]InjectionDisposition {
	if len(win) == 0 {
		return nil
	}
	out := map[string]InjectionDisposition{}
	for _, s := range win {
		for id, disp := range s {
			out[id] = disp
		}
	}
	return out
}

// StickyDepth is the scene-inertia window (A5): entries exact-matched in the previous
// N chunks of the SAME chapter are carried into a pronominal chunk that names nobody.
// A code rule (Р2), reset at each chapter boundary (a new chapter is a scene change).
const StickyDepth = 2

// InjectionDisposition is the three-way per-record decision (registry A2) — the core
// mechanism that converts silent degradation into loud. Distinct from the chunk×stage
// Disposition (disposition.go): that is ok|flagged|skipped for a completion; this is
// how much to TRUST an injected glossary record.
type InjectionDisposition string

const (
	Confirmed InjectionDisposition = "confirmed" // exact key/alias, approved, spoiler-valid → authoritative
	Ambiguous InjectionDisposition = "ambiguous" // auto/draft term → inject "unverified" + forced post-check
	Reject    InjectionDisposition = "reject"    // spoiler-window violation → dropped and logged (C1 safety)
)

// declInfo is the parsed decl column: the dst declension forms the post-check accepts.
type declInfo struct {
	Invariant bool     `json:"invariant"`
	Forms     []string `json:"forms"`
}

// entry is one frozen, materialized glossary entry with its matchable keys
// pre-normalized. Immutable for a job.
type entry struct {
	id      string // stable unique id: src\x1f sense\x1f since\x1f until (= the UNIQUE key)
	src     string // raw source key
	dst     string // raw approved translation (may be "" for a ruby candidate)
	status  string // auto|draft|approved
	sense   string
	gender  string // male|female|hidden|"" (C2) — feeds the DC3 gender constraint on the editor block
	sinceCh int
	untilCh int
	// normKeys are the normalized source surfaces (src + aliases) ELIGIBLE to fire
	// (text.SignificantLen ≥ minKeyLen OR allow_short). A key too short is dropped here, so
	// it is never in the automaton — the single-key ban is structural, not a runtime skip.
	normKeys []string
	// declForms are the normalized-target dst forms the post-check accepts. Empty →
	// the post-check falls back to the single normalized dst (the naive base form that
	// research/14 shows false-flags on inflection — the reason full decl matters).
	declForms     []string
	declInvariant bool
	// allowShort is the author's explicit override of the single-key ban AND the
	// collision-prone disposition downgrade (full trust in a short key).
	allowShort bool
}

// Bank is a book's frozen glossary materialized for one job: the entries, the
// Aho-Corasick automaton over all eligible keys, and the F1 memoryVersion. Built once
// (Materialize) before the chunk loop and never mutated.
type Bank struct {
	entries   []entry
	ac        *ahoCorasick
	keyOwners map[string][]int // normalized key → indices of entries that contributed it
	// enrichedVersion folds ALL foldable rows incl Source:mined — the memory version of the EDIT
	// wave (edit-wave snapshot). baseVersion EXCLUDES Source:mined — the memory version of the DRAFT wave
	// (draft-wave snapshot), so a the bank-mining stop bank-enrichment (adding mined rows) moves ONLY edit-wave snapshot, keeping
	// draft-wave checkpoints valid (WS1 §1в, «re-paid ONCE»). The two are DOMAIN-SEPARATED (a "base-excl-
	// mined" tag in the base fold), so base ≠ enriched for EVERY row set — even a mined-free book (a
	// the draft wave job must never content-address to a the edit wave checkpoint). The load-bearing invariant is not
	// equality but STABILITY: base stays fixed as mined rows are added; only enriched moves.
	enrichedVersion string
	baseVersion     string
	// voices/pairs are the pack-19 record types. They are bank CONTENT but never matchable SURFACES, so
	// they sit beside the automaton rather than inside it: nothing here can put them in keyOwners.
	voices []store.VoiceProfile
	pairs  []store.AddressPair
	// stemmer is the target-language decl stemmer (bank-quality §3): it lets the post-check accept an oblique
	// case of a term against its nominative without the seed listing every form. The zero value is inert (a
	// target with no decl_suffix registry), so a book that ships none keeps exact-match behaviour.
	stemmer lang.TargetStemmer
}

// PickedEntry is one selected record for a chunk with its firing key and disposition.
type PickedEntry struct {
	entry *entry // the bank row (opaque outside the bank)
	// Via is the normalized key this record fired on. It is EMPTY for a sticky carry — see Sticky,
	// which is the fact callers actually branch on.
	Via string
	// Sticky marks a scene-inertia carry (A5): the record did NOT fire in this chunk, it was carried
	// from the sticky window. It is a typed field rather than the old via=="sticky" sentinel, which was
	// indistinguishable from a real key literally spelled "sticky" and made every consumer re-know the
	// magic string (the post-check skips sticky records, the retrieval-state counts them, the budget
	// ranks them last).
	Sticky bool
	Disp   InjectionDisposition // the trust the record was injected with
}

// valid reports whether this record actually carries a bank row. PickedEntry is exported WITH exported
// fields, so a caller outside this package can construct a zero value whose private row is nil; every
// consumer here would then nil-deref. Rather than document a landmine, the consumers skip such a record:
// a record with no row has nothing to inject and nothing to post-check. Selection always produces rows, so
// no real path changes (pack-16 tail).
func (p PickedEntry) valid() bool { return p.entry != nil }

// Src is the bank row's SOURCE surface — the only thing about a record a caller outside the bank can NAME.
// It exists because n_evicted counted budget drops without ever saying WHICH rows the model did not get to
// see, and a bare count is not something an operator can act on. Empty for a zero value.
func (p PickedEntry) Src() string {
	if p.entry == nil {
		return ""
	}
	return p.entry.src
}

// TrustGateEvent records a longest-match suppression the DISPOSITION-gate REFUSED (D39 layer 4):
// a longer but LOWER-trust key (draft/ambiguous, e.g. draft 四代族长) that would have deleted a nested
// HIGHER-trust key (approved/confirmed, e.g. approved 族长). The draft is editor-excluded (the block
// is CONFIRMED-only), so suppressing the approved term would have left the reader with NOTHING — the
// term-drift code root (L3). The nested higher-trust match survives and injects; this record turns
// the near-silent drop into a loud, operator-visible signal (research/13 §7). A non-empty list means
// the seed has a draft term nesting over an approved one — reconcile the seed so it stops recurring.
type TrustGateEvent struct {
	Suppressor     string `json:"suppressor"`      // the longer, lower-trust key blocked from suppressing
	SuppressorDisp string `json:"suppressor_disp"` // its best injection disposition (ambiguous)
	Protected      string `json:"protected"`       // the nested, higher-trust key it would have eaten
	ProtectedDisp  string `json:"protected_disp"`  // its best injection disposition (confirmed)
}

// Selection is the hot path's output for one chunk.
type Selection struct {
	Injected []PickedEntry // in budget priority order (what the model sees)
	Rejected []PickedEntry // spoiler-window rejects (C1, logged)
	Evicted  []PickedEntry // dropped by the token budget (F2, logged)
	// trustGated are the longest-match suppressions the disposition-gate refused (a lower-trust
	// longer key would have eaten a higher-trust nested one) — the term-drift code root, surfaced
	// loudly instead of a silent drop (research/13 §7). Deterministic (ms order, deduped by pair).
	TrustGated []TrustGateEvent
	// activeIDs are the exact-matched ids (NOT sticky) → the next chunk's sticky_prev,
	// each mapped to the DISPOSITION it fired with. Sticky carries this disposition
	// forward instead of recomputing it (D16.2): a sticky carry has no firing key, so it
	// cannot re-derive the collision-prone downgrade, and recomputing from status alone
	// would silently upgrade a collision-prone AMBIGUOUS match to CONFIRMED.
	ActiveIDs map[string]InjectionDisposition
}

// Version is the ENRICHED memory version (all approved rows incl mined) — the edit-wave / whole-book
// memory component. BaseVersion excludes Source:mined (the draft-wave component). They are
// DOMAIN-SEPARATED — base ≠ enriched for ANY row set, including a mined-free book (never assert
// equality) — but base stays STABLE when a the bank-mining stop pass adds mined rows, which is what keeps the draft wave
// checkpoints valid («re-paid ONCE»).
func (b *Bank) Version() string     { return b.enrichedVersion }
func (b *Bank) BaseVersion() string { return b.baseVersion }

// BankInput is everything a book's bank is materialized and hashed from: the glossary rows plus the two
// pack-19 record types. It exists so the fold has ONE argument that can grow, and so the separation that
// matters is carried by the type system: only Rows ever reaches the matcher, so a voice profile cannot
// become a matchable term by anyone forgetting a filter.
type BankInput struct {
	Rows   []store.GlossaryEntry
	Voices []store.VoiceProfile
	Pairs  []store.AddressPair
	// InjectVoice reports whether the voice/address rows reach the WIRE on this run. It is the CONDITION
	// of their fold: while they only feed the $0 flagger (which recomputes every run and self-heals),
	// hashing them would re-bill a book for authoring a profile that changed no byte the model sees.
	// The moment they are injected, they must fold — the D39.42 п.3 class, in the other direction.
	InjectVoice bool
	// TargetStemmer is the decl-aware post-check stemmer (bank-quality §3). Zero value → inert (exact match
	// only), so a caller that supplies none keeps the pre-§3 behaviour byte-for-byte.
	TargetStemmer lang.TargetStemmer
}

// Materialize builds a Bank from the book's stored glossary rows alone — the pre-pack-19 form, kept for
// every caller that has no voice/address content.
func Materialize(rows []store.GlossaryEntry, gateOn bool) *Bank {
	return MaterializeBank(BankInput{Rows: rows}, gateOn)
}

// MaterializeBank builds a Bank from the book's stored bank content (ORDER BY-stable — GlossaryForBook /
// VoiceProfilesForBook / AddressPairsForBook). Pure and deterministic. It computes memoryVersion as a
// content hash of the frozen APPROVED rows (D8: content-hash, not a version-counter —
// drift-proof) plus the normalization + matcher algorithm versions, so ANY change to
// the approved glossary OR to the deterministic machinery is a loud --resnapshot (F1).
func MaterializeBank(in BankInput, gateOn bool) *Bank {
	rows := in.Rows
	b := &Bank{keyOwners: map[string][]int{}, voices: in.Voices, pairs: in.Pairs, stemmer: in.TargetStemmer}
	var allKeys []string
	seenKey := map[string]bool{}

	for _, row := range rows {
		e := entry{
			id:         row.Src + "\x1f" + row.Sense + "\x1f" + strconv.Itoa(row.SinceCh) + "\x1f" + strconv.Itoa(row.UntilCh),
			src:        row.Src,
			dst:        row.Dst,
			status:     row.Status,
			sense:      row.Sense,
			gender:     row.Gender,
			sinceCh:    row.SinceCh,
			untilCh:    row.UntilCh,
			allowShort: row.AllowShort,
		}
		// Eligible source surfaces (src + aliases) under the single-key ban. A record
		// with NO dst yet (a ruby auto-candidate) is NOT matchable: it renders nothing
		// (RenderGlossaryBlock/postcheck both skip empty dst), so admitting its keys would
		// let it consume the token budget, evict a renderable line, and inflate the
		// retrieval-state exact-hit count for a record the model never sees (self-review
		// #1). It stays in the store for future promotion, just inert on the hot path.
		var surfaces []string
		if strings.TrimSpace(row.Dst) != "" {
			surfaces = append(surfaces, row.Src)
			for _, a := range row.Aliases {
				surfaces = append(surfaces, a.Alias)
			}
		}
		for _, s := range surfaces {
			nk := text.NormalizeSourceKey(s)
			if nk == "" {
				continue
			}
			if text.SignificantLen(nk) < minKeyLenFor(nk) && !row.AllowShort {
				continue // A3: a too-short key never fires on its own (per-language floor)
			}
			e.normKeys = append(e.normKeys, nk)
		}
		// Parse decl forms for the post-check (normalized target side).
		if row.Decl != "" {
			var d declInfo
			if err := json.Unmarshal([]byte(row.Decl), &d); err == nil {
				e.declInvariant = d.Invariant
				for _, f := range d.Forms {
					if nf := text.NormalizeTargetForm(f); nf != "" {
						e.declForms = append(e.declForms, nf)
					}
				}
			}
		}
		idx := len(b.entries)
		b.entries = append(b.entries, e)
		for _, nk := range e.normKeys {
			b.keyOwners[nk] = append(b.keyOwners[nk], idx)
			if !seenKey[nk] {
				seenKey[nk] = true
				allKeys = append(allKeys, nk)
			}
		}
	}
	sort.Strings(allKeys) // deterministic automaton construction
	b.ac = buildAC(allKeys)
	b.enrichedVersion = ComputeVersionScopedIn(in, gateOn, false) // all approved incl mined (the edit wave)
	b.baseVersion = ComputeVersionScopedIn(in, gateOn, true)      // excl Source:mined (the draft wave)
	return b
}

// Voices returns the book's voice profiles (frozen for the job). Read-only: callers project them onto a
// chapter, never mutate them.
func (b *Bank) Voices() []store.VoiceProfile { return b.voices }

// Pairs returns the book's address-register journal entries (frozen for the job).
func (b *Bank) Pairs() []store.AddressPair { return b.pairs }

// ComputeVersion is the F1 content-hash: the frozen rows (ORDER BY-stable) plus the
// normalization and matcher algorithm versions. EVERY row folds whatever its status, and
// its whole content (incl. decl and status) with it — see the loop below for why the
// approved-only fold of D8/§8 was retired by pack-20 (D39.42 п.3). gateOn is folded as a
// FIELD rather than as a scope switch: it changes how the same rows resolve a chunk's
// disposition, so the two gate settings must not share a hash. The glossary.id
// autoincrement is deliberately EXCLUDED (fresh each replace); only content columns are hashed.
func ComputeVersion(rows []store.GlossaryEntry, gateOn bool) string {
	return ComputeVersionScoped(rows, gateOn, false)
}

// ComputeVersionScoped is ComputeVersion with an explicit Source-scope: excludeMined
// drops every Source:"mined" row from the fold (WS1 §1в base-bank-version). The DRAFT-wave snapshot
// (draft-wave snapshot) folds base (excludeMined=true) so a the bank-mining stop pass that ADDS mined rows moves ONLY the
// enriched (edit-wave) version — keeping draft-wave checkpoints valid, «re-paid ONCE». Deterministic (a
// Source filter over the same ORDER BY-stable rows). excludeMined=false is BYTE-IDENTICAL to the
// pre-split fold (no extra field), so the enriched hash / existing snapshots do not move; the
// excludeMined=true variant adds a domain separator so base and enriched never collide.
func ComputeVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool) string {
	return ComputeVersionScopedIn(BankInput{Rows: rows}, gateOn, excludeMined)
}

// ComputeVersionScopedIn is ComputeVersionScoped over the whole bank input — the form that also folds the
// pack-19 voice/address rows, CONDITIONALLY (BankInput.InjectVoice).
//
// The condition is the money contract of pack-19. While those rows only feed the $0 flagger they change
// no byte the model sees, and a book must not re-pay a wave for authoring a profile; the flagger's
// counters live in retrieval_state, which is recomputed every run and self-heals. The moment they are
// injected they DO change the wire, and then not folding them would reopen exactly the class D39.42 п.3
// closed — a wire change the snapshot cannot see, so projectRebill projects $0 for a real re-payment.
// A book with no voice rows, or a run with the injection off, hashes BYTE-IDENTICALLY to before this
// existed: nothing is written at all.
func ComputeVersionScopedIn(in BankInput, gateOn, excludeMined bool) string {
	rows := in.Rows
	h := sha256.New()
	h.Write([]byte("tm-memory-v2\x00"))
	h.Write([]byte(text.NormVersion() + "\x00" + matchVersion + "\x00"))
	h.Write([]byte("gate:" + strconv.FormatBool(gateOn) + "\x00"))
	hasUnverified := false
	hasNeuter := false // a neuter row now RENDERS a directive (bank-quality §3) where it rendered nothing before
	// inScope collects the characters this fold's ROW scope actually contains, so the voice/address fold
	// below can apply the SAME scope before deciding anything (see the loop at the end).
	inScope := map[[2]string]bool{}
	if excludeMined {
		h.Write([]byte("base-excl-mined\x00")) // domain separator: base ≠ enriched even over identical rows
	}
	for _, r := range rows {
		// EVERY row folds, whatever its status (pack-20 / D39.42 п.3). Before pack-20 an auto/draft row
		// was skipped unless the hard gate was on, on the reasoning that its injected content was caught
		// per chunk by content_hash. That reasoning had a hole the phase-1 sync measured: an unverified
		// row changes the rendered injection, hence content_hash, hence the resume fast-path — but NOT the
		// snapshot, and projectRebill projects re-payment from the SNAPSHOT alone, so the chunks it
		// silently re-translated never reached the Р6 consent contour. Folding every row closes that
		// class by construction rather than by a second guard: a bank edit of ANY status is now a loud
		// --resnapshot. A book whose bank is entirely approved hashes byte-identically to before, so
		// shipping this re-bills nobody who had nothing unverified.
		//
		if excludeMined && r.Source == "mined" {
			continue
		}
		if r.Status != "approved" {
			hasUnverified = true
		}
		if r.Gender == "neuter" || r.Gender == "n" {
			hasNeuter = true
		}
		inScope[[2]string{r.Src, r.Sense}] = true
		// A fixed, length-prefixed field layout so no content can forge a boundary.
		writeField(h, r.Status)
		writeField(h, r.Src)
		writeField(h, r.Dst)
		writeField(h, r.Sense)
		writeField(h, r.Type)
		writeField(h, r.Gender)
		writeField(h, r.Decl)
		writeField(h, strconv.Itoa(r.SinceCh))
		writeField(h, strconv.Itoa(r.UntilCh))
		writeField(h, strconv.FormatBool(r.AllowShort))
		writeField(h, strconv.Itoa(len(r.Aliases)))
		for _, a := range r.Aliases { // GlossaryForBook returns aliases ORDER BY alias
			writeField(h, a.Alias)
			writeField(h, a.AliasType)
		}
	}
	// The RENDER revision of the unverified section, folded only for a scope that actually HAS a
	// non-Confirmed row. Folding every row (above) closes the "which rows exist" hole, but not this one:
	// pack-20 changed how those rows are RENDERED to the editor (their own labelled section), and for a
	// book whose post-check gate was already ON the identical row set hashes identically — same version,
	// same snapshot, different wire bytes. The resume path would then re-translate on the content hash
	// while projectRebill, which skips a unit whose snapshot is unchanged (rebill.go), projects $0: a
	// silent re-payment outside the Р6 consent contour, the exact class folding every row was meant to
	// close. A book with nothing unverified renders no such section and hashes byte-identically to before,
	// so this re-bills nobody who had nothing to re-render — which a blanket RenderFormatVersion bump,
	// being un-re-pinnable (it is not a bank-only move), would have done to every book on earth.
	if hasUnverified {
		h.Write([]byte("editor-unverified-section-v1\x00"))
	}
	// The same scoped-render pattern for the §3 neuter directive: gender:neuter was a SILENT no-op before,
	// so a book carrying one renders new wire bytes on the same row set — the editor-unverified hole again.
	// Folded ONLY for a scope that HAS a neuter row, so a bank without one is byte-identical to before and
	// re-bills nobody; a neuter-bearing book takes a loud, RE-PINNABLE --resnapshot (bank-only move).
	if hasNeuter {
		h.Write([]byte("neuter-directive-v1\x00"))
	}
	// The pack-19 record types, appended AFTER everything above so a book without them is byte-identical
	// to the pre-pack-19 hash by construction, not by argument.
	//
	// The two loops copy the order the row loop above uses and for the same reason: the SCOPE exclusion is
	// applied FIRST, and only a row that survived it may set the tag condition. A profile whose character
	// is not in this scope (its term is Source:mined and this is the base/draft fold) describes somebody
	// the wave never sees — folding it would move the draft version when a mined-only character's profile
	// is edited, which is precisely the base/enriched separation the mined filter exists to keep.
	hasVoice := false
	for _, v := range in.Voices {
		if !in.InjectVoice || !inScope[[2]string{v.Src, v.Sense}] {
			continue
		}
		hasVoice = true
		for _, f := range []string{v.Src, v.Sense, v.Register, v.SelfRef, v.AddressDefault,
			v.LexiconMarkers, v.NGLexicon, v.Exemplars, v.Brightness,
			strconv.Itoa(v.SinceCh), strconv.Itoa(v.UntilCh)} {
			writeField(h, f)
		}
	}
	for _, p := range in.Pairs {
		if !in.InjectVoice ||
			!inScope[[2]string{p.SpeakerSrc, p.SpeakerSense}] || !inScope[[2]string{p.AddresseeSrc, p.AddresseeSense}] {
			continue
		}
		hasVoice = true
		for _, f := range []string{p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense,
			p.Register, p.Form, p.Closeness, strconv.Itoa(p.SinceCh), strconv.Itoa(p.UntilCh)} {
			writeField(h, f)
		}
	}
	if hasVoice {
		h.Write([]byte("voice-address-v1\x00"))
	}
	return hex.EncodeToString(h.Sum(nil))
}

func writeField(h interface{ Write([]byte) (int, error) }, s string) {
	var lb [8]byte
	n := uint64(len(s))
	for i := 0; i < 8; i++ {
		lb[i] = byte(n >> (8 * i))
	}
	h.Write(lb[:])
	h.Write([]byte(s))
}

// glossaryLineTokens is the injected token cost of one record's "src → dst" line — the
// unit the token budget (config glossary_token_budget) spends. It is FLOOR-FREE (the raw
// cjk + other/3 estimate, NOT EstimateTokens' 16-token minimum): the floor is for sizing
// a whole request's max_tokens, but applying it PER glossary line ~1.85×-overcounts a
// block of short name lines and needlessly evicts records that fit (self-review #7). The
// header's cost is small and constant; omitting it keeps the unit purely per-record.
// Shared by Select's budget and the eviction test so the two never diverge.
func glossaryLineTokens(e *entry) int {
	cjk, other := text.DenseSparseCounts(e.src + " → " + e.dst) // same sizing taxonomy as EstimateTokens
	return cjk + other/3
}

// Select is the hot path for ONE chunk: match → spoiler-reject → disposition → sticky
// → priority token budget. Pure and deterministic. stickyPrev is the set of ids
// exact-matched in the prior chunk(s) of the same chapter (A5). budgetTokens ≤ 0 means
// unbounded; otherwise records are kept in priority order while the cumulative injected
// token cost stays within budget — the rest are EVICTED (dropped but logged, F2; the
// budget may be underfilled, "better nothing than garbage").
func (b *Bank) Select(chunk string, chapter int, stickyPrev map[string]InjectionDisposition, budgetTokens int) Selection {
	// ZERO-VALUE GUARD (pack-16 tail): Bank is an exported type, so `var b membank.Bank` is constructible
	// outside this package, and its matcher would then be nil — a panic on the first non-empty chunk, in a
	// PAID run. A bank with no matcher has nothing to match, so the honest answer is an empty selection.
	// Materialize always builds the matcher, so this can never change the behaviour of a real bank.
	if b == nil || b.ac == nil {
		return Selection{}
	}
	ntext := []rune(text.NormalizeSourceKey(chunk))
	occ := b.ac.matches(ntext)
	// Source word-boundary for spaced-phonetic keys (D16.3): drop an occurrence of a Latin/
	// Cyrillic key that fired INSIDE a longer word of the same script ("rose" in "roseanne").
	// Closes the source-vs-target boundary asymmetry (the target side already has
	// containsWholeWord). Han/kana are NOT boundary-checked (no word segmentation) — deliberate.
	occ = b.suppressUnboundedPhonetic(occ, ntext)
	// Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when the
	// longer key belongs to a spoiler-VALID entry at this chapter (self-review #6) AND is at least
	// as TRUSTWORTHY as the nested match it would delete (D39 layer 4 — a lower-trust longer key must
	// not eat a higher-trust nested one). Refused suppressions are returned for a loud record.
	var trustGated []TrustGateEvent
	occ, trustGated = b.suppressContained(occ, chapter)

	// Map surviving key occurrences → entries, recording the LONGEST firing key per entry.
	matchedVia := map[int]string{}
	for _, m := range occ {
		k := b.ac.keys[m.keyIdx]
		for _, ei := range b.keyOwners[k] {
			if cur, ok := matchedVia[ei]; !ok || len([]rune(k)) > len([]rune(cur)) {
				matchedVia[ei] = k
			}
		}
	}

	sel := Selection{ActiveIDs: map[string]InjectionDisposition{}, TrustGated: trustGated}
	hits := map[string]PickedEntry{} // id → picked (exact)

	// Iterate entries in their frozen (ORDER BY-stable) order — never map order.
	for ei := range b.entries {
		via, ok := matchedVia[ei]
		if !ok {
			continue
		}
		e := &b.entries[ei]
		if blocked := spoilerBlocked(e, chapter); blocked {
			sel.Rejected = append(sel.Rejected, PickedEntry{entry: e, Via: via, Disp: Reject})
			continue
		}
		disp := dispositionFor(e, via)
		hits[e.id] = PickedEntry{entry: e, Via: via, Disp: disp}
		sel.ActiveIDs[e.id] = disp // exact-matched ids feed the next chunk's sticky WITH the disposition they fired with (D16.2)
	}

	// Sticky scene-inertia: carry prior-chunk exact matches not re-matched here, unless
	// the spoiler window blocks them (spoiler beats sticky). The carry INHERITS the prior
	// match's disposition (D16.2) — a sticky carry has no firing key, so it cannot re-derive
	// the collision-prone downgrade; recomputing status-based would silently upgrade a
	// collision-prone AMBIGUOUS match to CONFIRMED, past the post-check, into the editor
	// constraints. An AMBIGUOUS carry stays AMBIGUOUS (⟨проверить⟩, excluded from the editor).
	for ei := range b.entries {
		e := &b.entries[ei]
		carryDisp, sticky := stickyPrev[e.id]
		if !sticky {
			continue
		}
		if _, already := hits[e.id]; already {
			continue
		}
		if spoilerBlocked(e, chapter) {
			// RECORDED, not dropped (pack-19): a window-blocked carry is a spoiler reject like any other,
			// and a silent drop makes n_spoiler_blocked under-count. Unreachable through the production
			// driver — precomputeSticky resets the window at every chapter boundary, so a carry always
			// arrives at the SAME chapter it fired in and cannot have become blocked — but Select is
			// exported and the invariant that protects it lives in another package.
			sel.Rejected = append(sel.Rejected, PickedEntry{entry: e, Sticky: true, Disp: Reject})
			continue
		}
		hits[e.id] = PickedEntry{entry: e, Sticky: true, Disp: carryDisp}
	}

	// Deterministic priority order for the budget: confirmed>ambiguous, exact>sticky,
	// approved>auto. Collect in frozen-entry order (NOT map order) then stable-sort, so
	// ties keep the deterministic base order.
	order := make([]PickedEntry, 0, len(hits))
	for ei := range b.entries {
		if p, ok := hits[b.entries[ei].id]; ok {
			order = append(order, p)
		}
	}
	sort.SliceStable(order, func(i, j int) bool {
		return priorityRank(order[i]) < priorityRank(order[j])
	})

	if budgetTokens > 0 {
		used, cut := 0, len(order)
		for i := range order {
			used += glossaryLineTokens(order[i].entry)
			if used > budgetTokens {
				cut = i // this record and everything after it overflow the budget
				break
			}
		}
		sel.Injected = order[:cut]
		sel.Evicted = order[cut:] // F2: dropped, but LOGGED via the retrieval-state
	} else {
		sel.Injected = order
	}
	return sel
}

// spoilerBlocked reports whether the entry's since_ch/until_ch window excludes this
// chapter (C1: a hard reject gate — a fact the current chapter must not know yet).
func spoilerBlocked(e *entry, chapter int) bool {
	if e.sinceCh > 0 && chapter < e.sinceCh {
		return true
	}
	if e.untilCh > 0 && chapter > e.untilCh {
		return true
	}
	return false
}

// dispositionFor maps a term + its firing key to an injection disposition (A2). approved →
// CONFIRMED (authoritative); auto/draft → AMBIGUOUS. AND: even an approved entry matched by
// a COLLISION-PRONE key (short + purely phonetic — リン in リンゴ, AI in RAID) is downgraded to
// AMBIGUOUS (unverified + forced post-check), because such a key may have fired inside an
// unrelated word rather than on the entity (external-review major #2 — the A2
// "surface-collision → AMBIGUOUS" branch). allow_short is the author's explicit override.
// Called ONLY for an EXACT match (a real firing key): a sticky carry INHERITS its prior
// disposition instead (D16.2), never routing through here, so it cannot silently upgrade a
// collision-prone match.
func dispositionFor(e *entry, via string) InjectionDisposition {
	if !e.allowShort && collisionProneKey(via) {
		return Ambiguous
	}
	if e.status == "approved" {
		return Confirmed
	}
	return Ambiguous
}

// priorityRank orders the budget: confirmed before ambiguous, exact before sticky,
// approved before auto/draft (F2 eviction keeps the most trustworthy records).
func priorityRank(p PickedEntry) int {
	rank := 0
	if p.Disp != Confirmed {
		rank |= 1 << 2
	}
	if p.Sticky {
		rank |= 1 << 1
	}
	if !p.valid() {
		return rank | 1<<0 // a row-less record ranks with the unapproved; it is skipped before rendering
	}
	if p.entry.status != "approved" {
		rank |= 1 << 0
	}
	return rank
}

// RenderGlossaryBlock serializes the selected records into the injection message
// (§C). Records with no dst yet (ruby candidates) are skipped — a "src → " line
// carries nothing. Returns "" when nothing renders, so an empty selection injects NO
// message at all ("better nothing than garbage"). Deterministic: the injected order is the
// budget priority order fixed by Select. tx is the TARGET-language wire-text (pair-14 §2): a
// target with no injection texts (HasData()==false) injects nothing (a non-ru book gets no
// Russian block), so the whole render is gated on it.
func RenderGlossaryBlock(injected []PickedEntry, tx lang.InjectionTexts) string {
	if !tx.HasData() {
		return ""
	}
	var lines []string
	for _, p := range injected {
		if !p.valid() || strings.TrimSpace(p.entry.dst) == "" {
			continue
		}
		line := p.entry.src + " → " + p.entry.dst
		if p.Disp != Confirmed {
			line += tx.UnverifiedMarker
		} else {
			// pack-13 injection-completeness fix (D39.21 owner directive: «род должен доезжать»): the
			// gender of a CONFIRMED named term (蛊 Надежды = ж.р., a cicada gu, …) now reaches the DRAFT
			// wire too, not only the editor — so the TRANSLATOR renders the right родовые формы FIRST,
			// instead of leaving the editor to repair a wrong gender. "" for a genderless term (the common
			// case → byte-identical). Mirrors the editor block's confirmed-only gender (RenderEditorConstraintBlock).
			line += genderConstraintNote(p.entry.gender, tx)
		}
		lines = append(lines, line)
	}
	if len(lines) == 0 {
		return ""
	}
	return tx.GlossaryHeader + "\n" + strings.Join(lines, "\n")
}

// RenderFormatVersion versions the FORMAT of the role-injection renderers whose output is NOT
// captured by matchVersion (a scope/matcher-ALGORITHM version, not a render-layout version):
// the editor constraint block's src→dst layout (WS2 §2в) and the DC3 gender-constraint annotation
// (WS5 §5(б)). A format change shifts the injected bytes → the wire, so it must be a loud --resnapshot;
// folding it as a dedicated snapshot component (snapshot.go) keeps that loudness a MECHANISM, not
// discipline — matchVersion (const memory.go) would not move on a render edit.
// v2 (WS5 R4): the editor constraint line now carries a gender annotation for a gendered CONFIRMED
// term (DC3 injection — the Bai Ninbing fix: the injection DIRECTS the editor, no coreference needed).
// v3 (pack-13, D39.21 injection-completeness fix): the DRAFT glossary block (RenderGlossaryBlock) now
// ALSO appends the confirmed-gender annotation, so a named term's gender reaches the TRANSLATOR wire, not
// only the editor (the owner's «род должен доезжать» directive). A gendered confirmed term shifts the
// draft injected bytes → the wire, so a loud --resnapshot; a genderless bank is byte-identical to v2.
// (The bank-quality §3 neuter directive is NOT versioned here: a blanket bump is un-re-pinnable and would
// re-snapshot every book on earth. It is folded SCOPED — like the editor-unverified section — so only a
// book that actually carries a neuter row moves; see the neuter-directive tag in ComputeVersionScopedIn.)
const RenderFormatVersion = "renderfmt-v3-draft-gender+editor-src2dst+dc3-gender"

// The editor's canonical-constraint block header and the glossary header are TARGET-language wire-text
// (lang.InjectionTexts, pair-14 §2) — they gave the BILINGUAL editor (D30.1) the approved src→dst bindings
// as consistency constraints. Relocated out of the pipeline (a Russian header rendered for a →en book was a
// leak); now gated on the target and sourced from embedded per-target data.

// RenderEditorConstraintBlock serializes the selected records' CONFIRMED renderings into the
// editor's injection as a src→dst MAPPING (WS2 §2в resolves the D30.1-open question): "源термин →
// «dst»", like the translator block, NOT bare canonical Russian. Rationale (the single load-bearing
// one, review-1 F4): the editor is BILINGUAL (editor.md feeds it the source), and binding the canon
// to its SOURCE term disambiguates HOMONYMIC dst (one Russian surface for two entities) — bare
// forms cannot. It stays CONFIRMED-only on purpose: an AMBIGUOUS record is a candidate the
// translator MAY have legitimately rejected, so forcing the editor to rewrite toward it would
// corrupt a correct translation (mirrors the gate's CONFIRMED-only discipline, external-review #1);
// AMBIGUOUS/mined-draft rows never enter here. Deduped by (src,dst) — an alias and its main entry
// share the pair — which only WIDENS the list (harmless), in the budget priority order fixed by
// Select. Returns "" when nothing CONFIRMED renders → the editor gets NO injection (plain draft-only
// layout). The block is a subset of the already budget-limited memSel.Injected, so it needs no
// separate token budget. The src→dst FORMAT is snapshot-folded via RenderFormatVersion (a format
// edit is a loud --resnapshot); the injection is a message, so it also enters request_hash
// directly (no silent false-hit).
// PACK-20 EXTENSION (D39.42 п.3). In the auto mode the bank legitimately holds renderings nobody has
// signed — the terminologist's consolidation over the whole book — and the owner's decision is that those
// reach the editor «С ПОМЕТКОЙ». They are rendered as a SECOND, separately headed section, never mixed
// into the canon list: the canon is binding ("bring any divergence to this form"), the unverified section
// is a working version the editor is explicitly free to reject. Mixing them would either forfeit the
// canon's force or give an unsigned guess that force — the exact corruption the CONFIRMED-only rule was
// protecting against, which is why the rule is preserved rather than relaxed.
//
// A book with no non-Confirmed rows renders the second section as "" and is byte-identical to before.
func RenderEditorConstraintBlock(injected []PickedEntry, tx lang.InjectionTexts) string {
	if !tx.HasData() {
		return ""
	}
	canon := editorLines(injected, tx, true)
	if len(canon) == 0 && tx.EditorUnverifiedHeader == "" {
		return ""
	}
	var b strings.Builder
	if len(canon) > 0 {
		b.WriteString(tx.EditorHeader + "\n" + strings.Join(canon, "\n"))
	}
	if unverified := editorLines(injected, tx, false); len(unverified) > 0 && tx.EditorUnverifiedHeader != "" {
		if b.Len() > 0 {
			b.WriteString("\n\n")
		}
		b.WriteString(tx.EditorUnverifiedHeader + "\n" + strings.Join(unverified, "\n"))
	}
	return b.String()
}

// editorLines renders one section of the editor block: the SIGNED canon (signed=true) or the unverified
// working versions (signed=false). Deduped by (src,dst) within the section, in the budget priority order
// Select fixed. The gender directive stays CONFIRMED-only — an unsigned row's gender is as unverified as
// its rendering, and directing the editor with it would smuggle an unverified datum in as an instruction.
//
// The section follows PROVENANCE (status), not the injection disposition, and the two answer different
// questions (cold-run session, backlog 19 — the sibling of the banknote src rule on the same axis).
// dispositionFor downgrades an APPROVED row to AMBIGUOUS when its firing key is collision-prone — short
// and purely phonetic (memory.go:644) — which says "this key may have fired inside an unrelated word",
// a statement about the MATCH. Routing on it put an owner-SIGNED rendering under the header that says
// nobody has confirmed this, a statement about the AUTHORITY. On zh the branch cannot fire (Han keys
// anchor at 2), so the misrouting was invisible; on a phonetic source it would hit exactly the short
// signed names. Ambiguous still does its own job: the forced post-check and the ⟨проверить⟩ marker in the
// TRANSLATOR block are unchanged.
func editorLines(injected []PickedEntry, tx lang.InjectionTexts, signed bool) []string {
	var lines []string
	seen := map[[2]string]bool{}
	for _, p := range injected {
		if !p.valid() || (p.entry.status == "approved") != signed {
			continue
		}
		src := strings.TrimSpace(p.entry.src)
		dst := strings.TrimSpace(p.entry.dst)
		if dst == "" {
			continue
		}
		key := [2]string{src, dst}
		if seen[key] {
			continue
		}
		seen[key] = true
		line := "- " + src + " → «" + dst + "»"
		if signed {
			// The gender directive still keys on the MATCH being confirmed: on a collision-prone firing
			// the row may not be about the entity on the page, and a gender instruction is the one part of
			// the line that would then be actively wrong rather than merely redundant.
			if p.Disp == Confirmed {
				line += genderConstraintNote(p.entry.gender, tx)
			}
		} else {
			line += tx.UnverifiedMarker
		}
		lines = append(lines, line)
	}
	return lines
}

// genderConstraintNote is the DC3 gender directive appended to a CONFIRMED editor-constraint line (WS5
// §5(б)): the injection DIRECTS the editor to the character's grammatical gender, which is the fix for
// the Bai Ninbing class — no coreference needed. male/female ⇒ hard gender forms; hidden ⇒ a mandate to
// AVOID gender-marking constructions until the reveal (a masculine default when unavoidable, D19.3). ""
// for a term with no gender datum (the common case → the line is unchanged, byte-identical to before).
func genderConstraintNote(gender string, tx lang.InjectionTexts) string {
	switch gender {
	case "male", "m":
		return tx.GenderMale
	case "female", "f":
		return tx.GenderFemale
	case "neuter", "n":
		return tx.GenderNeuter
	case "hidden":
		return tx.GenderHidden
	}
	return ""
}

// --- Aho-Corasick multi-pattern automaton over runes ----------------------------

type acMatch struct {
	keyIdx     int
	start, end int // rune indices [start, end)
}

type acNode struct {
	next map[rune]int
	fail int
	out  []int // key indices whose pattern ends at this node (incl. via fail links)
}

type ahoCorasick struct {
	nodes []acNode
	keys  []string
	klen  []int // rune length of each key
}

// buildAC constructs the automaton from the (already sorted, unique) normalized keys.
// Deterministic in RESULT regardless of map iteration order: fail links and outputs are
// a function of the key set, not the BFS visit order.
func buildAC(keys []string) *ahoCorasick {
	ac := &ahoCorasick{keys: keys, klen: make([]int, len(keys))}
	ac.nodes = []acNode{{next: map[rune]int{}}} // root = node 0
	for i, k := range keys {
		kr := []rune(k)
		ac.klen[i] = len(kr)
		cur := 0
		for _, r := range kr {
			nxt, ok := ac.nodes[cur].next[r]
			if !ok {
				nxt = len(ac.nodes)
				ac.nodes = append(ac.nodes, acNode{next: map[rune]int{}})
				ac.nodes[cur].next[r] = nxt
			}
			cur = nxt
		}
		ac.nodes[cur].out = append(ac.nodes[cur].out, i)
	}
	// BFS to compute fail links; propagate outputs down fail chains one level (each
	// node inherits its fail node's already-complete output set).
	var queue []int
	for _, c := range ac.nodes[0].next {
		ac.nodes[c].fail = 0
		queue = append(queue, c)
	}
	for len(queue) > 0 {
		cur := queue[0]
		queue = queue[1:]
		for r, nxt := range ac.nodes[cur].next {
			queue = append(queue, nxt)
			f := ac.nodes[cur].fail
			for f != 0 {
				if _, ok := ac.nodes[f].next[r]; ok {
					break
				}
				f = ac.nodes[f].fail
			}
			if fn, ok := ac.nodes[f].next[r]; ok && fn != nxt {
				ac.nodes[nxt].fail = fn
			} else {
				ac.nodes[nxt].fail = 0
			}
			ac.nodes[nxt].out = append(ac.nodes[nxt].out, ac.nodes[ac.nodes[nxt].fail].out...)
		}
	}
	return ac
}

// matches returns every key occurrence in text (rune indices), sorted deterministically.
func (ac *ahoCorasick) matches(text []rune) []acMatch {
	var out []acMatch
	cur := 0
	for i, r := range text {
		for cur != 0 {
			if _, ok := ac.nodes[cur].next[r]; ok {
				break
			}
			cur = ac.nodes[cur].fail
		}
		if nxt, ok := ac.nodes[cur].next[r]; ok {
			cur = nxt
		} else {
			cur = 0
		}
		for _, ki := range ac.nodes[cur].out {
			out = append(out, acMatch{keyIdx: ki, start: i - ac.klen[ki] + 1, end: i + 1})
		}
	}
	sort.Slice(out, func(a, b int) bool {
		if out[a].start != out[b].start {
			return out[a].start < out[b].start
		}
		if out[a].end != out[b].end {
			return out[a].end < out[b].end
		}
		return out[a].keyIdx < out[b].keyIdx
	})
	return out
}

// trustRank orders injection dispositions by how much the reader may TRUST the record: CONFIRMED
// (approved, authoritative) outranks AMBIGUOUS (draft/auto/collision-prone), which outranks REJECT
// (spoiler-blocked / unset). It decides the longest-match containment contest below.
func trustRank(d InjectionDisposition) int {
	switch d {
	case Confirmed:
		return 2
	case Ambiguous:
		return 1
	default: // Reject / unset
		return 0
	}
}

// matchTrust returns the BEST (highest-trust) disposition an occurrence of the given key could
// inject with at this chapter — the max over its spoiler-VALID owner entries — and whether ANY
// valid owner exists (this second value subsumes the old validSuppressor: no valid owner ⇒ the
// occurrence injects nothing). CONFIRMED beats AMBIGUOUS. It is the trust the disposition-gate
// decides a containment contest on (D39 layer 4).
func (b *Bank) matchTrust(m acMatch, chapter int) (disp InjectionDisposition, hasValid bool) {
	key := b.ac.keys[m.keyIdx]
	for _, ei := range b.keyOwners[key] {
		e := &b.entries[ei]
		if spoilerBlocked(e, chapter) {
			continue
		}
		if d := dispositionFor(e, key); !hasValid || trustRank(d) > trustRank(disp) {
			disp, hasValid = d, true
		}
	}
	return disp, hasValid
}

// suppressContained applies longest-match / whole-entity replacement (A3), DISPOSITION-GATED (D39
// layer 4). It drops an occurrence fully contained inside a STRICTLY longer one, but ONLY when the
// longer one is BOTH (a) owned by a spoiler-VALID entry at this chapter (self-review #6: a spoiler-
// blocked longer key like 林动的父亲, until_ch=3, read at ch10, must not eat a valid nested 林动) AND
// (b) at least as TRUSTWORTHY as the nested match it would delete. A longer but LOWER-trust key
// (a draft/ambiguous 四代族长) must NOT suppress a nested HIGHER-trust key (an approved 族长): the draft
// is editor-excluded (RenderEditorConstraintBlock is CONFIRMED-only), so eating the approved term
// leaves the reader with nothing AND the drop is invisible to post-check/retrieval-state — the exact
// term-drift code root the D38.5 seed-promote only patched by data (L3-recurrence). The whole-entity
// longest-match is preserved when the longer key is ≥ trust (approved 四代族长 still beats approved
// 族长; draft 四代族长 still eats a nested draft 族长). Each REFUSED suppression is recorded as a
// TrustGateEvent so the near-silent drop is surfaced loudly (research/13 §7). Equal-length overlaps
// are both kept (genuine surface ambiguity). Deterministic (ms is sorted; events deduped by pair in
// ms order). O(n²) over the few matches in a chunk.
func (b *Bank) suppressContained(ms []acMatch, chapter int) ([]acMatch, []TrustGateEvent) {
	var kept []acMatch
	var gated []TrustGateEvent
	for i, m := range ms {
		conDisp, conValid := b.matchTrust(m, chapter)
		contained := false
		var refused *TrustGateEvent // a lower-trust valid longer key that WANTED to suppress m
		for j, o := range ms {
			if i == j {
				continue
			}
			// o must STRICTLY contain m's span to be a longest-match suppressor.
			if !(o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start)) {
				continue
			}
			supDisp, supValid := b.matchTrust(o, chapter)
			if !supValid {
				continue // spoiler-blocked longer key — never a valid suppressor (self-review #6)
			}
			if !conValid || trustRank(supDisp) >= trustRank(conDisp) {
				contained = true // a valid, ≥-trust container wins the whole-entity longest-match
				break
			}
			// supValid && conValid && supDisp < conDisp: a lower-trust longer key wants to eat a
			// higher-trust nested one — REFUSE (the L3 fix). Remember the FIRST such refusal for the
			// loud record; keep scanning in case a higher-trust container legitimately suppresses m.
			if refused == nil {
				refused = &TrustGateEvent{
					Suppressor: b.ac.keys[o.keyIdx], SuppressorDisp: string(supDisp),
					Protected: b.ac.keys[m.keyIdx], ProtectedDisp: string(conDisp),
				}
			}
		}
		if contained {
			continue // m suppressed by the whole-entity longest-match; any refusal above was moot
		}
		kept = append(kept, m)
		if refused != nil {
			gated = append(gated, *refused) // m survived AND a lower-trust key tried to eat it — loud
		}
	}
	return kept, dedupeTrustGate(gated)
}

// dedupeTrustGate collapses identical (suppressor→protected) events to a distinct set, preserving
// first-occurrence (ms) order so the count and detail are deterministic and count distinct term
// collisions rather than raw occurrences.
func dedupeTrustGate(evs []TrustGateEvent) []TrustGateEvent {
	if len(evs) < 2 {
		return evs
	}
	seen := map[[2]string]bool{}
	var out []TrustGateEvent
	for _, e := range evs {
		k := [2]string{e.Suppressor, e.Protected}
		if seen[k] {
			continue
		}
		seen[k] = true
		out = append(out, e)
	}
	return out
}

// suppressUnboundedPhonetic drops an occurrence of a SPACED-SCRIPT phonetic key (Latin or
// Cyrillic — scripts that delimit words with whitespace/non-letters, exactly like the
// target-side containsWholeWord) when a neighbouring rune is a letter of the SAME script:
// the key fired INSIDE a longer word ("rose" in "roseanne"/"roses"), not on the entity
// (D16.3). This closes the source-vs-target boundary asymmetry the ≤3 collision-downgrade
// does not (a ≥4 phonetic key fires CONFIRMED with no boundary today). A cross-script
// neighbour (a Latin name abutting a Han char in unspaced source) is a valid boundary, so
// only a SAME-script letter suppresses — mirroring "non-letter or foreign-script".
// KANA and HAN are deliberately NOT boundary-checked: they have no word segmentation, so a
// letter-boundary rule would false-negative the ubiquitous kana name+particle case (すずきは)
// or contradict the long-key-CONFIRMED contract — their ≥4 precision is deferred to the
// kana-precision measurement + the B6 tokenizer (D16; flagged to the orchestrator). ntext is
// the already-normalized chunk. O(n) over the few matches in a chunk.
func (b *Bank) suppressUnboundedPhonetic(ms []acMatch, ntext []rune) []acMatch {
	var kept []acMatch
	for _, m := range ms {
		script := spacedPhoneticScript(b.ac.keys[m.keyIdx])
		if script != nil {
			beforeBad := m.start > 0 && letterInScript(ntext[m.start-1], script)
			afterBad := m.end < len(ntext) && letterInScript(ntext[m.end], script)
			if beforeBad || afterBad {
				continue // the key is inside a longer same-script word — not a whole-entity match
			}
		}
		kept = append(kept, m)
	}
	return kept
}

// spacedPhoneticScript returns the word-delimited alphabetic script a normalized key belongs
// to for the source word-boundary check — unicode.Latin or unicode.Cyrillic — or nil when the
// key carries a Han ideograph or kana (no word segmentation) or mixes the two spaced scripts.
// Digits/punctuation don't set the script but don't disqualify (so "o'brien" is still Latin).
func spacedPhoneticScript(normKey string) *unicode.RangeTable {
	var script *unicode.RangeTable
	for _, r := range normKey {
		switch {
		case text.DenseScript(r):
			return nil // an ideographic/kana anchor is never boundary-checked here
		case unicode.In(r, unicode.Latin):
			if script == unicode.Cyrillic {
				return nil // mixed spaced scripts — do not boundary-check
			}
			script = unicode.Latin
		case unicode.In(r, unicode.Cyrillic):
			if script == unicode.Latin {
				return nil
			}
			script = unicode.Cyrillic
		}
	}
	return script
}

// letterInScript reports whether r is a LETTER of the given spaced script — the "same-script
// letter" that breaks a word boundary. A digit, punctuation, space, or a letter of another
// script is a valid boundary and returns false.
func letterInScript(r rune, script *unicode.RangeTable) bool {
	return unicode.IsLetter(r) && unicode.In(r, script)
}

// InjectivityCollisions reports approved dst collisions (B2): two distinct source
// terms mapped to the SAME dst (one Russian surface for two entities → the reader
// cannot tell them apart), returned as human-readable strings for a load-time warning.
// A pure diagnostic; it does not reject (some collisions are legitimate, e.g. a title
// shared by rank tiers), so the caller logs it, not aborts.
func InjectivityCollisions(rows []store.GlossaryEntry) []string {
	bySurface := map[string][]string{}
	for _, r := range rows {
		if r.Status != "approved" || strings.TrimSpace(r.Dst) == "" {
			continue
		}
		key := text.NormalizeTargetForm(r.Dst)
		bySurface[key] = append(bySurface[key], r.Src)
	}
	var out []string
	// Deterministic order: iterate a sorted key list, not the map.
	keys := slices.Sorted(maps.Keys(bySurface))
	for _, k := range keys {
		srcs := bySurface[k]
		if len(distinct(srcs)) > 1 {
			sort.Strings(srcs)
			out = append(out, "dst "+strconv.Quote(k)+" ← "+strings.Join(distinct(srcs), ", "))
		}
	}
	return out
}

func distinct(ss []string) []string {
	seen := map[string]bool{}
	var out []string
	for _, s := range ss {
		if !seen[s] {
			seen[s] = true
			out = append(out, s)
		}
	}
	sort.Strings(out)
	return out
}