textmachine/backend/internal/pipeline/memory.go

package pipeline

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"maps"
	"slices"
	"sort"
	"strconv"
	"strings"
	"unicode"

	"textmachine/backend/internal/store"
)

// memory.go: the DETERMINISTIC hot path of the memory bank v2 (registry §Контракт
// горячего пути / research/13 Q3). $0, no LLM, no embeddings, no FTS5 — a multi-pattern
// exact match (Aho-Corasick) of glossary keys/aliases over the NORMALIZED chunk, with
// the single-key ban (A3), longest-match whole-entity replacement, the spoiler
// hard-reject window (C1), sticky scene-inertia (A5), a priority token budget with a
// logged eviction (F2), and the three-way injection disposition (A2). Pure and
// deterministic (no map-order output, no time/rand): the injected block is a pure
// function of (frozen bank, normalized chunk, chapter, sticky, budget), and it enters
// request_hash via the rendered messages — so a resumed chunk reproduces it for free.
//
// The registry's principle drives every choice here: "лучше ничего, чем мусор" and
// PRECISION over recall. A wrong injection (a homograph / short alias firing on the
// wrong sense) is the MAIN source of silent degradation (2510.00829) — worse than an
// empty match, which is a safe fallback to the model's base behaviour. So the matcher
// errs toward NOT firing, and the post-check (memcheck) makes what did fire observable.

// memoryMatchVersion versions the matcher + injection-selection ALGORITHM (min-key
// ban, longest-match containment, spoiler gate, sticky, budget priority order). Folded
// into memoryVersion() → a change is a loud --resnapshot (it shifts the injected bytes
// → the wire → a resumed checkpoint's content). Sibling of memoryNormVersion.
// v3 (D16.2/D16.3): sticky now INHERITS the original match's disposition instead of
// recomputing it status-based (a collision-prone AMBIGUOUS carry no longer silently
// upgrades to CONFIRMED), and a spaced-phonetic (Latin/Cyrillic) source key is validated
// against a word boundary (approved "rose" no longer fires inside "roseanne").
// v4 (D39 слой 4): the longest-match suppressor is now DISPOSITION-GATED (suppressContained) —
// a longer but LOWER-trust key (draft/ambiguous) no longer deletes a nested HIGHER-trust key
// (approved/confirmed), closing the term-drift code root the D38.5 seed-promote only patched by
// data (L3-suppressor-disposition-blind). A refused suppression is recorded loudly (trustGated).
// The injection SELECTION changes for a draft-longer-over-approved-shorter chunk (the approved term
// now survives), so a loud --resnapshot + golden re-capture (invariant №8).
const memoryMatchVersion = "memmatch-v4-perlang-minkey+collision-ambiguous+longest-trustgated+spoiler+sticky-inherit-disp+phonetic-srcboundary+budget+postcheck-declaware"

// minKeyLenHan / minKeyLenPhonetic are the per-language single-key floors (A3, registry
// "min_key_len per-язык"). An ideographic (Han) key is semantically distinct at 2 chars
// (empirically precision-1.0 on the zh retrieval_bench homograph traps). A PURELY PHONETIC
// key (kana/latin/cyrillic, no Han anchor) is collision-prone at 2 — short phonetic
// sequences appear INSIDE ordinary words (リン in リンゴ/apple, AI in RAID) — so it needs ≥3.
// external-review major #3: the MIN=2 measurement was on zh-Han and does NOT transfer to
// ja-kana; 3 is a conservative default pending a ja-kana precision measurement (an E1-
// protocol extension). A per-entry allow_short overrides both (rare, guarded).
const (
	minKeyLenHan      = 2
	minKeyLenPhonetic = 3
)

// minKeyLenFor is the floor for a normalized key: any Han ideograph anchors it (2), an
// all-phonetic key needs 3.
func minKeyLenFor(normKey string) int {
	if runesAnyHan(normKey) {
		return minKeyLenHan
	}
	return minKeyLenPhonetic
}

// collisionProneKey reports whether a key is short AND purely phonetic (no Han anchor),
// so it is likely to fire inside an unrelated word — the A2 "surface collision" case,
// injected as AMBIGUOUS (unverified) rather than authoritatively CONFIRMED.
func collisionProneKey(normKey string) bool {
	return !runesAnyHan(normKey) && significantLen(normKey) <= minKeyLenPhonetic
}

// stickyDepth is the scene-inertia window (A5): entries exact-matched in the previous
// N chunks of the SAME chapter are carried into a pronominal chunk that names nobody.
// A code rule (Р2), reset at each chapter boundary (a new chapter is a scene change).
const stickyDepth = 2

// injectionDisposition is the three-way per-record decision (registry A2) — the core
// mechanism that converts silent degradation into loud. Distinct from the chunk×stage
// Disposition (disposition.go): that is ok|flagged|skipped for a completion; this is
// how much to TRUST an injected glossary record.
type injectionDisposition string

const (
	memConfirmed injectionDisposition = "confirmed" // exact key/alias, approved, spoiler-valid → authoritative
	memAmbiguous injectionDisposition = "ambiguous" // auto/draft term → inject "unverified" + forced post-check
	memReject    injectionDisposition = "reject"    // spoiler-window violation → dropped and logged (C1 safety)
)

// declInfo is the parsed decl column: the dst declension forms the post-check accepts.
type declInfo struct {
	Invariant bool     `json:"invariant"`
	Forms     []string `json:"forms"`
}

// memoryEntry is one frozen, materialized glossary entry with its matchable keys
// pre-normalized. Immutable for a job.
type memoryEntry struct {
	id      string // stable unique id: src\x1f sense\x1f since\x1f until (= the UNIQUE key)
	src     string // raw source key
	dst     string // raw approved translation (may be "" for a ruby candidate)
	status  string // auto|draft|approved
	sense   string
	gender  string // male|female|hidden|"" (C2) — feeds the DC3 gender constraint on the editor block
	sinceCh int
	untilCh int
	// normKeys are the normalized source surfaces (src + aliases) ELIGIBLE to fire
	// (significantLen ≥ minKeyLen OR allow_short). A key too short is dropped here, so
	// it is never in the automaton — the single-key ban is structural, not a runtime skip.
	normKeys []string
	// declForms are the normalized-target dst forms the post-check accepts. Empty →
	// the post-check falls back to the single normalized dst (the naive base form that
	// research/14 shows false-flags on inflection — the reason full decl matters).
	declForms     []string
	declInvariant bool
	// allowShort is the author's explicit override of the single-key ban AND the
	// collision-prone disposition downgrade (full trust in a short key).
	allowShort bool
}

// MemoryBank is a book's frozen glossary materialized for one job: the entries, the
// Aho-Corasick automaton over all eligible keys, and the F1 memoryVersion. Built once
// (materializeMemory) before the chunk loop and never mutated.
type MemoryBank struct {
	entries   []memoryEntry
	ac        *ahoCorasick
	keyOwners map[string][]int // normalized key → indices of entries that contributed it
	// enrichedVersion folds ALL foldable rows incl Source:mined — the memory version of the EDIT
	// wave (edit-wave snapshot). baseVersion EXCLUDES Source:mined — the memory version of the DRAFT wave
	// (draft-wave snapshot), so a the bank-mining stop bank-enrichment (adding mined rows) moves ONLY edit-wave snapshot, keeping
	// draft-wave checkpoints valid (WS1 §1в, «переоплата ОДНА»). The two are DOMAIN-SEPARATED (a "base-excl-
	// mined" tag in the base fold), so base ≠ enriched for EVERY row set — even a mined-free book (a
	// the draft wave job must never content-address to a the edit wave checkpoint). The load-bearing invariant is not
	// equality but STABILITY: base stays fixed as mined rows are added; only enriched moves.
	enrichedVersion string
	baseVersion     string
}

// pickedEntry is one selected record for a chunk with its firing key and disposition.
type pickedEntry struct {
	entry *memoryEntry
	via   string // the matching key, or "sticky"
	disp  injectionDisposition
}

// trustGateEvent records a longest-match suppression the DISPOSITION-gate REFUSED (D39 слой 4):
// a longer but LOWER-trust key (draft/ambiguous, e.g. draft 四代族长) that would have deleted a nested
// HIGHER-trust key (approved/confirmed, e.g. approved 族长). The draft is editor-excluded (the block
// is CONFIRMED-only), so suppressing the approved term would have left the reader with NOTHING — the
// term-drift code root (L3). The nested higher-trust match survives and injects; this record turns
// the near-silent drop into a loud, operator-visible signal (research/13 §7). A non-empty list means
// the seed has a draft term nesting over an approved one — reconcile the seed so it stops recurring.
type trustGateEvent struct {
	Suppressor     string `json:"suppressor"`      // the longer, lower-trust key blocked from suppressing
	SuppressorDisp string `json:"suppressor_disp"` // its best injection disposition (ambiguous)
	Protected      string `json:"protected"`       // the nested, higher-trust key it would have eaten
	ProtectedDisp  string `json:"protected_disp"`  // its best injection disposition (confirmed)
}

// memorySelection is the hot path's output for one chunk.
type memorySelection struct {
	injected []pickedEntry // in budget priority order (what the model sees)
	rejected []pickedEntry // spoiler-window rejects (C1, logged)
	evicted  []pickedEntry // dropped by the token budget (F2, logged)
	// trustGated are the longest-match suppressions the disposition-gate refused (a lower-trust
	// longer key would have eaten a higher-trust nested one) — the term-drift code root, surfaced
	// loudly instead of a silent drop (research/13 §7). Deterministic (ms order, deduped by pair).
	trustGated []trustGateEvent
	// activeIDs are the exact-matched ids (NOT sticky) → the next chunk's sticky_prev,
	// each mapped to the DISPOSITION it fired with. Sticky carries this disposition
	// forward instead of recomputing it (D16.2): a sticky carry has no firing key, so it
	// cannot re-derive the collision-prone downgrade, and recomputing from status alone
	// would silently upgrade a collision-prone AMBIGUOUS match to CONFIRMED.
	activeIDs map[string]injectionDisposition
}

// Version is the ENRICHED memory version (all approved rows incl mined) — the edit-wave / whole-book
// memory component. BaseVersion excludes Source:mined (the draft-wave component). They are
// DOMAIN-SEPARATED — base ≠ enriched for ANY row set, including a mined-free book (never assert
// equality) — but base stays STABLE when a the bank-mining stop pass adds mined rows, which is what keeps the draft wave
// checkpoints valid («переоплата ОДНА»).
func (b *MemoryBank) Version() string     { return b.enrichedVersion }
func (b *MemoryBank) BaseVersion() string { return b.baseVersion }

// materializeMemory builds a MemoryBank from the book's stored glossary rows (ORDER
// BY-stable — GlossaryForBook). Pure and deterministic. It computes memoryVersion as a
// content hash of the frozen APPROVED rows (D8: content-hash, not a version-counter —
// drift-proof) plus the normalization + matcher algorithm versions, so ANY change to
// the approved glossary OR to the deterministic machinery is a loud --resnapshot (F1).
func materializeMemory(rows []store.GlossaryEntry, gateOn bool) *MemoryBank {
	b := &MemoryBank{keyOwners: map[string][]int{}}
	var allKeys []string
	seenKey := map[string]bool{}

	for _, row := range rows {
		e := memoryEntry{
			id:         row.Src + "\x1f" + row.Sense + "\x1f" + strconv.Itoa(row.SinceCh) + "\x1f" + strconv.Itoa(row.UntilCh),
			src:        row.Src,
			dst:        row.Dst,
			status:     row.Status,
			sense:      row.Sense,
			gender:     row.Gender,
			sinceCh:    row.SinceCh,
			untilCh:    row.UntilCh,
			allowShort: row.AllowShort,
		}
		// Eligible source surfaces (src + aliases) under the single-key ban. A record
		// with NO dst yet (a ruby auto-candidate) is NOT matchable: it renders nothing
		// (renderGlossaryBlock/postcheck both skip empty dst), so admitting its keys would
		// let it consume the token budget, evict a renderable line, and inflate the
		// retrieval-state exact-hit count for a record the model never sees (self-review
		// #1). It stays in the store for future promotion, just inert on the hot path.
		var surfaces []string
		if strings.TrimSpace(row.Dst) != "" {
			surfaces = append(surfaces, row.Src)
			for _, a := range row.Aliases {
				surfaces = append(surfaces, a.Alias)
			}
		}
		for _, s := range surfaces {
			nk := normalizeSourceKey(s)
			if nk == "" {
				continue
			}
			if significantLen(nk) < minKeyLenFor(nk) && !row.AllowShort {
				continue // A3: a too-short key never fires on its own (per-language floor)
			}
			e.normKeys = append(e.normKeys, nk)
		}
		// Parse decl forms for the post-check (normalized target side).
		if row.Decl != "" {
			var d declInfo
			if err := json.Unmarshal([]byte(row.Decl), &d); err == nil {
				e.declInvariant = d.Invariant
				for _, f := range d.Forms {
					if nf := normalizeTargetForm(f); nf != "" {
						e.declForms = append(e.declForms, nf)
					}
				}
			}
		}
		idx := len(b.entries)
		b.entries = append(b.entries, e)
		for _, nk := range e.normKeys {
			b.keyOwners[nk] = append(b.keyOwners[nk], idx)
			if !seenKey[nk] {
				seenKey[nk] = true
				allKeys = append(allKeys, nk)
			}
		}
	}
	sort.Strings(allKeys) // deterministic automaton construction
	b.ac = buildAC(allKeys)
	b.enrichedVersion = computeMemoryVersion(rows, gateOn)         // all approved incl mined (the edit wave)
	b.baseVersion = computeMemoryVersionScoped(rows, gateOn, true) // excl Source:mined (the draft wave)
	return b
}

// computeMemoryVersion is the F1 content-hash: the frozen rows (ORDER BY-stable) plus the
// normalization and matcher algorithm versions. Approved-only per D8/§8 when the post-check
// hard gate is OFF (auto/draft injected-content changes are caught at the per-chunk
// content_hash level; their decl changes affect only the recomputed retrieval-state).
// When the gate is ON, the resolved chunk disposition depends on the decl forms of ALL
// injected records (approved AND auto/draft), and those are in NEITHER content_hash (decl
// is not injected) NOR the approved-only hash — so a decl edit would silently flip a
// resumed chunk's disposition (self-review #4). So gateOn folds every row's content
// (incl. decl + status) into the hash, making any such edit a loud --resnapshot. The
// glossary.id autoincrement is deliberately EXCLUDED (fresh each replace); only content
// columns are hashed.
func computeMemoryVersion(rows []store.GlossaryEntry, gateOn bool) string {
	return computeMemoryVersionScoped(rows, gateOn, false)
}

// computeMemoryVersionScoped is computeMemoryVersion with an explicit Source-scope: excludeMined
// drops every Source:"mined" row from the fold (WS1 §1в base-bank-version). The DRAFT-wave snapshot
// (draft-wave snapshot) folds base (excludeMined=true) so a the bank-mining stop pass that ADDS mined rows moves ONLY the
// enriched (edit-wave) version — keeping draft-wave checkpoints valid, «переоплата ОДНА». Deterministic (a
// Source filter over the same ORDER BY-stable rows). excludeMined=false is BYTE-IDENTICAL to the
// pre-split fold (no extra field), so the enriched hash / existing snapshots do not move; the
// excludeMined=true variant adds a domain separator so base and enriched never collide.
func computeMemoryVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool) string {
	h := sha256.New()
	h.Write([]byte("tm-memory-v2\x00"))
	h.Write([]byte(memoryNormVersion + "\x00" + memoryMatchVersion + "\x00"))
	h.Write([]byte("gate:" + strconv.FormatBool(gateOn) + "\x00"))
	if excludeMined {
		h.Write([]byte("base-excl-mined\x00")) // domain separator: base ≠ enriched even over identical rows
	}
	for _, r := range rows {
		if r.Status != "approved" && !gateOn {
			continue
		}
		if excludeMined && r.Source == "mined" {
			continue
		}
		// A fixed, length-prefixed field layout so no content can forge a boundary.
		writeField(h, r.Status)
		writeField(h, r.Src)
		writeField(h, r.Dst)
		writeField(h, r.Sense)
		writeField(h, r.Type)
		writeField(h, r.Gender)
		writeField(h, r.Decl)
		writeField(h, strconv.Itoa(r.SinceCh))
		writeField(h, strconv.Itoa(r.UntilCh))
		writeField(h, strconv.FormatBool(r.AllowShort))
		writeField(h, strconv.Itoa(len(r.Aliases)))
		for _, a := range r.Aliases { // GlossaryForBook returns aliases ORDER BY alias
			writeField(h, a.Alias)
			writeField(h, a.AliasType)
		}
	}
	return hex.EncodeToString(h.Sum(nil))
}

func writeField(h interface{ Write([]byte) (int, error) }, s string) {
	var lb [8]byte
	n := uint64(len(s))
	for i := 0; i < 8; i++ {
		lb[i] = byte(n >> (8 * i))
	}
	h.Write(lb[:])
	h.Write([]byte(s))
}

// glossaryLineTokens is the injected token cost of one record's "src → dst" line — the
// unit the token budget (config glossary_token_budget) spends. It is FLOOR-FREE (the raw
// cjk + other/3 estimate, NOT EstimateTokens' 16-token minimum): the floor is for sizing
// a whole request's max_tokens, but applying it PER glossary line ~1.85×-overcounts a
// block of short name lines and needlessly evicts records that fit (self-review #7). The
// header's cost is small and constant; omitting it keeps the unit purely per-record.
// Shared by Select's budget and the eviction test so the two never diverge.
func glossaryLineTokens(e *memoryEntry) int {
	cjk, other := 0, 0
	for _, r := range e.src + " → " + e.dst {
		switch {
		case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul):
			cjk++
		case unicode.IsSpace(r):
		default:
			other++
		}
	}
	return cjk + other/3
}

// Select is the hot path for ONE chunk: match → spoiler-reject → disposition → sticky
// → priority token budget. Pure and deterministic. stickyPrev is the set of ids
// exact-matched in the prior chunk(s) of the same chapter (A5). budgetTokens ≤ 0 means
// unbounded; otherwise records are kept in priority order while the cumulative injected
// token cost stays within budget — the rest are EVICTED (dropped but logged, F2; the
// budget may be underfilled, "лучше ничего, чем мусор").
func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]injectionDisposition, budgetTokens int) memorySelection {
	ntext := []rune(normalizeSourceKey(chunk))
	occ := b.ac.matches(ntext)
	// Source word-boundary for spaced-phonetic keys (D16.3): drop an occurrence of a Latin/
	// Cyrillic key that fired INSIDE a longer word of the same script ("rose" in "roseanne").
	// Closes the source-vs-target boundary asymmetry (the target side already has
	// containsWholeWord). Han/kana are NOT boundary-checked (no word segmentation) — deliberate.
	occ = b.suppressUnboundedPhonetic(occ, ntext)
	// Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when the
	// longer key belongs to a spoiler-VALID entry at this chapter (self-review #6) AND is at least
	// as TRUSTWORTHY as the nested match it would delete (D39 слой 4 — a lower-trust longer key must
	// not eat a higher-trust nested one). Refused suppressions are returned for a loud record.
	var trustGated []trustGateEvent
	occ, trustGated = b.suppressContained(occ, chapter)

	// Map surviving key occurrences → entries, recording the LONGEST firing key per entry.
	matchedVia := map[int]string{}
	for _, m := range occ {
		k := b.ac.keys[m.keyIdx]
		for _, ei := range b.keyOwners[k] {
			if cur, ok := matchedVia[ei]; !ok || len([]rune(k)) > len([]rune(cur)) {
				matchedVia[ei] = k
			}
		}
	}

	sel := memorySelection{activeIDs: map[string]injectionDisposition{}, trustGated: trustGated}
	hits := map[string]pickedEntry{} // id → picked (exact)

	// Iterate entries in their frozen (ORDER BY-stable) order — never map order.
	for ei := range b.entries {
		via, ok := matchedVia[ei]
		if !ok {
			continue
		}
		e := &b.entries[ei]
		if blocked := spoilerBlocked(e, chapter); blocked {
			sel.rejected = append(sel.rejected, pickedEntry{e, via, memReject})
			continue
		}
		disp := dispositionFor(e, via)
		hits[e.id] = pickedEntry{e, via, disp}
		sel.activeIDs[e.id] = disp // exact-matched ids feed the next chunk's sticky WITH the disposition they fired with (D16.2)
	}

	// Sticky scene-inertia: carry prior-chunk exact matches not re-matched here, unless
	// the spoiler window blocks them (spoiler beats sticky). The carry INHERITS the prior
	// match's disposition (D16.2) — a sticky carry has no firing key, so it cannot re-derive
	// the collision-prone downgrade; recomputing status-based would silently upgrade a
	// collision-prone AMBIGUOUS match to CONFIRMED, past the post-check, into the editor
	// constraints. An AMBIGUOUS carry stays AMBIGUOUS (⟨проверить⟩, excluded from the editor).
	for ei := range b.entries {
		e := &b.entries[ei]
		carryDisp, sticky := stickyPrev[e.id]
		if !sticky {
			continue
		}
		if _, already := hits[e.id]; already {
			continue
		}
		if spoilerBlocked(e, chapter) {
			continue
		}
		hits[e.id] = pickedEntry{e, "sticky", carryDisp}
	}

	// Deterministic priority order for the budget: confirmed>ambiguous, exact>sticky,
	// approved>auto. Collect in frozen-entry order (NOT map order) then stable-sort, so
	// ties keep the deterministic base order.
	order := make([]pickedEntry, 0, len(hits))
	for ei := range b.entries {
		if p, ok := hits[b.entries[ei].id]; ok {
			order = append(order, p)
		}
	}
	sort.SliceStable(order, func(i, j int) bool {
		return priorityRank(order[i]) < priorityRank(order[j])
	})

	if budgetTokens > 0 {
		used, cut := 0, len(order)
		for i := range order {
			used += glossaryLineTokens(order[i].entry)
			if used > budgetTokens {
				cut = i // this record and everything after it overflow the budget
				break
			}
		}
		sel.injected = order[:cut]
		sel.evicted = order[cut:] // F2: dropped, but LOGGED via the retrieval-state
	} else {
		sel.injected = order
	}
	return sel
}

// spoilerBlocked reports whether the entry's since_ch/until_ch window excludes this
// chapter (C1: a hard reject gate — a fact the current chapter must not know yet).
func spoilerBlocked(e *memoryEntry, chapter int) bool {
	if e.sinceCh > 0 && chapter < e.sinceCh {
		return true
	}
	if e.untilCh > 0 && chapter > e.untilCh {
		return true
	}
	return false
}

// dispositionFor maps a term + its firing key to an injection disposition (A2). approved →
// CONFIRMED (authoritative); auto/draft → AMBIGUOUS. AND: even an approved entry matched by
// a COLLISION-PRONE key (short + purely phonetic — リン in リンゴ, AI in RAID) is downgraded to
// AMBIGUOUS (unverified + forced post-check), because such a key may have fired inside an
// unrelated word rather than on the entity (external-review major #2 — the A2
// "surface-collision → AMBIGUOUS" branch). allow_short is the author's explicit override.
// Called ONLY for an EXACT match (a real firing key): a sticky carry INHERITS its prior
// disposition instead (D16.2), never routing through here, so it cannot silently upgrade a
// collision-prone match. The via!="sticky" guard is kept defensive.
func dispositionFor(e *memoryEntry, via string) injectionDisposition {
	if via != "sticky" && !e.allowShort && collisionProneKey(via) {
		return memAmbiguous
	}
	if e.status == "approved" {
		return memConfirmed
	}
	return memAmbiguous
}

// priorityRank orders the budget: confirmed before ambiguous, exact before sticky,
// approved before auto/draft (F2 eviction keeps the most trustworthy records).
func priorityRank(p pickedEntry) int {
	rank := 0
	if p.disp != memConfirmed {
		rank |= 1 << 2
	}
	if p.via == "sticky" {
		rank |= 1 << 1
	}
	if p.entry.status != "approved" {
		rank |= 1 << 0
	}
	return rank
}

// glossaryBlockHeader introduces the injected glossary block. The layout mirrors the
// SakuraLLM/GalTransl convention "src → dst", with unverified (ambiguous) records
// tagged so the model — and a human reviewer — see they are not authoritative (A2).
const glossaryBlockHeader = "ГЛОССАРИЙ (используй эти утверждённые переводы имён и терминов последовательно; строки с пометкой ⟨проверить⟩ — неподтверждённые кандидаты):"

// renderGlossaryBlock serializes the selected records into the injection message
// (§C). Records with no dst yet (ruby candidates) are skipped — a "src → " line
// carries nothing. Returns "" when nothing renders, so an empty selection injects NO
// message at all ("лучше ничего, чем мусор"). Deterministic: the injected order is the
// budget priority order fixed by Select.
func renderGlossaryBlock(injected []pickedEntry) string {
	var lines []string
	for _, p := range injected {
		if strings.TrimSpace(p.entry.dst) == "" {
			continue
		}
		line := p.entry.src + " → " + p.entry.dst
		if p.disp != memConfirmed {
			line += "  ⟨проверить⟩"
		}
		lines = append(lines, line)
	}
	if len(lines) == 0 {
		return ""
	}
	return glossaryBlockHeader + "\n" + strings.Join(lines, "\n")
}

// renderFormatVersion versions the FORMAT of the role-injection renderers whose output is NOT
// captured by memoryMatchVersion (a scope/matcher-ALGORITHM version, not a render-layout version):
// the editor constraint block's src→dst layout (WS2 §2в) and the DC3 gender-constraint annotation
// (WS5 §5(б)). A format change shifts the injected bytes → the wire, so it must be a loud --resnapshot;
// folding it as a dedicated snapshot component (snapshot.go) keeps that loudness a MECHANISM, not
// discipline — memoryMatchVersion (const memory.go) would not move on a render edit.
// v2 (WS5 R4): the editor constraint line now carries a gender annotation for a gendered CONFIRMED
// term (DC3 injection — the Bai Ninbing fix: the injection DIRECTS the editor, no coreference needed).
const renderFormatVersion = "renderfmt-v2-editor-src2dst+dc3-gender"

// editorConstraintHeader introduces the editor's canonical-constraint block. It gives the BILINGUAL
// editor (D30.1) the approved src→dst bindings as consistency constraints — it must render each
// listed source term with EXACTLY the paired canonical form (inflecting for context) and touch
// nothing else.
const editorConstraintHeader = "КАНОНИЧЕСКИЕ ПЕРЕВОДЫ имён и терминов (в черновике термин исходника слева ДОЛЖЕН быть передан именно указанной формой справа — приводи к ней любые расхождения, склоняя по контексту; не вводи иных вариантов и не меняй ничего другого):"

// renderEditorConstraintBlock serializes the selected records' CONFIRMED renderings into the
// editor's injection as a src→dst MAPPING (WS2 §2в resolves the D30.1-open question): "源термин →
// «dst»", like the translator block, NOT bare canonical Russian. Rationale (the single load-bearing
// one, ревью-1 F4): the editor is BILINGUAL (editor.md feeds it the source), and binding the canon
// to its SOURCE term disambiguates HOMONYMIC dst (one Russian surface for two entities) — bare
// forms cannot. It stays CONFIRMED-only on purpose: an AMBIGUOUS record is a candidate the
// translator MAY have legitimately rejected, so forcing the editor to rewrite toward it would
// corrupt a correct translation (mirrors the gate's CONFIRMED-only discipline, external-review #1);
// AMBIGUOUS/mined-draft rows never enter here. Deduped by (src,dst) — an alias and its main entry
// share the pair — which only WIDENS the list (harmless), in the budget priority order fixed by
// Select. Returns "" when nothing CONFIRMED renders → the editor gets NO injection (plain draft-only
// layout). The block is a subset of the already budget-limited memSel.injected, so it needs no
// separate token budget. The src→dst FORMAT is snapshot-folded via renderFormatVersion (a format
// edit is a loud --resnapshot); the injection is a message, so it also enters request_hash
// directly (no silent false-hit).
func renderEditorConstraintBlock(injected []pickedEntry) string {
	var lines []string
	seen := map[[2]string]bool{}
	for _, p := range injected {
		if p.disp != memConfirmed {
			continue
		}
		src := strings.TrimSpace(p.entry.src)
		dst := strings.TrimSpace(p.entry.dst)
		if dst == "" {
			continue
		}
		key := [2]string{src, dst}
		if seen[key] {
			continue
		}
		seen[key] = true
		lines = append(lines, "- "+src+" → «"+dst+"»"+genderConstraintNote(p.entry.gender))
	}
	if len(lines) == 0 {
		return ""
	}
	return editorConstraintHeader + "\n" + strings.Join(lines, "\n")
}

// genderConstraintNote is the DC3 gender directive appended to a CONFIRMED editor-constraint line (WS5
// §5(б)): the injection DIRECTS the editor to the character's grammatical gender, which is the fix for
// the Bai Ninbing class — no coreference needed. male/female ⇒ hard gender forms; hidden ⇒ a mandate to
// AVOID gender-marking constructions until the reveal (a masculine default when unavoidable, D19.3). ""
// for a term with no gender datum (the common case → the line is unchanged, byte-identical to before).
func genderConstraintNote(gender string) string {
	switch gender {
	case "male", "m":
		return " (муж. — мужские родовые формы)"
	case "female", "f":
		return " (жен. — женские родовые формы)"
	case "hidden":
		return " (пол СКРЫТ до раскрытия — избегай родовых форм; при неизбежности — мужские)"
	}
	return ""
}

// --- Aho-Corasick multi-pattern automaton over runes ----------------------------

type acMatch struct {
	keyIdx     int
	start, end int // rune indices [start, end)
}

type acNode struct {
	next map[rune]int
	fail int
	out  []int // key indices whose pattern ends at this node (incl. via fail links)
}

type ahoCorasick struct {
	nodes []acNode
	keys  []string
	klen  []int // rune length of each key
}

// buildAC constructs the automaton from the (already sorted, unique) normalized keys.
// Deterministic in RESULT regardless of map iteration order: fail links and outputs are
// a function of the key set, not the BFS visit order.
func buildAC(keys []string) *ahoCorasick {
	ac := &ahoCorasick{keys: keys, klen: make([]int, len(keys))}
	ac.nodes = []acNode{{next: map[rune]int{}}} // root = node 0
	for i, k := range keys {
		kr := []rune(k)
		ac.klen[i] = len(kr)
		cur := 0
		for _, r := range kr {
			nxt, ok := ac.nodes[cur].next[r]
			if !ok {
				nxt = len(ac.nodes)
				ac.nodes = append(ac.nodes, acNode{next: map[rune]int{}})
				ac.nodes[cur].next[r] = nxt
			}
			cur = nxt
		}
		ac.nodes[cur].out = append(ac.nodes[cur].out, i)
	}
	// BFS to compute fail links; propagate outputs down fail chains one level (each
	// node inherits its fail node's already-complete output set).
	var queue []int
	for _, c := range ac.nodes[0].next {
		ac.nodes[c].fail = 0
		queue = append(queue, c)
	}
	for len(queue) > 0 {
		cur := queue[0]
		queue = queue[1:]
		for r, nxt := range ac.nodes[cur].next {
			queue = append(queue, nxt)
			f := ac.nodes[cur].fail
			for f != 0 {
				if _, ok := ac.nodes[f].next[r]; ok {
					break
				}
				f = ac.nodes[f].fail
			}
			if fn, ok := ac.nodes[f].next[r]; ok && fn != nxt {
				ac.nodes[nxt].fail = fn
			} else {
				ac.nodes[nxt].fail = 0
			}
			ac.nodes[nxt].out = append(ac.nodes[nxt].out, ac.nodes[ac.nodes[nxt].fail].out...)
		}
	}
	return ac
}

// matches returns every key occurrence in text (rune indices), sorted deterministically.
func (ac *ahoCorasick) matches(text []rune) []acMatch {
	var out []acMatch
	cur := 0
	for i, r := range text {
		for cur != 0 {
			if _, ok := ac.nodes[cur].next[r]; ok {
				break
			}
			cur = ac.nodes[cur].fail
		}
		if nxt, ok := ac.nodes[cur].next[r]; ok {
			cur = nxt
		} else {
			cur = 0
		}
		for _, ki := range ac.nodes[cur].out {
			out = append(out, acMatch{keyIdx: ki, start: i - ac.klen[ki] + 1, end: i + 1})
		}
	}
	sort.Slice(out, func(a, b int) bool {
		if out[a].start != out[b].start {
			return out[a].start < out[b].start
		}
		if out[a].end != out[b].end {
			return out[a].end < out[b].end
		}
		return out[a].keyIdx < out[b].keyIdx
	})
	return out
}

// trustRank orders injection dispositions by how much the reader may TRUST the record: CONFIRMED
// (approved, authoritative) outranks AMBIGUOUS (draft/auto/collision-prone), which outranks REJECT
// (spoiler-blocked / unset). It decides the longest-match containment contest below.
func trustRank(d injectionDisposition) int {
	switch d {
	case memConfirmed:
		return 2
	case memAmbiguous:
		return 1
	default: // memReject / unset
		return 0
	}
}

// matchTrust returns the BEST (highest-trust) disposition an occurrence of the given key could
// inject with at this chapter — the max over its spoiler-VALID owner entries — and whether ANY
// valid owner exists (this second value subsumes the old validSuppressor: no valid owner ⇒ the
// occurrence injects nothing). CONFIRMED beats AMBIGUOUS. It is the trust the disposition-gate
// decides a containment contest on (D39 слой 4).
func (b *MemoryBank) matchTrust(m acMatch, chapter int) (disp injectionDisposition, hasValid bool) {
	key := b.ac.keys[m.keyIdx]
	for _, ei := range b.keyOwners[key] {
		e := &b.entries[ei]
		if spoilerBlocked(e, chapter) {
			continue
		}
		if d := dispositionFor(e, key); !hasValid || trustRank(d) > trustRank(disp) {
			disp, hasValid = d, true
		}
	}
	return disp, hasValid
}

// suppressContained applies longest-match / whole-entity replacement (A3), DISPOSITION-GATED (D39
// слой 4). It drops an occurrence fully contained inside a STRICTLY longer one, but ONLY when the
// longer one is BOTH (a) owned by a spoiler-VALID entry at this chapter (self-review #6: a spoiler-
// blocked longer key like 林动的父亲, until_ch=3, read at ch10, must not eat a valid nested 林动) AND
// (b) at least as TRUSTWORTHY as the nested match it would delete. A longer but LOWER-trust key
// (a draft/ambiguous 四代族长) must NOT suppress a nested HIGHER-trust key (an approved 族长): the draft
// is editor-excluded (renderEditorConstraintBlock is CONFIRMED-only), so eating the approved term
// leaves the reader with nothing AND the drop is invisible to post-check/retrieval-state — the exact
// term-drift code root the D38.5 seed-promote only patched by data (L3-recurrence). The whole-entity
// longest-match is preserved when the longer key is ≥ trust (approved 四代族长 still beats approved
// 族长; draft 四代族长 still eats a nested draft 族长). Each REFUSED suppression is recorded as a
// trustGateEvent so the near-silent drop is surfaced loudly (research/13 §7). Equal-length overlaps
// are both kept (genuine surface ambiguity). Deterministic (ms is sorted; events deduped by pair in
// ms order). O(n²) over the few matches in a chunk.
func (b *MemoryBank) suppressContained(ms []acMatch, chapter int) ([]acMatch, []trustGateEvent) {
	var kept []acMatch
	var gated []trustGateEvent
	for i, m := range ms {
		conDisp, conValid := b.matchTrust(m, chapter)
		contained := false
		var refused *trustGateEvent // a lower-trust valid longer key that WANTED to suppress m
		for j, o := range ms {
			if i == j {
				continue
			}
			// o must STRICTLY contain m's span to be a longest-match suppressor.
			if !(o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start)) {
				continue
			}
			supDisp, supValid := b.matchTrust(o, chapter)
			if !supValid {
				continue // spoiler-blocked longer key — never a valid suppressor (self-review #6)
			}
			if !conValid || trustRank(supDisp) >= trustRank(conDisp) {
				contained = true // a valid, ≥-trust container wins the whole-entity longest-match
				break
			}
			// supValid && conValid && supDisp < conDisp: a lower-trust longer key wants to eat a
			// higher-trust nested one — REFUSE (the L3 fix). Remember the FIRST such refusal for the
			// loud record; keep scanning in case a higher-trust container legitimately suppresses m.
			if refused == nil {
				refused = &trustGateEvent{
					Suppressor: b.ac.keys[o.keyIdx], SuppressorDisp: string(supDisp),
					Protected: b.ac.keys[m.keyIdx], ProtectedDisp: string(conDisp),
				}
			}
		}
		if contained {
			continue // m suppressed by the whole-entity longest-match; any refusal above was moot
		}
		kept = append(kept, m)
		if refused != nil {
			gated = append(gated, *refused) // m survived AND a lower-trust key tried to eat it — loud
		}
	}
	return kept, dedupeTrustGate(gated)
}

// dedupeTrustGate collapses identical (suppressor→protected) events to a distinct set, preserving
// first-occurrence (ms) order so the count and detail are deterministic and count distinct term
// collisions rather than raw occurrences.
func dedupeTrustGate(evs []trustGateEvent) []trustGateEvent {
	if len(evs) < 2 {
		return evs
	}
	seen := map[[2]string]bool{}
	var out []trustGateEvent
	for _, e := range evs {
		k := [2]string{e.Suppressor, e.Protected}
		if seen[k] {
			continue
		}
		seen[k] = true
		out = append(out, e)
	}
	return out
}

// suppressUnboundedPhonetic drops an occurrence of a SPACED-SCRIPT phonetic key (Latin or
// Cyrillic — scripts that delimit words with whitespace/non-letters, exactly like the
// target-side containsWholeWord) when a neighbouring rune is a letter of the SAME script:
// the key fired INSIDE a longer word ("rose" in "roseanne"/"roses"), not on the entity
// (D16.3). This closes the source-vs-target boundary asymmetry the ≤3 collision-downgrade
// does not (a ≥4 phonetic key fires CONFIRMED with no boundary today). A cross-script
// neighbour (a Latin name abutting a Han char in unspaced source) is a valid boundary, so
// only a SAME-script letter suppresses — mirroring "не-буквенных или чужескриптовых".
// KANA and HAN are deliberately NOT boundary-checked: they have no word segmentation, so a
// letter-boundary rule would false-negative the ubiquitous kana name+particle case (すずきは)
// or contradict the long-key-CONFIRMED contract — their ≥4 precision is deferred to the
// kana-precision measurement + the B6 tokenizer (D16; flagged to the orchestrator). ntext is
// the already-normalized chunk. O(n) over the few matches in a chunk.
func (b *MemoryBank) suppressUnboundedPhonetic(ms []acMatch, ntext []rune) []acMatch {
	var kept []acMatch
	for _, m := range ms {
		script := spacedPhoneticScript(b.ac.keys[m.keyIdx])
		if script != nil {
			beforeBad := m.start > 0 && letterInScript(ntext[m.start-1], script)
			afterBad := m.end < len(ntext) && letterInScript(ntext[m.end], script)
			if beforeBad || afterBad {
				continue // the key is inside a longer same-script word — not a whole-entity match
			}
		}
		kept = append(kept, m)
	}
	return kept
}

// spacedPhoneticScript returns the word-delimited alphabetic script a normalized key belongs
// to for the source word-boundary check — unicode.Latin or unicode.Cyrillic — or nil when the
// key carries a Han ideograph or kana (no word segmentation) or mixes the two spaced scripts.
// Digits/punctuation don't set the script but don't disqualify (so "o'brien" is still Latin).
func spacedPhoneticScript(normKey string) *unicode.RangeTable {
	var script *unicode.RangeTable
	for _, r := range normKey {
		switch {
		case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul):
			return nil // an ideographic/kana anchor is never boundary-checked here
		case unicode.In(r, unicode.Latin):
			if script == unicode.Cyrillic {
				return nil // mixed spaced scripts — do not boundary-check
			}
			script = unicode.Latin
		case unicode.In(r, unicode.Cyrillic):
			if script == unicode.Latin {
				return nil
			}
			script = unicode.Cyrillic
		}
	}
	return script
}

// letterInScript reports whether r is a LETTER of the given spaced script — the "same-script
// letter" that breaks a word boundary. A digit, punctuation, space, or a letter of another
// script is a valid boundary and returns false.
func letterInScript(r rune, script *unicode.RangeTable) bool {
	return unicode.IsLetter(r) && unicode.In(r, script)
}

// injectivityCollisions reports approved dst collisions (B2): two distinct source
// terms mapped to the SAME dst (one Russian surface for two entities → the reader
// cannot tell them apart), returned as human-readable strings for a load-time warning.
// A pure diagnostic; it does not reject (some collisions are legitimate, e.g. a title
// shared by rank tiers), so the caller logs it, not aborts.
func injectivityCollisions(rows []store.GlossaryEntry) []string {
	bySurface := map[string][]string{}
	for _, r := range rows {
		if r.Status != "approved" || strings.TrimSpace(r.Dst) == "" {
			continue
		}
		key := normalizeTargetForm(r.Dst)
		bySurface[key] = append(bySurface[key], r.Src)
	}
	var out []string
	// Deterministic order: iterate a sorted key list, not the map.
	keys := slices.Sorted(maps.Keys(bySurface))
	for _, k := range keys {
		srcs := bySurface[k]
		if len(distinct(srcs)) > 1 {
			sort.Strings(srcs)
			out = append(out, "dst "+strconv.Quote(k)+" ← "+strings.Join(distinct(srcs), ", "))
		}
	}
	return out
}

func distinct(ss []string) []string {
	seen := map[string]bool{}
	var out []string
	for _, s := range ss {
		if !seen[s] {
			seen[s] = true
			out = append(out, s)
		}
	}
	sort.Strings(out)
	return out
}