// Package membank is the book's deterministic memory bank: the frozen glossary materialized into a // matcher, the per-chunk retrieval that decides WHICH records a model sees, the injection renderers // that turn them into wire text, and the post-check that verifies the rendering came back. // // The hot path is deterministic by construction — an exact multi-pattern match (Aho-Corasick) over the // normalized chunk, no vectors, no FTS, no LLM, no clock — because the injected block is folded into // request_hash: the same chunk under the same bank must produce the same bytes on resume, or a paid // checkpoint is silently invalidated. Bank content is versioned (Version/BaseVersion) so a glossary // edit is a loud --resnapshot rather than a silent change of what the model was told. // // It owns the seed side of that contract too (loading, validating and colliding-checking the seed // YAML), and holds no run state: it takes rows and text, returns a selection. package membank import ( "crypto/sha256" "encoding/hex" "encoding/json" "maps" "slices" "sort" "strconv" "strings" "unicode" "textmachine/backend/internal/lang" "textmachine/backend/internal/store" "textmachine/backend/internal/text" ) // memory.go: the DETERMINISTIC hot path of the memory bank v2 (registry §Hot-path // Contract / research/13 Q3). $0, no LLM, no embeddings, no FTS5 — a multi-pattern // exact match (Aho-Corasick) of glossary keys/aliases over the NORMALIZED chunk, with // the single-key ban (A3), longest-match whole-entity replacement, the spoiler // hard-reject window (C1), sticky scene-inertia (A5), a priority token budget with a // logged eviction (F2), and the three-way injection disposition (A2). Pure and // deterministic (no map-order output, no time/rand): the injected block is a pure // function of (frozen bank, normalized chunk, chapter, sticky, budget), and it enters // request_hash via the rendered messages — so a resumed chunk reproduces it for free. // // The registry's principle drives every choice here: "better nothing than garbage" and // PRECISION over recall. A wrong injection (a homograph / short alias firing on the // wrong sense) is the MAIN source of silent degradation (2510.00829) — worse than an // empty match, which is a safe fallback to the model's base behaviour. So the matcher // errs toward NOT firing, and the post-check (memcheck) makes what did fire observable. // matchVersion versions the matcher + injection-selection ALGORITHM (min-key // ban, longest-match containment, spoiler gate, sticky, budget priority order). Folded // into memoryVersion() → a change is a loud --resnapshot (it shifts the injected bytes // → the wire → a resumed checkpoint's content). Sibling of text.NormVersion(). // v3 (D16.2/D16.3): sticky now INHERITS the original match's disposition instead of // recomputing it status-based (a collision-prone AMBIGUOUS carry no longer silently // upgrades to CONFIRMED), and a spaced-phonetic (Latin/Cyrillic) source key is validated // against a word boundary (approved "rose" no longer fires inside "roseanne"). // v4 (D39 layer 4): the longest-match suppressor is now DISPOSITION-GATED (suppressContained) — // a longer but LOWER-trust key (draft/ambiguous) no longer deletes a nested HIGHER-trust key // (approved/confirmed), closing the term-drift code root the D38.5 seed-promote only patched by // data (L3-suppressor-disposition-blind). A refused suppression is recorded loudly (trustGated). // The injection SELECTION changes for a draft-longer-over-approved-shorter chunk (the approved term // now survives), so a loud --resnapshot + golden re-capture (invariant №8). const matchVersion = "memmatch-v4-perlang-minkey+collision-ambiguous+longest-trustgated+spoiler+sticky-inherit-disp+phonetic-srcboundary+budget+postcheck-declaware" // minKeyLenHan / minKeyLenPhonetic are the per-language single-key floors (A3, registry // "min_key_len per-language"). An ideographic (Han) key is semantically distinct at 2 chars // (empirically precision-1.0 on the zh retrieval_bench homograph traps). A PURELY PHONETIC // key (kana/latin/cyrillic, no Han anchor) is collision-prone at 2 — short phonetic // sequences appear INSIDE ordinary words (リン in リンゴ/apple, AI in RAID) — so it needs ≥3. // external-review major #3: the MIN=2 measurement was on zh-Han and does NOT transfer to // ja-kana; 3 is a conservative default pending a ja-kana precision measurement (an E1- // protocol extension). A per-entry allow_short overrides both (rare, guarded). const ( minKeyLenHan = 2 minKeyLenPhonetic = 3 ) // minKeyLenFor is the floor for a normalized key: any Han ideograph anchors it (2), an // all-phonetic key needs 3. func minKeyLenFor(normKey string) int { if runesAnyHan(normKey) { return minKeyLenHan } return minKeyLenPhonetic } // collisionProneKey reports whether a key is short AND purely phonetic (no Han anchor), // so it is likely to fire inside an unrelated word — the A2 "surface collision" case, // injected as AMBIGUOUS (unverified) rather than authoritatively CONFIRMED. func collisionProneKey(normKey string) bool { return !runesAnyHan(normKey) && text.SignificantLen(normKey) <= minKeyLenPhonetic } // InjectionTrust is what a record's FIRING KEY established about it, and it is TWO facts rather than // one. The disposition folds them together — an AMBIGUOUS record is one whose row is unsigned, OR one // whose key may have fired inside an unrelated word, or both — and folding was right while the two led // to the same place. They no longer do: D39.104 п.1 made the bank LAW on the wire regardless of a row's // STATUS, so a directive may no longer be withheld for being unsigned, while withholding it from a // doubtful MATCH is the opposite of pedantry (a gender instruction attached to the wrong entity is // actively wrong, not merely redundant). The two questions therefore travel separately. type InjectionTrust struct { Disp InjectionDisposition // KeyTrusted is the MATCH half: the key that fired is not collision-prone, or the author declared // the surface safe (allow_short). False says «this record may not be about the entity on the page». KeyTrusted bool } // UnionSticky merges the recent chunks' exact-matched ids into one sticky_prev, carrying // what each id FIRED AS (D16.2). When an id fired in several recent chunks, the MOST RECENT // firing wins (win is ordered oldest→newest), so a re-established CONFIRMED match overrides // an older collision-prone AMBIGUOUS one — and vice-versa, never silently upgrading. func UnionSticky(win []map[string]InjectionTrust) map[string]InjectionTrust { if len(win) == 0 { return nil } out := map[string]InjectionTrust{} for _, s := range win { for id, tr := range s { out[id] = tr } } return out } // StickyDepth is the scene-inertia window (A5): entries exact-matched in the previous // N chunks of the SAME chapter are carried into a pronominal chunk that names nobody. // A code rule (Р2), reset at each chapter boundary (a new chapter is a scene change). const StickyDepth = 2 // InjectionDisposition is the three-way per-record decision (registry A2) — the core // mechanism that converts silent degradation into loud. Distinct from the chunk×stage // Disposition (disposition.go): that is ok|flagged|skipped for a completion; this is // how much to TRUST an injected glossary record. type InjectionDisposition string const ( Confirmed InjectionDisposition = "confirmed" // exact key/alias, approved, spoiler-valid → authoritative Ambiguous InjectionDisposition = "ambiguous" // auto/draft term → inject "unverified" + forced post-check Reject InjectionDisposition = "reject" // spoiler-window violation → dropped and logged (C1 safety) ) // declInfo is the parsed decl column: the dst declension forms the post-check accepts. type declInfo struct { Invariant bool `json:"invariant"` Forms []string `json:"forms"` } // entry is one frozen, materialized glossary entry with its matchable keys // pre-normalized. Immutable for a job. type entry struct { id string // stable unique id: src\x1f sense\x1f since\x1f until (= the UNIQUE key) src string // raw source key dst string // raw approved translation (may be "" for a ruby candidate) status string // auto|draft|approved sense string gender string // male|female|hidden|"" (C2) — feeds the DC3 gender constraint on the editor block sinceCh int untilCh int // normKeys are the normalized source surfaces (src + aliases) ELIGIBLE to fire // (text.SignificantLen ≥ minKeyLen OR allow_short). A key too short is dropped here, so // it is never in the automaton — the single-key ban is structural, not a runtime skip. normKeys []string // declForms are the normalized-target dst forms the post-check accepts. Empty → // the post-check falls back to the single normalized dst (the naive base form that // research/14 shows false-flags on inflection — the reason full decl matters). declForms []string declInvariant bool // allowShort is the author's explicit override of the single-key ban AND the // collision-prone disposition downgrade (full trust in a short key). allowShort bool } // Bank is a book's frozen glossary materialized for one job: the entries, the // Aho-Corasick automaton over all eligible keys, and the F1 memoryVersion. Built once // (Materialize) before the chunk loop and never mutated. type Bank struct { entries []entry ac *ahoCorasick keyOwners map[string][]int // normalized key → indices of entries that contributed it // enrichedVersion folds ALL foldable rows incl Source:mined — the memory version of the EDIT // wave (edit-wave snapshot). baseVersion EXCLUDES Source:mined — the memory version of the DRAFT wave // (draft-wave snapshot), so a bank-mining stop's enrichment (adding mined rows) moves ONLY the edit-wave snapshot, keeping // draft-wave checkpoints valid (WS1 §1в, «re-paid ONCE»). The two are DOMAIN-SEPARATED (a "base-excl- // mined" tag in the base fold), so base ≠ enriched for EVERY row set — even a mined-free book (a // the draft wave job must never content-address to an edit wave checkpoint). The load-bearing invariant is not // equality but STABILITY: base stays fixed as mined rows are added; only enriched moves. enrichedVersion string baseVersion string // voices/pairs are the pack-19 record types. They are bank CONTENT but never matchable SURFACES, so // they sit beside the automaton rather than inside it: nothing here can put them in keyOwners. voices []store.VoiceProfile pairs []store.AddressPair // stemmer is the target-language decl stemmer (bank-quality §3): it lets the post-check accept an oblique // case of a term against its nominative without the seed listing every form. The zero value is inert (a // target with no decl_suffix registry), so a book that ships none keeps exact-match behaviour. stemmer lang.TargetStemmer } // PickedEntry is one selected record for a chunk with its firing key and disposition. type PickedEntry struct { entry *entry // the bank row (opaque outside the bank) // Via is the normalized key this record fired on. It is EMPTY for a sticky carry — see Sticky, // which is the fact callers actually branch on. Via string // Sticky marks a scene-inertia carry (A5): the record did NOT fire in this chunk, it was carried // from the sticky window. It is a typed field rather than the old via=="sticky" sentinel, which was // indistinguishable from a real key literally spelled "sticky" and made every consumer re-know the // magic string (the post-check skips sticky records, the retrieval-state counts them, the budget // ranks them last). Sticky bool Disp InjectionDisposition // the trust the record was injected with // KeyTrusted is the MATCH half of that trust, kept apart from Disp because the wire now asks the // two questions separately — see InjectionTrust. Carried forward unchanged by a sticky record. KeyTrusted bool } // valid reports whether this record actually carries a bank row. PickedEntry is exported WITH exported // fields, so a caller outside this package can construct a zero value whose private row is nil; every // consumer here would then nil-deref. Rather than document a landmine, the consumers skip such a record: // a record with no row has nothing to inject and nothing to post-check. Selection always produces rows, so // no real path changes (pack-16 tail). func (p PickedEntry) valid() bool { return p.entry != nil } // Src is the bank row's SOURCE surface — the only thing about a record a caller outside the bank can NAME. // It exists because n_evicted counted budget drops without ever saying WHICH rows the model did not get to // see, and a bare count is not something an operator can act on. Empty for a zero value. func (p PickedEntry) Src() string { if p.entry == nil { return "" } return p.entry.src } // TrustGateEvent records a longest-match suppression the DISPOSITION-gate REFUSED (D39 layer 4): // a longer but LOWER-trust key (draft/ambiguous, e.g. draft 四代族长) that would have deleted a nested // HIGHER-trust key (approved/confirmed, e.g. approved 族长). The draft is editor-excluded (the block // is CONFIRMED-only), so suppressing the approved term would have left the reader with NOTHING — the // term-drift code root (L3). The nested higher-trust match survives and injects; this record turns // the near-silent drop into a loud, operator-visible signal (research/13 §7). A non-empty list means // the seed has a draft term nesting over an approved one — reconcile the seed so it stops recurring. type TrustGateEvent struct { Suppressor string `json:"suppressor"` // the longer, lower-trust key blocked from suppressing SuppressorDisp string `json:"suppressor_disp"` // its best injection disposition (ambiguous) Protected string `json:"protected"` // the nested, higher-trust key it would have eaten ProtectedDisp string `json:"protected_disp"` // its best injection disposition (confirmed) } // Selection is the hot path's output for one chunk. type Selection struct { Injected []PickedEntry // in budget priority order (what the model sees) Rejected []PickedEntry // spoiler-window rejects (C1, logged) Evicted []PickedEntry // dropped by the token budget (F2, logged) // trustGated are the longest-match suppressions the disposition-gate refused (a lower-trust // longer key would have eaten a higher-trust nested one) — the term-drift code root, surfaced // loudly instead of a silent drop (research/13 §7). Deterministic (ms order, deduped by pair). TrustGated []TrustGateEvent // activeIDs are the exact-matched ids (NOT sticky) → the next chunk's sticky_prev, // each mapped to what it FIRED AS. Sticky carries that forward instead of recomputing it // (D16.2): a sticky carry has no firing key, so it cannot re-derive the collision-prone // downgrade, and recomputing from status alone would silently upgrade a collision-prone // AMBIGUOUS match to CONFIRMED. The MATCH half rides here for the same reason and no other: // it is equally underivable without the key, and the draft wire now gates a directive on it. ActiveIDs map[string]InjectionTrust } // Version is the ENRICHED memory version (all approved rows incl mined) — the edit-wave / whole-book // memory component. BaseVersion excludes Source:mined (the draft-wave component). They are // DOMAIN-SEPARATED — base ≠ enriched for ANY row set, including a mined-free book (never assert // equality) — but base stays STABLE when a bank-mining stop adds mined rows, which is what keeps the draft wave // checkpoints valid («re-paid ONCE»). func (b *Bank) Version() string { return b.enrichedVersion } func (b *Bank) BaseVersion() string { return b.baseVersion } // BankInput is everything a book's bank is materialized and hashed from: the glossary rows plus the two // pack-19 record types. It exists so the fold has ONE argument that can grow, and so the separation that // matters is carried by the type system: only Rows ever reaches the matcher, so a voice profile cannot // become a matchable term by anyone forgetting a filter. type BankInput struct { Rows []store.GlossaryEntry Voices []store.VoiceProfile Pairs []store.AddressPair // InjectVoice reports whether the voice/address rows reach the WIRE on this run. It is the CONDITION // of their fold: while they only feed the $0 flagger (which recomputes every run and self-heals), // hashing them would re-bill a book for authoring a profile that changed no byte the model sees. // The moment they are injected, they must fold — the D39.42 п.3 class, in the other direction. InjectVoice bool // TargetStemmer is the decl-aware post-check stemmer (bank-quality §3). Zero value → inert (exact match // only), so a caller that supplies none keeps the pre-§3 behaviour byte-for-byte. TargetStemmer lang.TargetStemmer } // Materialize builds a Bank from the book's stored glossary rows alone — the pre-pack-19 form, kept for // every caller that has no voice/address content. func Materialize(rows []store.GlossaryEntry, gateOn bool) *Bank { return MaterializeBank(BankInput{Rows: rows}, gateOn) } // MaterializeBank builds a Bank from the book's stored bank content (ORDER BY-stable — GlossaryForBook / // VoiceProfilesForBook / AddressPairsForBook). Pure and deterministic. It computes memoryVersion as a // content hash of the frozen APPROVED rows (D8: content-hash, not a version-counter — // drift-proof) plus the normalization + matcher algorithm versions, so ANY change to // the approved glossary OR to the deterministic machinery is a loud --resnapshot (F1). func MaterializeBank(in BankInput, gateOn bool) *Bank { rows := in.Rows b := &Bank{keyOwners: map[string][]int{}, voices: in.Voices, pairs: in.Pairs, stemmer: in.TargetStemmer} var allKeys []string seenKey := map[string]bool{} for _, row := range rows { e := entry{ id: row.Src + "\x1f" + row.Sense + "\x1f" + strconv.Itoa(row.SinceCh) + "\x1f" + strconv.Itoa(row.UntilCh), src: row.Src, dst: row.Dst, status: row.Status, sense: row.Sense, gender: row.Gender, sinceCh: row.SinceCh, untilCh: row.UntilCh, allowShort: row.AllowShort, } // Eligible source surfaces (src + aliases) under the single-key ban. A record // with NO dst yet (a ruby auto-candidate) is NOT matchable: it renders nothing // (RenderGlossaryBlock/postcheck both skip empty dst), so admitting its keys would // let it consume the token budget, evict a renderable line, and inflate the // retrieval-state exact-hit count for a record the model never sees (self-review // #1). It stays in the store for future promotion, just inert on the hot path. var surfaces []string if strings.TrimSpace(row.Dst) != "" { surfaces = append(surfaces, row.Src) for _, a := range row.Aliases { surfaces = append(surfaces, a.Alias) } } for _, s := range surfaces { nk := text.NormalizeSourceKey(s) if nk == "" { continue } if text.SignificantLen(nk) < minKeyLenFor(nk) && !row.AllowShort { continue // A3: a too-short key never fires on its own (per-language floor) } e.normKeys = append(e.normKeys, nk) } // Parse decl forms for the post-check (normalized target side). if row.Decl != "" { var d declInfo if err := json.Unmarshal([]byte(row.Decl), &d); err == nil { e.declInvariant = d.Invariant for _, f := range d.Forms { if nf := text.NormalizeTargetForm(f); nf != "" { e.declForms = append(e.declForms, nf) } } } } idx := len(b.entries) b.entries = append(b.entries, e) for _, nk := range e.normKeys { b.keyOwners[nk] = append(b.keyOwners[nk], idx) if !seenKey[nk] { seenKey[nk] = true allKeys = append(allKeys, nk) } } } sort.Strings(allKeys) // deterministic automaton construction b.ac = buildAC(allKeys) b.enrichedVersion = ComputeVersionScopedIn(in, gateOn, false) // all approved incl mined (the edit wave) b.baseVersion = ComputeVersionScopedIn(in, gateOn, true) // excl Source:mined (the draft wave) return b } // Voices / Pairs return the book's voice profiles and address-register journal (frozen for the job). // Read-only: a caller projects them onto a chapter, never mutates them. // // ⚠ NEITHER HAS A CALLER TODAY, and both are kept: they are the read side of state the bank already loads // and holds, waiting on the consumer backlog row 13б describes. Deleting them would not shrink the load — // the fields stay — it would only make the held state unreachable. func (b *Bank) Voices() []store.VoiceProfile { return b.voices } func (b *Bank) Pairs() []store.AddressPair { return b.pairs } // ComputeVersion is the F1 content-hash: the frozen rows (ORDER BY-stable) plus the // normalization and matcher algorithm versions. EVERY row folds whatever its status, and // its whole content (incl. decl and status) with it — see the loop below for why the // approved-only fold of D8/§8 was retired by pack-20 (D39.42 п.3). gateOn is folded as a // FIELD rather than as a scope switch: it changes how the same rows resolve a chunk's // disposition, so the two gate settings must not share a hash. The glossary.id // autoincrement is deliberately EXCLUDED (fresh each replace); only content columns are hashed. func ComputeVersion(rows []store.GlossaryEntry, gateOn bool) string { return ComputeVersionScoped(rows, gateOn, false) } // ComputeVersionScoped is ComputeVersion with an explicit Source-scope: excludeMined // drops every Source:"mined" row from the fold (WS1 §1в base-bank-version). The DRAFT-wave snapshot // (draft-wave snapshot) folds base (excludeMined=true) so a bank-mining stop that ADDS mined rows moves ONLY the // enriched (edit-wave) version — keeping draft-wave checkpoints valid, «re-paid ONCE». Deterministic (a // Source filter over the same ORDER BY-stable rows). excludeMined=false is BYTE-IDENTICAL to the // pre-split fold (no extra field), so the enriched hash / existing snapshots do not move; the // excludeMined=true variant adds a domain separator so base and enriched never collide. func ComputeVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool) string { return ComputeVersionScopedIn(BankInput{Rows: rows}, gateOn, excludeMined) } // ComputeVersionScopedIn is ComputeVersionScoped over the whole bank input — the form that also folds the // pack-19 voice/address rows, CONDITIONALLY (BankInput.InjectVoice). // // The condition is the money contract of pack-19. While those rows only feed the $0 flagger they change // no byte the model sees, and a book must not re-pay a wave for authoring a profile; the flagger's // counters live in retrieval_state, which is recomputed every run and self-heals. The moment they are // injected they DO change the wire, and then not folding them would reopen exactly the class D39.42 п.3 // closed — a wire change the snapshot cannot see, so projectRebill projects $0 for a real re-payment. // A book with no voice rows, or a run with the injection off, hashes BYTE-IDENTICALLY to before this // existed: nothing is written at all. func ComputeVersionScopedIn(in BankInput, gateOn, excludeMined bool) string { rows := in.Rows h := sha256.New() h.Write([]byte("tm-memory-v2\x00")) h.Write([]byte(text.NormVersion() + "\x00" + matchVersion + "\x00")) h.Write([]byte("gate:" + strconv.FormatBool(gateOn) + "\x00")) hasUnverified := false hasNeuter := false // a neuter row now RENDERS a directive (bank-quality §3) where it rendered nothing before hasUnfit := false // a row the wire fence stops now renders NOTHING where it used to render itself // inScope collects the characters this fold's ROW scope actually contains, so the voice/address fold // below can apply the SAME scope before deciding anything (see the loop at the end). inScope := map[[2]string]bool{} if excludeMined { h.Write([]byte("base-excl-mined\x00")) // domain separator: base ≠ enriched even over identical rows } for _, r := range rows { // EVERY row folds, whatever its status (pack-20 / D39.42 п.3). Before pack-20 an auto/draft row // was skipped unless the hard gate was on, on the reasoning that its injected content was caught // per chunk by content_hash. That reasoning had a hole the phase-1 sync measured: an unverified // row changes the rendered injection, hence content_hash, hence the resume fast-path — but NOT the // snapshot, and projectRebill projects re-payment from the SNAPSHOT alone, so the chunks it // silently re-translated never reached the Р6 consent contour. Folding every row closes that // class by construction rather than by a second guard: a bank edit of ANY status is now a loud // --resnapshot. A book whose bank is entirely approved hashes byte-identically to before, so // shipping this re-bills nobody who had nothing unverified. // if excludeMined && r.Source == "mined" { continue } if r.Status != "approved" { hasUnverified = true } if r.Gender == "neuter" || r.Gender == "n" { hasNeuter = true } if WireUnfitRow(r.Src, r.Dst) { hasUnfit = true } inScope[[2]string{r.Src, r.Sense}] = true // A fixed, length-prefixed field layout so no content can forge a boundary. writeField(h, r.Status) writeField(h, r.Src) writeField(h, r.Dst) writeField(h, r.Sense) writeField(h, r.Type) writeField(h, r.Gender) writeField(h, r.Decl) writeField(h, strconv.Itoa(r.SinceCh)) writeField(h, strconv.Itoa(r.UntilCh)) writeField(h, strconv.FormatBool(r.AllowShort)) writeField(h, strconv.Itoa(len(r.Aliases))) for _, a := range r.Aliases { // GlossaryForBook returns aliases ORDER BY alias writeField(h, a.Alias) writeField(h, a.AliasType) } } // The RENDER revision of the UNSIGNED ROWS, folded only for a scope that actually HAS one. ⚠ The // literal below is FROZEN BYTES, not a description: it was named for the separately-headed section // pack-20 gave those rows, and row 134 has since folded that section into the one law block — but the // condition is unchanged (a bank with an unsigned row renders differently from one without), and // renaming the token would move memory_version for every such book to say nothing new. Folding every row (above) closes the "which rows exist" hole, but not this one: // pack-20 changed how those rows are RENDERED to the editor (their own labelled section), and for a // book whose post-check gate was already ON the identical row set hashes identically — same version, // same snapshot, different wire bytes. The resume path would then re-translate on the content hash // while projectRebill, which skips a unit whose snapshot is unchanged (rebill.go), projects $0: a // silent re-payment outside the Р6 consent contour, the exact class folding every row was meant to // close. A book with nothing unverified renders no such section and hashes byte-identically to before, // so this re-bills nobody who had nothing to re-render — which a blanket RenderFormatVersion bump, // being un-re-pinnable (it is not a bank-only move), would have done to every book on earth. if hasUnverified { h.Write([]byte("editor-unverified-section-v1\x00")) } // The same scoped-render pattern for the §3 neuter directive: gender:neuter was a SILENT no-op before, // so a book carrying one renders new wire bytes on the same row set — the editor-unverified hole again. // Folded ONLY for a scope that HAS a neuter row, so a bank without one is byte-identical to before and // re-bills nobody; a neuter-bearing book takes a loud, RE-PINNABLE --resnapshot (bank-only move). if hasNeuter { h.Write([]byte("neuter-directive-v1\x00")) } // The wire fence, folded by the SAME scoped pattern and for the same reason (wirefence.go). A row the // fence stops used to render itself into the system block and now renders nothing, so the wire moves // on an unchanged row set — the editor-unverified hole a third time. Folded ONLY for a scope that // HOLDS such a row: a bank whose every value is fit hashes byte-identically to before this existed and // re-bills nobody, while a book that really did carry a poisoned row takes a loud, re-pinnable // --resnapshot rather than a silent re-payment outside the Р6 consent contour. if hasUnfit { h.Write([]byte("wire-fence-v1\x00")) } // The pack-19 record types, appended AFTER everything above so a book without them is byte-identical // to the pre-pack-19 hash by construction, not by argument. // // The two loops copy the order the row loop above uses and for the same reason: the SCOPE exclusion is // applied FIRST, and only a row that survived it may set the tag condition. A profile whose character // is not in this scope (its term is Source:mined and this is the base/draft fold) describes somebody // the wave never sees — folding it would move the draft version when a mined-only character's profile // is edited, which is precisely the base/enriched separation the mined filter exists to keep. hasVoice := false for _, v := range in.Voices { if !in.InjectVoice || !inScope[[2]string{v.Src, v.Sense}] { continue } hasVoice = true for _, f := range []string{v.Src, v.Sense, v.Register, v.SelfRef, v.AddressDefault, v.LexiconMarkers, v.NGLexicon, v.Exemplars, v.Brightness, strconv.Itoa(v.SinceCh), strconv.Itoa(v.UntilCh)} { writeField(h, f) } } for _, p := range in.Pairs { if !in.InjectVoice || !inScope[[2]string{p.SpeakerSrc, p.SpeakerSense}] || !inScope[[2]string{p.AddresseeSrc, p.AddresseeSense}] { continue } hasVoice = true for _, f := range []string{p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense, p.Register, p.Form, p.Closeness, strconv.Itoa(p.SinceCh), strconv.Itoa(p.UntilCh)} { writeField(h, f) } } if hasVoice { h.Write([]byte("voice-address-v1\x00")) } return hex.EncodeToString(h.Sum(nil)) } func writeField(h interface{ Write([]byte) (int, error) }, s string) { var lb [8]byte n := uint64(len(s)) for i := 0; i < 8; i++ { lb[i] = byte(n >> (8 * i)) } h.Write(lb[:]) h.Write([]byte(s)) } // glossaryLineTokens is the injected token cost of one record's "src → dst" line — the // unit the token budget (config glossary_token_budget) spends. It is FLOOR-FREE (the raw // cjk + other/3 estimate, NOT EstimateTokens' 16-token minimum): the floor is for sizing // a whole request's max_tokens, but applying it PER glossary line ~1.85×-overcounts a // block of short name lines and needlessly evicts records that fit (self-review #7). The // header's cost is small and constant; omitting it keeps the unit purely per-record. // Shared by Select's budget and the eviction test so the two never diverge. func glossaryLineTokens(e *entry) int { cjk, other := text.DenseSparseCounts(e.src + " → " + e.dst) // same sizing taxonomy as EstimateTokens return cjk + other/3 } // Select is the hot path for ONE chunk: match → spoiler-reject → disposition → sticky // → priority token budget. Pure and deterministic. stickyPrev is the set of ids // exact-matched in the prior chunk(s) of the same chapter (A5). budgetTokens ≤ 0 means // unbounded; otherwise records are kept in priority order while the cumulative injected // token cost stays within budget — the rest are EVICTED (dropped but logged, F2; the // budget may be underfilled, "better nothing than garbage"). func (b *Bank) Select(chunk string, chapter int, stickyPrev map[string]InjectionTrust, budgetTokens int) Selection { // ZERO-VALUE GUARD (pack-16 tail): Bank is an exported type, so `var b membank.Bank` is constructible // outside this package, and its matcher would then be nil — a panic on the first non-empty chunk, in a // PAID run. A bank with no matcher has nothing to match, so the honest answer is an empty selection. // Materialize always builds the matcher, so this can never change the behaviour of a real bank. if b == nil || b.ac == nil { return Selection{} } ntext := []rune(text.NormalizeSourceKey(chunk)) occ := b.ac.matches(ntext) // Source word-boundary for spaced-phonetic keys (D16.3): drop an occurrence of a Latin/ // Cyrillic key that fired INSIDE a longer word of the same script ("rose" in "roseanne"). // Closes the source-vs-target boundary asymmetry (the target side already has // containsWholeWord). Han/kana are NOT boundary-checked (no word segmentation) — deliberate. occ = b.suppressUnboundedPhonetic(occ, ntext) // Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when the // longer key belongs to a spoiler-VALID entry at this chapter (self-review #6) AND is at least // as TRUSTWORTHY as the nested match it would delete (D39 layer 4 — a lower-trust longer key must // not eat a higher-trust nested one). Refused suppressions are returned for a loud record. var trustGated []TrustGateEvent occ, trustGated = b.suppressContained(occ, chapter) // Map surviving key occurrences → entries, recording the LONGEST firing key per entry. matchedVia := map[int]string{} for _, m := range occ { k := b.ac.keys[m.keyIdx] for _, ei := range b.keyOwners[k] { if cur, ok := matchedVia[ei]; !ok || len([]rune(k)) > len([]rune(cur)) { matchedVia[ei] = k } } } sel := Selection{ActiveIDs: map[string]InjectionTrust{}, TrustGated: trustGated} hits := map[string]PickedEntry{} // id → picked (exact) // Iterate entries in their frozen (ORDER BY-stable) order — never map order. for ei := range b.entries { via, ok := matchedVia[ei] if !ok { continue } e := &b.entries[ei] if blocked := spoilerBlocked(e, chapter); blocked { sel.Rejected = append(sel.Rejected, PickedEntry{entry: e, Via: via, Disp: Reject}) continue } disp := dispositionFor(e, via) keyOK := keyTrusted(e, via) hits[e.id] = PickedEntry{entry: e, Via: via, Disp: disp, KeyTrusted: keyOK} // Exact-matched ids feed the next chunk's sticky WITH what they fired as (D16.2). sel.ActiveIDs[e.id] = InjectionTrust{Disp: disp, KeyTrusted: keyOK} } // Sticky scene-inertia: carry prior-chunk exact matches not re-matched here, unless // the spoiler window blocks them (spoiler beats sticky). The carry INHERITS the prior // match's disposition (D16.2) — a sticky carry has no firing key, so it cannot re-derive // the collision-prone downgrade; recomputing status-based would silently upgrade a // collision-prone AMBIGUOUS match to CONFIRMED, past the post-check, into the editor // constraints. An AMBIGUOUS carry stays AMBIGUOUS (a forced post-check; unsigned in the editor). // The MATCH half rides along for exactly the same reason: a doubtful key stays doubtful when it is // carried, and the draft wire withholds the gender directive from it. for ei := range b.entries { e := &b.entries[ei] carried, sticky := stickyPrev[e.id] if !sticky { continue } if _, already := hits[e.id]; already { continue } if spoilerBlocked(e, chapter) { // RECORDED, not dropped (pack-19): a window-blocked carry is a spoiler reject like any other, // and a silent drop makes n_spoiler_blocked under-count. Unreachable through the production // driver — precomputeSticky resets the window at every chapter boundary, so a carry always // arrives at the SAME chapter it fired in and cannot have become blocked — but Select is // exported and the invariant that protects it lives in another package. sel.Rejected = append(sel.Rejected, PickedEntry{entry: e, Sticky: true, Disp: Reject}) continue } hits[e.id] = PickedEntry{entry: e, Sticky: true, Disp: carried.Disp, KeyTrusted: carried.KeyTrusted} } // Deterministic priority order for the budget: confirmed>ambiguous, exact>sticky, // approved>auto. Collect in frozen-entry order (NOT map order) then stable-sort, so // ties keep the deterministic base order. order := make([]PickedEntry, 0, len(hits)) for ei := range b.entries { if p, ok := hits[b.entries[ei].id]; ok { order = append(order, p) } } sort.SliceStable(order, func(i, j int) bool { return priorityRank(order[i]) < priorityRank(order[j]) }) if budgetTokens > 0 { used, cut := 0, len(order) for i := range order { used += glossaryLineTokens(order[i].entry) if used > budgetTokens { cut = i // this record and everything after it overflow the budget break } } sel.Injected = order[:cut] sel.Evicted = order[cut:] // F2: dropped, but LOGGED via the retrieval-state } else { sel.Injected = order } return sel } // spoilerBlocked reports whether the entry's since_ch/until_ch window excludes this // chapter (C1: a hard reject gate — a fact the current chapter must not know yet). func spoilerBlocked(e *entry, chapter int) bool { if e.sinceCh > 0 && chapter < e.sinceCh { return true } if e.untilCh > 0 && chapter > e.untilCh { return true } return false } // dispositionFor maps a term + its firing key to an injection disposition (A2). approved → // CONFIRMED (authoritative); auto/draft → AMBIGUOUS. AND: even an approved entry matched by // a COLLISION-PRONE key (short + purely phonetic — リン in リンゴ, AI in RAID) is downgraded to // AMBIGUOUS (unverified + forced post-check), because such a key may have fired inside an // unrelated word rather than on the entity (external-review major #2 — the A2 // "surface-collision → AMBIGUOUS" branch). allow_short is the author's explicit override. // Called ONLY for an EXACT match (a real firing key): a sticky carry INHERITS its prior // disposition instead (D16.2), never routing through here, so it cannot silently upgrade a // collision-prone match. func dispositionFor(e *entry, via string) InjectionDisposition { if !keyTrusted(e, via) { return Ambiguous } if e.status == "approved" { return Confirmed } return Ambiguous } // keyTrusted is the MATCH half of dispositionFor, lifted out so the two questions the disposition folds // together can be asked one at a time (InjectionTrust). It answers only «is this record about the entity // on the page» — a short purely-phonetic key (リン in リンゴ, AI in RAID) may have fired inside an // unrelated word — and says nothing about whether anybody signed the rendering. allow_short is the // author's explicit override and stays exactly that. func keyTrusted(e *entry, via string) bool { return e.allowShort || !collisionProneKey(via) } // priorityRank orders the budget: confirmed before ambiguous, exact before sticky, // approved before auto/draft (F2 eviction keeps the most trustworthy records). func priorityRank(p PickedEntry) int { rank := 0 if p.Disp != Confirmed { rank |= 1 << 2 } if p.Sticky { rank |= 1 << 1 } if !p.valid() { return rank | 1<<0 // a row-less record ranks with the unapproved; it is skipped before rendering } if p.entry.status != "approved" { rank |= 1 << 0 } return rank } // RenderGlossaryBlock serializes the selected records into the injection message // (§C). Records with no dst yet (ruby candidates) are skipped — a "src → " line // carries nothing. Returns "" when nothing renders, so an empty selection injects NO // message at all ("better nothing than garbage"). Deterministic: the injected order is the // budget priority order fixed by Select. tx is the TARGET-language wire-text (pair-14 §2): a // target with no injection texts (HasData()==false) injects nothing (a non-ru book gets no // Russian block), so the whole render is gated on it. func RenderGlossaryBlock(injected []PickedEntry, tx lang.InjectionTexts) string { if !tx.HasData() { return "" } var lines []string for _, p := range injected { if !p.valid() || strings.TrimSpace(p.entry.dst) == "" { continue } // The wire fence, as the LAST point a bank value can be stopped before it is bytes of a system // message (wirefence.go). The door and the seed loader refuse such a row earlier and louder, so // this is unreachable for every writer that exists today — and it is here because it is the only // point a writer added later cannot go around. if WireUnfitRow(p.entry.src, p.entry.dst) { continue } line := p.entry.src + " → " + p.entry.dst // ⚠ NO ⟨проверить⟩ MARKER, and no fork by status at all (D39.104 п.2, backlog row 134). The // doctrine is that the bank on the wire is LAW for every role and every row «независимо от // статуса строки» — the owner's «надо давать везде одинаково; если кто-то ошибся — это уровнем // выше» — and a marker that says «this one is a candidate» is the divergence channel that // doctrine closed: parallel chunks were free to render the same term differently, and nothing // repairs divergence afterwards. Measured beside it: the marker was behaviourally empty anyway // (research/24 §C). What it cost was tokens on every line of every request. // // The gender directive now rides the same rule, with ONE exception that is not about status // (D39.21 «род должен доезжать», the split ratified by the orchestrator on D39.104 п.1): it is // withheld from a record whose KEY may have fired inside an unrelated word. An unsigned // rendering shown as law is the doctrine's accepted cost; a gender instruction attached to the // wrong entity is not a cost but an error, and KeyTrusted is the half of the trust that asks // exactly that. "" for a term with no gender datum — the common case, byte-identical. if p.KeyTrusted { line += genderConstraintNote(p.entry.gender, tx) } lines = append(lines, line) } if len(lines) == 0 { return "" } return tx.GlossaryHeader + "\n" + strings.Join(lines, "\n") } // RenderFormatVersion versions the FORMAT of the role-injection renderers whose output is NOT // captured by matchVersion (a scope/matcher-ALGORITHM version, not a render-layout version): // the editor constraint block's src→dst layout (WS2 §2в) and the DC3 gender-constraint annotation // (WS5 §5(б)). A format change shifts the injected bytes → the wire, so it must be a loud --resnapshot; // folding it as a dedicated snapshot component (snapshot.go) keeps that loudness a MECHANISM, not // discipline — matchVersion (const memory.go) would not move on a render edit. // v2 (WS5 R4): the editor constraint line now carries a gender annotation for a gendered CONFIRMED // term (DC3 injection — the Bai Ninbing fix: the injection DIRECTS the editor, no coreference needed). // v3 (pack-13, D39.21 injection-completeness fix): the DRAFT glossary block (RenderGlossaryBlock) now // ALSO appends the confirmed-gender annotation, so a named term's gender reaches the TRANSLATOR wire, not // only the editor (the owner's «род должен доезжать» directive). A gendered confirmed term shifts the // draft injected bytes → the wire, so a loud --resnapshot; a genderless bank is byte-identical to v2. // v4 (wire batch, D39.104 п.2 / backlog row 134): the ⟨проверить⟩ marker leaves the wire in BOTH blocks // and the editor's two sections collapse into one law block; the gender directive stops asking who signed // the row — in BOTH blocks — and asks only whether the KEY is trustworthy. Every book with an unsigned row // moves, which is the whole point of the doctrine — and is why it lands in the common re-snapshot window // rather than on its own. // (The bank-quality §3 neuter directive is NOT versioned here: a blanket bump is un-re-pinnable and would // re-snapshot every book on earth. It is folded SCOPED — like the editor-unverified section — so only a // book that actually carries a neuter row moves; see the neuter-directive tag in ComputeVersionScopedIn.) const RenderFormatVersion = "renderfmt-v4-one-law-block+no-unverified-marker+draft-gender-on-match" // The editor's canonical-constraint block header and the glossary header are TARGET-language wire-text // (lang.InjectionTexts, pair-14 §2) — they gave the BILINGUAL editor (D30.1) the approved src→dst bindings // as consistency constraints. Relocated out of the pipeline (a Russian header rendered for a →en book was a // leak); now gated on the target and sourced from embedded per-target data. // RenderEditorConstraintBlock serializes the selected records' CONFIRMED renderings into the // editor's injection as a src→dst MAPPING (WS2 §2в resolves the D30.1-open question): "源термин → // «dst»", like the translator block, NOT bare canonical Russian. Rationale (the single load-bearing // one, review-1 F4): the editor is BILINGUAL (editor.md feeds it the source), and binding the canon // to its SOURCE term disambiguates HOMONYMIC dst (one Russian surface for two entities) — bare // forms cannot. It stays CONFIRMED-only on purpose: an AMBIGUOUS record is a candidate the // translator MAY have legitimately rejected, so forcing the editor to rewrite toward it would // corrupt a correct translation (mirrors the gate's CONFIRMED-only discipline, external-review #1); // AMBIGUOUS/mined-draft rows never enter here. Deduped by (src,dst) — an alias and its main entry // share the pair — which only WIDENS the list (harmless), in the budget priority order fixed by // Select. Returns "" when nothing CONFIRMED renders → the editor gets NO injection (plain draft-only // layout). The block is a subset of the already budget-limited memSel.Injected, so it needs no // separate token budget. The src→dst FORMAT is snapshot-folded via RenderFormatVersion (a format // edit is a loud --resnapshot); the injection is a message, so it also enters request_hash // directly (no silent false-hit). // ⚠ ONE BLOCK, NOT TWO (D39.104 п.2, backlog row 134). Pack-20 gave the unsigned rows a SECOND, separately // headed section, on the reasoning that mixing them into the canon would give an unsigned guess the canon's // binding force. D39.104 п.1 answered that reasoning rather than refined it: the bank on the wire is law for // every role «независимо от статуса строки», because the freedom the second header granted is the channel // through which parallel chunks of one book render one term two ways — and divergence, unlike a wrong row, // is repaired by nothing. A wrong row is fixed one level up, with one edit to the bank and a re-edit; that // is the cost the doctrine names and accepts. The probe that sent this shape measured the single law block // executed 21/21 (experiments/18 §A), and the header text below is the arm it sent, unchanged. // // The unsigned rows therefore join the canon list — same header, same shape, no ⟨проверить⟩. What does NOT // join them is the GENDER directive: see editorLines. A book whose every row is signed renders exactly what // it rendered before. func RenderEditorConstraintBlock(injected []PickedEntry, tx lang.InjectionTexts) string { if !tx.HasData() { return "" } law := editorLines(injected, tx) if len(law) == 0 { return "" } return tx.EditorHeader + "\n" + strings.Join(law, "\n") } // editorLines renders the editor's ONE law block: every injected record with a rendering, ONE LINE PER // SOURCE TERM, in the budget priority order Select fixed. Signed and unsigned rows sit in the same list and // look the same, which is the doctrine (see RenderEditorConstraintBlock). // // ⚠ AN UNSIGNED ROW YIELDS TO A SIGNED ONE OF THE SAME SOURCE TERM, and nothing else is dropped. While the // block had two headers, two renderings of one term were two DIFFERENT statements — «this is the canon» and // «this is an unverified working version» — and showing both was informative. Merged into one law block they // become two ORDERS for the same term, and «bring any divergence to this form» cannot be obeyed twice; if // the two carry different gender directives it is two contradictory orders about one character. The state is // reachable and not exotic: an owner-signed row and the engine's own consolidation of the same surface, the // very conflict the run already reports. // // ⚠ AND THE RULE IS ABOUT SIGNATURES, NOT ABOUT THE SURFACE — the first version of it deduped on `src` // alone and that was a REGRESSION, found by planting. The bank's uniqueness key is (src, sense, since, until), // so ONE source term legitimately carries several SIGNED canons: 青山 the surname and 青山 the direction are // two entities the owner decided about separately. Dropping one of them because they share a surface // discards a signature — the loss this project refuses everywhere else — and it did so silently, while the // translator's block still rendered both, so the two wires disagreed. Two signed rows are therefore both // law; what yields is only the row nobody signed, and only to a row somebody did. // // Among the rows that survive, the FIRST wins, which is not «first» in any accidental sense: Select hands // records in priority order and priorityRank puts Confirmed before Ambiguous and approved before auto. ⚠ The // conflict is not HIDDEN by any of this — it is routed. The engine already warns about it where an operator // can act (seeding.go): the MODEL gets one law per unsigned collision, the OPERATOR gets the disagreement. // Dropping the loser at load instead would touch what the bank CONTAINS, and this is a wire fix. // // A homonymic dst (two different source terms rendering to one Russian surface) is untouched: those are two // source terms and stay two lines — the whole reason the block binds each canon to its source (WS2 §2в). // // ⚠ THE GENDER DIRECTIVE ASKS THE MATCH AND NOTHING ELSE — the same question the draft block asks, and the // same answer, which is the point. It used to ask for a SIGNATURE on top, on the reasoning that the editor's // line is an ORDER («bring any divergence to this form») and an order carrying an unsigned datum smuggles // one in. That reasoning did not survive its own pack: with the two sections merged, the unsigned // RENDERING is already an order, so withholding only the gender was an exception with nothing left under // it. D39.104 п.1 says what the wire does — the bank is law for every role «независимо от статуса строки» // — and the invariant it preserves beside that sentence is about STATUSES and the signing table, neither // of which anything here touches. // // What survived scrutiny is the other half, and it was never about authority: a short purely-phonetic key // (リン in リンゴ) may have fired on a different entity, and gender is the part of the line that would then // be actively WRONG rather than merely unsigned. KeyTrusted is exactly that question. On zh the branch // cannot fire at all (Han keys anchor at 2); on a phonetic source it hits exactly the short names. // // ⚠ Provenance still decides nothing here, which is the S7 fix (cold-run session, backlog 19) surviving // its own subject: S7 stopped the SECTIONS being routed by a statement about the match, and the sections // are gone — the lesson that the two questions are different, and that this line asks the match, is what // remains of it. func editorLines(injected []PickedEntry, tx lang.InjectionTexts) []string { // Which source terms carry a SIGNED rendering at all. Collected first, because the rule is about what // else is in the selection: an unsigned row is law until a signed row of the same surface is present. signedSrc := map[string]bool{} for _, p := range injected { if p.valid() && p.entry.status == "approved" && strings.TrimSpace(p.entry.dst) != "" { signedSrc[strings.TrimSpace(p.entry.src)] = true } } var lines []string seen := map[string]bool{} for _, p := range injected { if !p.valid() { continue } src := strings.TrimSpace(p.entry.src) dst := strings.TrimSpace(p.entry.dst) if dst == "" { continue } // The same wire fence as the draft block, asked of the values THIS block renders (trimmed here, // raw there) — one predicate, two call sites, so the two wires cannot disagree about what a // record is allowed to say (wirefence.go). if WireUnfitRow(src, dst) { continue } if p.entry.status != "approved" && signedSrc[src] { continue // an unsigned rendering yields to the signed canon of the same surface } // Beyond that, one line per (src, dst): the same rendering reaching the block twice — an alias and // its main entry — is one order, and two signed SENSES of one surface are two. key := src + "\x00" + dst if seen[key] { continue } seen[key] = true line := "- " + src + " → «" + dst + "»" if p.KeyTrusted { line += genderConstraintNote(p.entry.gender, tx) } lines = append(lines, line) } return lines } // genderConstraintNote is the DC3 gender directive appended to a CONFIRMED editor-constraint line (WS5 // §5(б)): the injection DIRECTS the editor to the character's grammatical gender, which is the fix for // the Bai Ninbing class — no coreference needed. male/female ⇒ hard gender forms; hidden ⇒ a mandate to // AVOID gender-marking constructions until the reveal (a masculine default when unavoidable, D19.3). "" // for a term with no gender datum (the common case → the line is unchanged, byte-identical to before). func genderConstraintNote(gender string, tx lang.InjectionTexts) string { switch gender { case "male", "m": return tx.GenderMale case "female", "f": return tx.GenderFemale case "neuter", "n": return tx.GenderNeuter case "hidden": return tx.GenderHidden } return "" } // --- Aho-Corasick multi-pattern automaton over runes ---------------------------- type acMatch struct { keyIdx int start, end int // rune indices [start, end) } type acNode struct { next map[rune]int fail int out []int // key indices whose pattern ends at this node (incl. via fail links) } type ahoCorasick struct { nodes []acNode keys []string klen []int // rune length of each key } // buildAC constructs the automaton from the (already sorted, unique) normalized keys. // Deterministic in RESULT regardless of map iteration order: fail links and outputs are // a function of the key set, not the BFS visit order. func buildAC(keys []string) *ahoCorasick { ac := &ahoCorasick{keys: keys, klen: make([]int, len(keys))} ac.nodes = []acNode{{next: map[rune]int{}}} // root = node 0 for i, k := range keys { kr := []rune(k) ac.klen[i] = len(kr) cur := 0 for _, r := range kr { nxt, ok := ac.nodes[cur].next[r] if !ok { nxt = len(ac.nodes) ac.nodes = append(ac.nodes, acNode{next: map[rune]int{}}) ac.nodes[cur].next[r] = nxt } cur = nxt } ac.nodes[cur].out = append(ac.nodes[cur].out, i) } // BFS to compute fail links; propagate outputs down fail chains one level (each // node inherits its fail node's already-complete output set). var queue []int for _, c := range ac.nodes[0].next { ac.nodes[c].fail = 0 queue = append(queue, c) } for len(queue) > 0 { cur := queue[0] queue = queue[1:] for r, nxt := range ac.nodes[cur].next { queue = append(queue, nxt) f := ac.nodes[cur].fail for f != 0 { if _, ok := ac.nodes[f].next[r]; ok { break } f = ac.nodes[f].fail } if fn, ok := ac.nodes[f].next[r]; ok && fn != nxt { ac.nodes[nxt].fail = fn } else { ac.nodes[nxt].fail = 0 } ac.nodes[nxt].out = append(ac.nodes[nxt].out, ac.nodes[ac.nodes[nxt].fail].out...) } } return ac } // matches returns every key occurrence in text (rune indices), sorted deterministically. func (ac *ahoCorasick) matches(text []rune) []acMatch { var out []acMatch cur := 0 for i, r := range text { for cur != 0 { if _, ok := ac.nodes[cur].next[r]; ok { break } cur = ac.nodes[cur].fail } if nxt, ok := ac.nodes[cur].next[r]; ok { cur = nxt } else { cur = 0 } for _, ki := range ac.nodes[cur].out { out = append(out, acMatch{keyIdx: ki, start: i - ac.klen[ki] + 1, end: i + 1}) } } sort.Slice(out, func(a, b int) bool { if out[a].start != out[b].start { return out[a].start < out[b].start } if out[a].end != out[b].end { return out[a].end < out[b].end } return out[a].keyIdx < out[b].keyIdx }) return out } // trustRank orders injection dispositions by how much the reader may TRUST the record: CONFIRMED // (approved, authoritative) outranks AMBIGUOUS (draft/auto/collision-prone), which outranks REJECT // (spoiler-blocked / unset). It decides the longest-match containment contest below. func trustRank(d InjectionDisposition) int { switch d { case Confirmed: return 2 case Ambiguous: return 1 default: // Reject / unset return 0 } } // matchTrust returns the BEST (highest-trust) disposition an occurrence of the given key could // inject with at this chapter — the max over its spoiler-VALID owner entries — and whether ANY // valid owner exists (this second value subsumes the old validSuppressor: no valid owner ⇒ the // occurrence injects nothing). CONFIRMED beats AMBIGUOUS. It is the trust the disposition-gate // decides a containment contest on (D39 layer 4). func (b *Bank) matchTrust(m acMatch, chapter int) (disp InjectionDisposition, hasValid bool) { key := b.ac.keys[m.keyIdx] for _, ei := range b.keyOwners[key] { e := &b.entries[ei] if spoilerBlocked(e, chapter) { continue } if d := dispositionFor(e, key); !hasValid || trustRank(d) > trustRank(disp) { disp, hasValid = d, true } } return disp, hasValid } // suppressContained applies longest-match / whole-entity replacement (A3), DISPOSITION-GATED (D39 // layer 4). It drops an occurrence fully contained inside a STRICTLY longer one, but ONLY when the // longer one is BOTH (a) owned by a spoiler-VALID entry at this chapter (self-review #6: a spoiler- // blocked longer key like 林动的父亲, until_ch=3, read at ch10, must not eat a valid nested 林动) AND // (b) at least as TRUSTWORTHY as the nested match it would delete. A longer but LOWER-trust key // (a draft/ambiguous 四代族长) must NOT suppress a nested HIGHER-trust key (an approved 族长): the draft // is editor-excluded (RenderEditorConstraintBlock is CONFIRMED-only), so eating the approved term // leaves the reader with nothing AND the drop is invisible to post-check/retrieval-state — the exact // term-drift code root the D38.5 seed-promote only patched by data (L3-recurrence). The whole-entity // longest-match is preserved when the longer key is ≥ trust (approved 四代族长 still beats approved // 族长; draft 四代族长 still eats a nested draft 族长). Each REFUSED suppression is recorded as a // TrustGateEvent so the near-silent drop is surfaced loudly (research/13 §7). Equal-length overlaps // are both kept (genuine surface ambiguity). Deterministic (ms is sorted; events deduped by pair in // ms order). O(n²) over the few matches in a chunk. func (b *Bank) suppressContained(ms []acMatch, chapter int) ([]acMatch, []TrustGateEvent) { var kept []acMatch var gated []TrustGateEvent for i, m := range ms { conDisp, conValid := b.matchTrust(m, chapter) contained := false var refused *TrustGateEvent // a lower-trust valid longer key that WANTED to suppress m for j, o := range ms { if i == j { continue } // o must STRICTLY contain m's span to be a longest-match suppressor. if !(o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start)) { continue } supDisp, supValid := b.matchTrust(o, chapter) if !supValid { continue // spoiler-blocked longer key — never a valid suppressor (self-review #6) } if !conValid || trustRank(supDisp) >= trustRank(conDisp) { contained = true // a valid, ≥-trust container wins the whole-entity longest-match break } // supValid && conValid && supDisp < conDisp: a lower-trust longer key wants to eat a // higher-trust nested one — REFUSE (the L3 fix). Remember the FIRST such refusal for the // loud record; keep scanning in case a higher-trust container legitimately suppresses m. if refused == nil { refused = &TrustGateEvent{ Suppressor: b.ac.keys[o.keyIdx], SuppressorDisp: string(supDisp), Protected: b.ac.keys[m.keyIdx], ProtectedDisp: string(conDisp), } } } if contained { continue // m suppressed by the whole-entity longest-match; any refusal above was moot } kept = append(kept, m) if refused != nil { gated = append(gated, *refused) // m survived AND a lower-trust key tried to eat it — loud } } return kept, dedupeTrustGate(gated) } // dedupeTrustGate collapses identical (suppressor→protected) events to a distinct set, preserving // first-occurrence (ms) order so the count and detail are deterministic and count distinct term // collisions rather than raw occurrences. func dedupeTrustGate(evs []TrustGateEvent) []TrustGateEvent { if len(evs) < 2 { return evs } seen := map[[2]string]bool{} var out []TrustGateEvent for _, e := range evs { k := [2]string{e.Suppressor, e.Protected} if seen[k] { continue } seen[k] = true out = append(out, e) } return out } // suppressUnboundedPhonetic drops an occurrence of a SPACED-SCRIPT phonetic key (Latin or // Cyrillic — scripts that delimit words with whitespace/non-letters, exactly like the // target-side containsWholeWord) when a neighbouring rune is a letter of the SAME script: // the key fired INSIDE a longer word ("rose" in "roseanne"/"roses"), not on the entity // (D16.3). This closes the source-vs-target boundary asymmetry the ≤3 collision-downgrade // does not (a ≥4 phonetic key fires CONFIRMED with no boundary today). A cross-script // neighbour (a Latin name abutting a Han char in unspaced source) is a valid boundary, so // only a SAME-script letter suppresses — mirroring "non-letter or foreign-script". // KANA and HAN are deliberately NOT boundary-checked: they have no word segmentation, so a // letter-boundary rule would false-negative the ubiquitous kana name+particle case (すずきは) // or contradict the long-key-CONFIRMED contract — their ≥4 precision is deferred to the // kana-precision measurement + the B6 tokenizer (D16; flagged to the orchestrator). ntext is // the already-normalized chunk. O(n) over the few matches in a chunk. func (b *Bank) suppressUnboundedPhonetic(ms []acMatch, ntext []rune) []acMatch { var kept []acMatch for _, m := range ms { script := spacedPhoneticScript(b.ac.keys[m.keyIdx]) if script != nil { beforeBad := m.start > 0 && letterInScript(ntext[m.start-1], script) afterBad := m.end < len(ntext) && letterInScript(ntext[m.end], script) if beforeBad || afterBad { continue // the key is inside a longer same-script word — not a whole-entity match } } kept = append(kept, m) } return kept } // spacedPhoneticScript returns the word-delimited alphabetic script a normalized key belongs // to for the source word-boundary check — unicode.Latin or unicode.Cyrillic — or nil when the // key carries a Han ideograph or kana (no word segmentation) or mixes the two spaced scripts. // Digits/punctuation don't set the script but don't disqualify (so "o'brien" is still Latin). func spacedPhoneticScript(normKey string) *unicode.RangeTable { var script *unicode.RangeTable for _, r := range normKey { switch { case text.DenseScript(r): return nil // an ideographic/kana anchor is never boundary-checked here case unicode.In(r, unicode.Latin): if script == unicode.Cyrillic { return nil // mixed spaced scripts — do not boundary-check } script = unicode.Latin case unicode.In(r, unicode.Cyrillic): if script == unicode.Latin { return nil } script = unicode.Cyrillic } } return script } // letterInScript reports whether r is a LETTER of the given spaced script — the "same-script // letter" that breaks a word boundary. A digit, punctuation, space, or a letter of another // script is a valid boundary and returns false. func letterInScript(r rune, script *unicode.RangeTable) bool { return unicode.IsLetter(r) && unicode.In(r, script) } // InjectivityCollisions reports approved dst collisions (B2): two distinct source // terms mapped to the SAME dst (one Russian surface for two entities → the reader // cannot tell them apart), returned as human-readable strings for a load-time warning. // A pure diagnostic; it does not reject (some collisions are legitimate, e.g. a title // shared by rank tiers), so the caller logs it, not aborts. func InjectivityCollisions(rows []store.GlossaryEntry) []string { bySurface := map[string][]string{} for _, r := range rows { if r.Status != "approved" || strings.TrimSpace(r.Dst) == "" { continue } key := text.NormalizeTargetForm(r.Dst) bySurface[key] = append(bySurface[key], r.Src) } var out []string // Deterministic order: iterate a sorted key list, not the map. keys := slices.Sorted(maps.Keys(bySurface)) for _, k := range keys { srcs := bySurface[k] if len(distinct(srcs)) > 1 { sort.Strings(srcs) out = append(out, "dst "+strconv.Quote(k)+" ← "+strings.Join(distinct(srcs), ", ")) } } return out } func distinct(ss []string) []string { seen := map[string]bool{} var out []string for _, s := range ss { if !seen[s] { seen[s] = true out = append(out, s) } } sort.Strings(out) return out }