package store import ( "context" "database/sql" "errors" ) // glossary.go: durable storage for the memory bank v2 glossary (schema v5). Dumb // storage, mirroring ruby.go/chunkstatus.go: the pipeline owns aggregation, // classification and matching; this layer only replaces and reads back. A book's // glossary is a deterministic function of its inputs (seed file + classified ruby), // so ReplaceGlossary rewrites the WHOLE book set in one transaction — re-seeding the // identical inputs rewrites identical rows, a seed edit converges every column, and a // removed term disappears instead of lingering (the same full-replace idempotency the // ruby step adopted after external-review #6). // GlossaryEntry is one term row plus its aliases. Strings are stored raw; the // pipeline normalizes for matching (§3.6). Decl is a JSON blob the post-check reads. type GlossaryEntry struct { ID int64 BookID string Src string Dst string Type string Sense string Gender string Speech string Decl string // JSON {"invariant":bool,"forms":[...]} TranslitPolicy string FirstPerson string NicknameTranslation string SinceCh int UntilCh int Status string // auto|draft|approved AllowShort bool Source string // seed|ruby|auto RubyReading string RubyClass string Confidence int Note string Aliases []GlossaryAlias } // GlossaryAlias is one alias-graph edge (an alternative source surface + its type). type GlossaryAlias struct { Alias string AliasType string } // GlossaryRevision is one append-only editorial-time record (B1). type GlossaryRevision struct { Src string Sense string OldDst string NewDst string EditorialTS string Reason string } // RetrievalState is one per-chunk observability record (registry gate #4). type RetrievalState struct { BookID string Chapter int ChunkIdx int SnapshotID string NExactHits int NSticky int NAmbiguousFlagged int NSpoilerBlocked int NEvicted int EmbeddingTierUsed int NPostcheckMiss int PostcheckDetail string // JSON InjectedIDs string // JSON NStyleFlags int // cheap style/number gate hits (dialogue-dash, ё, translit-interj, 万/億) StyleDetail string // JSON breakdown of the style-flag hits // NTrustGatedSuppress counts refused lower-trust longest-match suppressions (D39 layer 4): a // longer draft/ambiguous key blocked from eating a nested approved/confirmed one. A nonzero // count is a loud seed-hygiene signal (a draft term nesting over an approved term to reconcile). NTrustGatedSuppress int TrustGateDetail string // JSON of the refused suppressor→protected pairs // NBanknoteLines / BanknoteParseFail / BanknoteTruncated are the banknote channel's per-chunk // telemetry (WS4 point 10): the accepted ⟦TM-BANK-v1⟧ line count and the parse-fail / truncation // flags (0/1) of the translator draft. Zero for every channel-off / non-banknote chunk. NBanknoteLines int BanknoteParseFail int BanknoteTruncated int // BanknoteDetail is the JSON of the chunk's PARSED banknote entries (src / proposed dst / type) — // the WHAT channel itself, which D39.36 found the code parsed and dropped into `_`. It is the fourth // *_detail column of this row and behaves like its siblings: re-derived deterministically from the // raw checkpoint on every run (self-healing), observability only, never wire-load-bearing. The // bank-mining stop reads it to attach the translator's PROPOSED rendering to the owner's signature // map; the proposal stays status:auto, so delivering the WHAT grants no trust. BanknoteDetail string // JSON // NUnverifiedShown / NUnverifiedFollowed are the $0 observability channel of the unverified wire // (pack-20): how many UNSIGNED bank rows actually fired in this chunk, and how many of those the // model went along with. Measured on the wire that SHOWED the row, and never a gate — an unverified // row is a candidate the model is entitled to reject. NUnverifiedShown int NUnverifiedFollowed int // NVoiceFlags / VoiceDetail are the deterministic voice flagger (pack-19, gates.voice): T/V // contradictions, flattened self-designations and forbidden lexemes in attributed replies. Zero for // every gate-off unit. NSpoilerLeaks / SpoilerLeakDetail are the reveal half of D21 п.3: a rendering // the spoiler window REJECTED for this chapter that reached the output anyway. Both are observability // (never a disposition) and re-derived each run, like every sibling on this row. NVoiceFlags int VoiceDetail string // JSON NSpoilerLeaks int SpoilerLeakDetail string // JSON } // ReplaceGlossary replaces a book's glossary with no voice/address rows — the pre-pack-19 form, kept // because most callers (and every glossary-only test) have nothing else to write. func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error { return s.ReplaceBank(bookID, entries, nil, nil) } // ReplaceBank replaces a book's ENTIRE bank — glossary rows + aliases + voice profiles + address pairs — // with the given content in ONE transaction, and appends a B1 revision record for every term whose // approved rendering (dst) changed since the previous set. Scoped to book_id, so other books are // untouched. Each entry's BookID must equal bookID. term_id linkage is established within the tx: a // glossary row is inserted, its rowid taken, then its aliases inserted against that id (ids are fresh // each replace — never hashed). // // The three record types share ONE transaction on purpose: they fold into one memory_version, so a // half-applied replace would make the materialized bank disagree with the hash the snapshot pins it by. // Voice/address rows carry no id linkage at all — they reference a character by the stable (src, sense). func (s *Store) ReplaceBank(bookID string, entries []GlossaryEntry, voices []VoiceProfile, pairs []AddressPair) error { ctx, cancel := opContext() defer cancel() tx, err := s.w.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() // Snapshot the prior (src,sense,window)→dst to detect editorial-time changes (B1). prior, err := priorRenderings(ctx, tx, bookID) if err != nil { return err } for _, table := range []string{"glossary_aliases", "glossary", "voice_profiles", "address_pairs"} { if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE book_id = ?`, bookID); err != nil { return err } } for _, e := range entries { res, err := tx.ExecContext(ctx, ` INSERT INTO glossary ( book_id, src, dst, type, sense, gender, speech, decl, translit_policy, first_person, nickname_translation, since_ch, until_ch, status, allow_short, source, ruby_reading, ruby_class, confidence, note ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, e.BookID, e.Src, e.Dst, e.Type, e.Sense, e.Gender, e.Speech, e.Decl, e.TranslitPolicy, e.FirstPerson, e.NicknameTranslation, e.SinceCh, e.UntilCh, e.Status, boolToInt(e.AllowShort), e.Source, e.RubyReading, e.RubyClass, e.Confidence, e.Note) if err != nil { return err } termID, err := res.LastInsertId() if err != nil { return err } for _, a := range e.Aliases { if _, err := tx.ExecContext(ctx, ` INSERT INTO glossary_aliases (book_id, term_id, alias, alias_type) VALUES (?, ?, ?, ?)`, e.BookID, termID, a.Alias, a.AliasType); err != nil { return err } } // B1: a term that existed with a non-empty dst and now renders differently // (also non-empty) is an editorial change → append the revision journal row. if old, ok := prior[[4]any{e.Src, e.Sense, e.SinceCh, e.UntilCh}]; ok && old != "" && e.Dst != "" && old != e.Dst { if _, err := tx.ExecContext(ctx, ` INSERT INTO glossary_revisions (book_id, src, sense, old_dst, new_dst, reason) VALUES (?, ?, ?, ?, ?, ?)`, e.BookID, e.Src, e.Sense, old, e.Dst, "seed-replace"); err != nil { return err } } } for _, v := range voices { if _, err := tx.ExecContext(ctx, ` INSERT INTO voice_profiles ( book_id, src, sense, register, self_ref, address_default, lexicon_markers, ng_lexicon, exemplars, brightness, since_ch, until_ch ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`, bookID, v.Src, v.Sense, v.Register, v.SelfRef, v.AddressDefault, v.LexiconMarkers, v.NGLexicon, v.Exemplars, v.Brightness, v.SinceCh, v.UntilCh); err != nil { return err } } for _, p := range pairs { if _, err := tx.ExecContext(ctx, ` INSERT INTO address_pairs ( book_id, speaker_src, speaker_sense, addressee_src, addressee_sense, register, form, closeness, since_ch, until_ch ) VALUES (?,?,?,?,?,?,?,?,?,?)`, bookID, p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense, p.Register, p.Form, p.Closeness, p.SinceCh, p.UntilCh); err != nil { return err } } return tx.Commit() } // GlossaryForBook returns every glossary entry for a book with its aliases, in a // DETERMINISTIC order (src, sense, since_ch, until_ch, status, dst) with aliases sorted // — the stable materialization the hot-path matcher and the F1 memoryVersion() consume. func (s *Store) GlossaryForBook(bookID string) ([]GlossaryEntry, error) { ctx, cancel := opContext() defer cancel() rows, err := s.r.QueryContext(ctx, ` SELECT id, src, dst, type, sense, gender, speech, decl, translit_policy, first_person, nickname_translation, since_ch, until_ch, status, allow_short, source, ruby_reading, ruby_class, confidence, note FROM glossary WHERE book_id = ? ORDER BY src, sense, since_ch, until_ch, status, dst`, bookID) if err != nil { return nil, err } defer rows.Close() var out []GlossaryEntry byID := map[int64]int{} // glossary.id → index in out for rows.Next() { e := GlossaryEntry{BookID: bookID} var allowShort int if err := rows.Scan(&e.ID, &e.Src, &e.Dst, &e.Type, &e.Sense, &e.Gender, &e.Speech, &e.Decl, &e.TranslitPolicy, &e.FirstPerson, &e.NicknameTranslation, &e.SinceCh, &e.UntilCh, &e.Status, &allowShort, &e.Source, &e.RubyReading, &e.RubyClass, &e.Confidence, &e.Note); err != nil { return nil, err } e.AllowShort = allowShort != 0 byID[e.ID] = len(out) out = append(out, e) } if err := rows.Err(); err != nil { return nil, err } // Aliases in a second pass, ordered so each entry's alias list is deterministic. arows, err := s.r.QueryContext(ctx, ` SELECT term_id, alias, alias_type FROM glossary_aliases WHERE book_id = ? ORDER BY term_id, alias`, bookID) if err != nil { return nil, err } defer arows.Close() for arows.Next() { var termID int64 var a GlossaryAlias if err := arows.Scan(&termID, &a.Alias, &a.AliasType); err != nil { return nil, err } if idx, ok := byID[termID]; ok { out[idx].Aliases = append(out[idx].Aliases, a) } } return out, arows.Err() } // GlossaryRevisionsForBook returns the editorial-time journal (B1) for a book, // newest first, for the report / a human audit. func (s *Store) GlossaryRevisionsForBook(bookID string) ([]GlossaryRevision, error) { return queryAll(s.r, ` SELECT src, sense, old_dst, new_dst, editorial_ts, reason FROM glossary_revisions WHERE book_id = ? ORDER BY id DESC`, func(rows *sql.Rows) (GlossaryRevision, error) { var r GlossaryRevision err := rows.Scan(&r.Src, &r.Sense, &r.OldDst, &r.NewDst, &r.EditorialTS, &r.Reason) return r, err }, bookID) } // UpsertRetrievalState writes (or overwrites) the per-chunk retrieval-state record. // Overwrite is the resolve semantics: recomputing the deterministic matcher over the // same chunk reproduces the same row (self-heals on resume). func (s *Store) UpsertRetrievalState(rs RetrievalState) error { ctx, cancel := opContext() defer cancel() _, err := s.w.ExecContext(ctx, ` INSERT INTO retrieval_state ( book_id, chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids, n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail, n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail, n_unverified_shown, n_unverified_followed, n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail, updated_at ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now')) ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET snapshot_id = excluded.snapshot_id, n_exact_hits = excluded.n_exact_hits, n_sticky = excluded.n_sticky, n_ambiguous_flagged = excluded.n_ambiguous_flagged, n_spoiler_blocked = excluded.n_spoiler_blocked, n_evicted = excluded.n_evicted, embedding_tier_used = excluded.embedding_tier_used, n_postcheck_miss = excluded.n_postcheck_miss, postcheck_detail = excluded.postcheck_detail, injected_ids = excluded.injected_ids, n_style_flags = excluded.n_style_flags, style_detail = excluded.style_detail, n_trust_gated_suppress = excluded.n_trust_gated_suppress, trust_gate_detail = excluded.trust_gate_detail, n_banknote_lines = excluded.n_banknote_lines, banknote_detail = excluded.banknote_detail, banknote_parse_fail = excluded.banknote_parse_fail, banknote_truncated = excluded.banknote_truncated, n_unverified_shown = excluded.n_unverified_shown, n_unverified_followed = excluded.n_unverified_followed, n_voice_flags = excluded.n_voice_flags, voice_detail = excluded.voice_detail, n_spoiler_leaks = excluded.n_spoiler_leaks, spoiler_leak_detail = excluded.spoiler_leak_detail, updated_at = excluded.updated_at`, rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID, rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted, rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs, rs.NStyleFlags, rs.StyleDetail, rs.NTrustGatedSuppress, rs.TrustGateDetail, rs.NBanknoteLines, rs.BanknoteParseFail, rs.BanknoteTruncated, rs.BanknoteDetail, rs.NUnverifiedShown, rs.NUnverifiedFollowed, rs.NVoiceFlags, rs.VoiceDetail, rs.NSpoilerLeaks, rs.SpoilerLeakDetail) return err } // GetRetrievalState returns the record for a chunk, or nil if none. func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*RetrievalState, error) { ctx, cancel := opContext() defer cancel() rs := RetrievalState{BookID: bookID, Chapter: chapter, ChunkIdx: chunkIdx} err := s.r.QueryRowContext(ctx, ` SELECT snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids, n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail, n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail, n_unverified_shown, n_unverified_followed, n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`, bookID, chapter, chunkIdx).Scan( &rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs, &rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail, &rs.NBanknoteLines, &rs.BanknoteParseFail, &rs.BanknoteTruncated, &rs.BanknoteDetail, &rs.NUnverifiedShown, &rs.NUnverifiedFollowed, &rs.NVoiceFlags, &rs.VoiceDetail, &rs.NSpoilerLeaks, &rs.SpoilerLeakDetail) if errors.Is(err, sql.ErrNoRows) { return nil, nil } if err != nil { return nil, err } return &rs, nil } // RetrievalStatesForBook returns every retrieval-state record for a book, ordered for // a stable report (the memory section of tmctl report). func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error) { return queryAll(s.r, ` SELECT chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids, n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail, n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail, n_unverified_shown, n_unverified_followed, n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail FROM retrieval_state WHERE book_id = ? ORDER BY chapter, chunk_idx`, func(rows *sql.Rows) (RetrievalState, error) { rs := RetrievalState{BookID: bookID} err := rows.Scan(&rs.Chapter, &rs.ChunkIdx, &rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs, &rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail, &rs.NBanknoteLines, &rs.BanknoteParseFail, &rs.BanknoteTruncated, &rs.BanknoteDetail, &rs.NUnverifiedShown, &rs.NUnverifiedFollowed, &rs.NVoiceFlags, &rs.VoiceDetail, &rs.NSpoilerLeaks, &rs.SpoilerLeakDetail) return rs, err }, bookID) } func boolToInt(b bool) int { if b { return 1 } return 0 } // priorRenderings reads the (src,sense,window)→dst map a replace overwrites. Separate function so the // rows close before the caller's DELETEs run on the same tx — a tx holds one connection. func priorRenderings(ctx context.Context, tx *sql.Tx, bookID string) (map[[4]any]string, error) { rows, err := tx.QueryContext(ctx, `SELECT src, sense, since_ch, until_ch, dst FROM glossary WHERE book_id = ?`, bookID) if err != nil { return nil, err } defer rows.Close() prior := map[[4]any]string{} for rows.Next() { var src, sense, dst string var since, until int if err := rows.Scan(&src, &sense, &since, &until, &dst); err != nil { return nil, err } prior[[4]any{src, sense, since, until}] = dst } return prior, rows.Err() }