413 lines
18 KiB
Go
413 lines
18 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
)
|
|
|
|
// glossary.go: durable storage for the memory bank v2 glossary (schema v5). Dumb
|
|
// storage, mirroring ruby.go/chunkstatus.go: the pipeline owns aggregation,
|
|
// classification and matching; this layer only replaces and reads back. A book's
|
|
// glossary is a deterministic function of its inputs (seed file + classified ruby),
|
|
// so ReplaceGlossary rewrites the WHOLE book set in one transaction — re-seeding the
|
|
// identical inputs rewrites identical rows, a seed edit converges every column, and a
|
|
// removed term disappears instead of lingering (the same full-replace idempotency the
|
|
// ruby step adopted after external-review #6).
|
|
|
|
// GlossaryEntry is one term row plus its aliases. Strings are stored raw; the
|
|
// pipeline normalizes for matching (§3.6). Decl is a JSON blob the post-check reads.
|
|
type GlossaryEntry struct {
|
|
ID int64
|
|
BookID string
|
|
Src string
|
|
Dst string
|
|
Type string
|
|
Sense string
|
|
Gender string
|
|
Speech string
|
|
Decl string // JSON {"invariant":bool,"forms":[...]}
|
|
TranslitPolicy string
|
|
FirstPerson string
|
|
NicknameTranslation string
|
|
SinceCh int
|
|
UntilCh int
|
|
Status string // auto|draft|approved
|
|
AllowShort bool
|
|
Source string // seed|ruby|auto
|
|
RubyReading string
|
|
RubyClass string
|
|
Confidence int
|
|
Note string
|
|
Aliases []GlossaryAlias
|
|
}
|
|
|
|
// GlossaryAlias is one alias-graph edge (an alternative source surface + its type).
|
|
type GlossaryAlias struct {
|
|
Alias string
|
|
AliasType string
|
|
}
|
|
|
|
// GlossaryRevision is one append-only editorial-time record (B1).
|
|
type GlossaryRevision struct {
|
|
Src string
|
|
Sense string
|
|
OldDst string
|
|
NewDst string
|
|
EditorialTS string
|
|
Reason string
|
|
}
|
|
|
|
// RetrievalState is one per-chunk observability record (registry gate #4).
|
|
type RetrievalState struct {
|
|
BookID string
|
|
Chapter int
|
|
ChunkIdx int
|
|
SnapshotID string
|
|
NExactHits int
|
|
NSticky int
|
|
NAmbiguousFlagged int
|
|
NSpoilerBlocked int
|
|
NEvicted int
|
|
EmbeddingTierUsed int
|
|
NPostcheckMiss int
|
|
PostcheckDetail string // JSON
|
|
InjectedIDs string // JSON
|
|
NStyleFlags int // cheap style/number gate hits (dialogue-dash, ё, translit-interj, 万/億)
|
|
StyleDetail string // JSON breakdown of the style-flag hits
|
|
// NTrustGatedSuppress counts refused lower-trust longest-match suppressions (D39 layer 4): a
|
|
// longer draft/ambiguous key blocked from eating a nested approved/confirmed one. A nonzero
|
|
// count is a loud seed-hygiene signal (a draft term nesting over an approved term to reconcile).
|
|
NTrustGatedSuppress int
|
|
TrustGateDetail string // JSON of the refused suppressor→protected pairs
|
|
// NBanknoteLines / BanknoteParseFail / BanknoteTruncated are the banknote channel's per-chunk
|
|
// telemetry (WS4 point 10): the accepted ⟦TM-BANK-v1⟧ line count and the parse-fail / truncation
|
|
// flags (0/1) of the translator draft. Zero for every channel-off / non-banknote chunk.
|
|
NBanknoteLines int
|
|
BanknoteParseFail int
|
|
BanknoteTruncated int
|
|
// BanknoteDetail is the JSON of the chunk's PARSED banknote entries (src / proposed dst / type) —
|
|
// the WHAT channel itself, which D39.36 found the code parsed and dropped into `_`. It is the fourth
|
|
// *_detail column of this row and behaves like its siblings: re-derived deterministically from the
|
|
// raw checkpoint on every run (self-healing), observability only, never wire-load-bearing. The
|
|
// bank-mining stop reads it to attach the translator's PROPOSED rendering to the owner's signature
|
|
// map; the proposal stays status:auto, so delivering the WHAT grants no trust.
|
|
BanknoteDetail string // JSON
|
|
// NUnverifiedShown / NUnverifiedFollowed are the $0 observability channel of the unverified wire
|
|
// (pack-20): how many UNSIGNED bank rows actually fired in this chunk, and how many of those the
|
|
// model went along with. Measured on the wire that SHOWED the row, and never a gate — an unverified
|
|
// row is a candidate the model is entitled to reject.
|
|
NUnverifiedShown int
|
|
NUnverifiedFollowed int
|
|
// NVoiceFlags / VoiceDetail are the deterministic voice flagger (pack-19, gates.voice): T/V
|
|
// contradictions, flattened self-designations and forbidden lexemes in attributed replies. Zero for
|
|
// every gate-off unit. NSpoilerLeaks / SpoilerLeakDetail are the reveal half of D21 п.3: a rendering
|
|
// the spoiler window REJECTED for this chapter that reached the output anyway. Both are observability
|
|
// (never a disposition) and re-derived each run, like every sibling on this row.
|
|
NVoiceFlags int
|
|
VoiceDetail string // JSON
|
|
NSpoilerLeaks int
|
|
SpoilerLeakDetail string // JSON
|
|
}
|
|
|
|
// ReplaceGlossary replaces a book's glossary with no voice/address rows — the pre-pack-19 form, kept
|
|
// because most callers (and every glossary-only test) have nothing else to write.
|
|
func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error {
|
|
return s.ReplaceBank(bookID, entries, nil, nil)
|
|
}
|
|
|
|
// ReplaceBank replaces a book's ENTIRE bank — glossary rows + aliases + voice profiles + address pairs —
|
|
// with the given content in ONE transaction, and appends a B1 revision record for every term whose
|
|
// approved rendering (dst) changed since the previous set. Scoped to book_id, so other books are
|
|
// untouched. Each entry's BookID must equal bookID. term_id linkage is established within the tx: a
|
|
// glossary row is inserted, its rowid taken, then its aliases inserted against that id (ids are fresh
|
|
// each replace — never hashed).
|
|
//
|
|
// The three record types share ONE transaction on purpose: they fold into one memory_version, so a
|
|
// half-applied replace would make the materialized bank disagree with the hash the snapshot pins it by.
|
|
// Voice/address rows carry no id linkage at all — they reference a character by the stable (src, sense).
|
|
func (s *Store) ReplaceBank(bookID string, entries []GlossaryEntry, voices []VoiceProfile, pairs []AddressPair) error {
|
|
ctx, cancel := opContext()
|
|
defer cancel()
|
|
tx, err := s.w.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
// Snapshot the prior (src,sense,window)→dst to detect editorial-time changes (B1).
|
|
prior, err := priorRenderings(ctx, tx, bookID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, table := range []string{"glossary_aliases", "glossary", "voice_profiles", "address_pairs"} {
|
|
if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE book_id = ?`, bookID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
for _, e := range entries {
|
|
res, err := tx.ExecContext(ctx, `
|
|
INSERT INTO glossary (
|
|
book_id, src, dst, type, sense, gender, speech, decl,
|
|
translit_policy, first_person, nickname_translation,
|
|
since_ch, until_ch, status, allow_short, source,
|
|
ruby_reading, ruby_class, confidence, note
|
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
e.BookID, e.Src, e.Dst, e.Type, e.Sense, e.Gender, e.Speech, e.Decl,
|
|
e.TranslitPolicy, e.FirstPerson, e.NicknameTranslation,
|
|
e.SinceCh, e.UntilCh, e.Status, boolToInt(e.AllowShort), e.Source,
|
|
e.RubyReading, e.RubyClass, e.Confidence, e.Note)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
termID, err := res.LastInsertId()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, a := range e.Aliases {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO glossary_aliases (book_id, term_id, alias, alias_type)
|
|
VALUES (?, ?, ?, ?)`, e.BookID, termID, a.Alias, a.AliasType); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// B1: a term that existed with a non-empty dst and now renders differently
|
|
// (also non-empty) is an editorial change → append the revision journal row.
|
|
if old, ok := prior[[4]any{e.Src, e.Sense, e.SinceCh, e.UntilCh}]; ok && old != "" && e.Dst != "" && old != e.Dst {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO glossary_revisions (book_id, src, sense, old_dst, new_dst, reason)
|
|
VALUES (?, ?, ?, ?, ?, ?)`, e.BookID, e.Src, e.Sense, old, e.Dst, "seed-replace"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
for _, v := range voices {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO voice_profiles (
|
|
book_id, src, sense, register, self_ref, address_default,
|
|
lexicon_markers, ng_lexicon, exemplars, brightness, since_ch, until_ch
|
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
bookID, v.Src, v.Sense, v.Register, v.SelfRef, v.AddressDefault,
|
|
v.LexiconMarkers, v.NGLexicon, v.Exemplars, v.Brightness, v.SinceCh, v.UntilCh); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, p := range pairs {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO address_pairs (
|
|
book_id, speaker_src, speaker_sense, addressee_src, addressee_sense,
|
|
register, form, closeness, since_ch, until_ch
|
|
) VALUES (?,?,?,?,?,?,?,?,?,?)`,
|
|
bookID, p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense,
|
|
p.Register, p.Form, p.Closeness, p.SinceCh, p.UntilCh); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// GlossaryForBook returns every glossary entry for a book with its aliases, in a
|
|
// DETERMINISTIC order (src, sense, since_ch, until_ch, status, dst) with aliases sorted
|
|
// — the stable materialization the hot-path matcher and the F1 memoryVersion() consume.
|
|
func (s *Store) GlossaryForBook(bookID string) ([]GlossaryEntry, error) {
|
|
ctx, cancel := opContext()
|
|
defer cancel()
|
|
rows, err := s.r.QueryContext(ctx, `
|
|
SELECT id, src, dst, type, sense, gender, speech, decl,
|
|
translit_policy, first_person, nickname_translation,
|
|
since_ch, until_ch, status, allow_short, source,
|
|
ruby_reading, ruby_class, confidence, note
|
|
FROM glossary WHERE book_id = ?
|
|
ORDER BY src, sense, since_ch, until_ch, status, dst`, bookID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []GlossaryEntry
|
|
byID := map[int64]int{} // glossary.id → index in out
|
|
for rows.Next() {
|
|
e := GlossaryEntry{BookID: bookID}
|
|
var allowShort int
|
|
if err := rows.Scan(&e.ID, &e.Src, &e.Dst, &e.Type, &e.Sense, &e.Gender, &e.Speech, &e.Decl,
|
|
&e.TranslitPolicy, &e.FirstPerson, &e.NicknameTranslation,
|
|
&e.SinceCh, &e.UntilCh, &e.Status, &allowShort, &e.Source,
|
|
&e.RubyReading, &e.RubyClass, &e.Confidence, &e.Note); err != nil {
|
|
return nil, err
|
|
}
|
|
e.AllowShort = allowShort != 0
|
|
byID[e.ID] = len(out)
|
|
out = append(out, e)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
// Aliases in a second pass, ordered so each entry's alias list is deterministic.
|
|
arows, err := s.r.QueryContext(ctx, `
|
|
SELECT term_id, alias, alias_type FROM glossary_aliases
|
|
WHERE book_id = ? ORDER BY term_id, alias`, bookID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer arows.Close()
|
|
for arows.Next() {
|
|
var termID int64
|
|
var a GlossaryAlias
|
|
if err := arows.Scan(&termID, &a.Alias, &a.AliasType); err != nil {
|
|
return nil, err
|
|
}
|
|
if idx, ok := byID[termID]; ok {
|
|
out[idx].Aliases = append(out[idx].Aliases, a)
|
|
}
|
|
}
|
|
return out, arows.Err()
|
|
}
|
|
|
|
// GlossaryRevisionsForBook returns the editorial-time journal (B1) for a book,
|
|
// newest first, for the report / a human audit.
|
|
func (s *Store) GlossaryRevisionsForBook(bookID string) ([]GlossaryRevision, error) {
|
|
return queryAll(s.r, `
|
|
SELECT src, sense, old_dst, new_dst, editorial_ts, reason
|
|
FROM glossary_revisions WHERE book_id = ?
|
|
ORDER BY id DESC`,
|
|
func(rows *sql.Rows) (GlossaryRevision, error) {
|
|
var r GlossaryRevision
|
|
err := rows.Scan(&r.Src, &r.Sense, &r.OldDst, &r.NewDst, &r.EditorialTS, &r.Reason)
|
|
return r, err
|
|
}, bookID)
|
|
}
|
|
|
|
// UpsertRetrievalState writes (or overwrites) the per-chunk retrieval-state record.
|
|
// Overwrite is the resolve semantics: recomputing the deterministic matcher over the
|
|
// same chunk reproduces the same row (self-heals on resume).
|
|
func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
|
|
ctx, cancel := opContext()
|
|
defer cancel()
|
|
_, err := s.w.ExecContext(ctx, `
|
|
INSERT INTO retrieval_state (
|
|
book_id, chapter, chunk_idx, snapshot_id,
|
|
n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted,
|
|
embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
|
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail,
|
|
n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail,
|
|
n_unverified_shown, n_unverified_followed,
|
|
n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail, updated_at
|
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
|
|
ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET
|
|
snapshot_id = excluded.snapshot_id,
|
|
n_exact_hits = excluded.n_exact_hits,
|
|
n_sticky = excluded.n_sticky,
|
|
n_ambiguous_flagged = excluded.n_ambiguous_flagged,
|
|
n_spoiler_blocked = excluded.n_spoiler_blocked,
|
|
n_evicted = excluded.n_evicted,
|
|
embedding_tier_used = excluded.embedding_tier_used,
|
|
n_postcheck_miss = excluded.n_postcheck_miss,
|
|
postcheck_detail = excluded.postcheck_detail,
|
|
injected_ids = excluded.injected_ids,
|
|
n_style_flags = excluded.n_style_flags,
|
|
style_detail = excluded.style_detail,
|
|
n_trust_gated_suppress = excluded.n_trust_gated_suppress,
|
|
trust_gate_detail = excluded.trust_gate_detail,
|
|
n_banknote_lines = excluded.n_banknote_lines,
|
|
banknote_detail = excluded.banknote_detail,
|
|
banknote_parse_fail = excluded.banknote_parse_fail,
|
|
banknote_truncated = excluded.banknote_truncated,
|
|
n_unverified_shown = excluded.n_unverified_shown,
|
|
n_unverified_followed = excluded.n_unverified_followed,
|
|
n_voice_flags = excluded.n_voice_flags,
|
|
voice_detail = excluded.voice_detail,
|
|
n_spoiler_leaks = excluded.n_spoiler_leaks,
|
|
spoiler_leak_detail = excluded.spoiler_leak_detail,
|
|
updated_at = excluded.updated_at`,
|
|
rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID,
|
|
rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted,
|
|
rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs,
|
|
rs.NStyleFlags, rs.StyleDetail, rs.NTrustGatedSuppress, rs.TrustGateDetail,
|
|
rs.NBanknoteLines, rs.BanknoteParseFail, rs.BanknoteTruncated, rs.BanknoteDetail,
|
|
rs.NUnverifiedShown, rs.NUnverifiedFollowed,
|
|
rs.NVoiceFlags, rs.VoiceDetail, rs.NSpoilerLeaks, rs.SpoilerLeakDetail)
|
|
return err
|
|
}
|
|
|
|
// GetRetrievalState returns the record for a chunk, or nil if none.
|
|
func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*RetrievalState, error) {
|
|
ctx, cancel := opContext()
|
|
defer cancel()
|
|
rs := RetrievalState{BookID: bookID, Chapter: chapter, ChunkIdx: chunkIdx}
|
|
err := s.r.QueryRowContext(ctx, `
|
|
SELECT snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked,
|
|
n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
|
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail,
|
|
n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail,
|
|
n_unverified_shown, n_unverified_followed,
|
|
n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail
|
|
FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`,
|
|
bookID, chapter, chunkIdx).Scan(
|
|
&rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked,
|
|
&rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
|
|
&rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail,
|
|
&rs.NBanknoteLines, &rs.BanknoteParseFail, &rs.BanknoteTruncated, &rs.BanknoteDetail,
|
|
&rs.NUnverifiedShown, &rs.NUnverifiedFollowed,
|
|
&rs.NVoiceFlags, &rs.VoiceDetail, &rs.NSpoilerLeaks, &rs.SpoilerLeakDetail)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &rs, nil
|
|
}
|
|
|
|
// RetrievalStatesForBook returns every retrieval-state record for a book, ordered for
|
|
// a stable report (the memory section of tmctl report).
|
|
func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error) {
|
|
return queryAll(s.r, `
|
|
SELECT chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged,
|
|
n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
|
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail,
|
|
n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail,
|
|
n_unverified_shown, n_unverified_followed,
|
|
n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail
|
|
FROM retrieval_state WHERE book_id = ?
|
|
ORDER BY chapter, chunk_idx`,
|
|
func(rows *sql.Rows) (RetrievalState, error) {
|
|
rs := RetrievalState{BookID: bookID}
|
|
err := rows.Scan(&rs.Chapter, &rs.ChunkIdx, &rs.SnapshotID, &rs.NExactHits, &rs.NSticky,
|
|
&rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed,
|
|
&rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
|
|
&rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail,
|
|
&rs.NBanknoteLines, &rs.BanknoteParseFail, &rs.BanknoteTruncated, &rs.BanknoteDetail,
|
|
&rs.NUnverifiedShown, &rs.NUnverifiedFollowed,
|
|
&rs.NVoiceFlags, &rs.VoiceDetail, &rs.NSpoilerLeaks, &rs.SpoilerLeakDetail)
|
|
return rs, err
|
|
}, bookID)
|
|
}
|
|
|
|
func boolToInt(b bool) int {
|
|
if b {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// priorRenderings reads the (src,sense,window)→dst map a replace overwrites. Separate function so the
|
|
// rows close before the caller's DELETEs run on the same tx — a tx holds one connection.
|
|
func priorRenderings(ctx context.Context, tx *sql.Tx, bookID string) (map[[4]any]string, error) {
|
|
rows, err := tx.QueryContext(ctx, `SELECT src, sense, since_ch, until_ch, dst FROM glossary WHERE book_id = ?`, bookID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
prior := map[[4]any]string{}
|
|
for rows.Next() {
|
|
var src, sense, dst string
|
|
var since, until int
|
|
if err := rows.Scan(&src, &sense, &since, &until, &dst); err != nil {
|
|
return nil, err
|
|
}
|
|
prior[[4]any{src, sense, since, until}] = dst
|
|
}
|
|
return prior, rows.Err()
|
|
}
|