312 lines
12 KiB
Go
312 lines
12 KiB
Go
package store
|
|
|
|
import (
|
|
"database/sql"
|
|
"errors"
|
|
)
|
|
|
|
// glossary.go: durable storage for the memory bank v2 glossary (schema v5). Dumb
|
|
// storage, mirroring ruby.go/chunkstatus.go: the pipeline owns aggregation,
|
|
// classification and matching; this layer only replaces and reads back. A book's
|
|
// glossary is a deterministic function of its inputs (seed file + classified ruby),
|
|
// so ReplaceGlossary rewrites the WHOLE book set in one transaction — re-seeding the
|
|
// identical inputs rewrites identical rows, a seed edit converges every column, and a
|
|
// removed term disappears instead of lingering (the same full-replace idempotency the
|
|
// ruby step adopted after external-review #6).
|
|
|
|
// GlossaryEntry is one term row plus its aliases. Strings are stored raw; the
|
|
// pipeline normalizes for matching (§3.6). Decl is a JSON blob the post-check reads.
|
|
type GlossaryEntry struct {
|
|
ID int64
|
|
BookID string
|
|
Src string
|
|
Dst string
|
|
Type string
|
|
Sense string
|
|
Gender string
|
|
Speech string
|
|
Decl string // JSON {"invariant":bool,"forms":[...]}
|
|
TranslitPolicy string
|
|
FirstPerson string
|
|
NicknameTranslation string
|
|
SinceCh int
|
|
UntilCh int
|
|
Status string // auto|draft|approved
|
|
AllowShort bool
|
|
Source string // seed|ruby|auto
|
|
RubyReading string
|
|
RubyClass string
|
|
Confidence int
|
|
Note string
|
|
Aliases []GlossaryAlias
|
|
}
|
|
|
|
// GlossaryAlias is one alias-graph edge (an alternative source surface + its type).
|
|
type GlossaryAlias struct {
|
|
Alias string
|
|
AliasType string
|
|
}
|
|
|
|
// GlossaryRevision is one append-only editorial-time record (B1).
|
|
type GlossaryRevision struct {
|
|
Src string
|
|
Sense string
|
|
OldDst string
|
|
NewDst string
|
|
EditorialTS string
|
|
Reason string
|
|
}
|
|
|
|
// RetrievalState is one per-chunk observability record (registry gate #4).
|
|
type RetrievalState struct {
|
|
BookID string
|
|
Chapter int
|
|
ChunkIdx int
|
|
SnapshotID string
|
|
NExactHits int
|
|
NSticky int
|
|
NAmbiguousFlagged int
|
|
NSpoilerBlocked int
|
|
NEvicted int
|
|
EmbeddingTierUsed int
|
|
NPostcheckMiss int
|
|
PostcheckDetail string // JSON
|
|
InjectedIDs string // JSON
|
|
NStyleFlags int // cheap style/number gate hits (dialogue-dash, ё, translit-interj, 万/億)
|
|
StyleDetail string // JSON breakdown of the style-flag hits
|
|
}
|
|
|
|
// ReplaceGlossary replaces a book's ENTIRE glossary (rows + aliases) with the given
|
|
// entries in ONE transaction, and appends a B1 revision record for every term whose
|
|
// approved rendering (dst) changed since the previous set. Scoped to book_id, so other
|
|
// books are untouched. Each entry's BookID must equal bookID. term_id linkage is
|
|
// established within the tx: a glossary row is inserted, its rowid taken, then its
|
|
// aliases inserted against that id (ids are fresh each replace — never hashed).
|
|
func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error {
|
|
ctx, cancel := opContext()
|
|
defer cancel()
|
|
tx, err := s.w.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
// Snapshot the prior (src,sense,window)→dst to detect editorial-time changes (B1).
|
|
prior := map[[4]any]string{}
|
|
rows, err := tx.QueryContext(ctx, `SELECT src, sense, since_ch, until_ch, dst FROM glossary WHERE book_id = ?`, bookID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for rows.Next() {
|
|
var src, sense, dst string
|
|
var since, until int
|
|
if err := rows.Scan(&src, &sense, &since, &until, &dst); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
prior[[4]any{src, sense, since, until}] = dst
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
rows.Close()
|
|
|
|
if _, err := tx.ExecContext(ctx, `DELETE FROM glossary_aliases WHERE book_id = ?`, bookID); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `DELETE FROM glossary WHERE book_id = ?`, bookID); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, e := range entries {
|
|
res, err := tx.ExecContext(ctx, `
|
|
INSERT INTO glossary (
|
|
book_id, src, dst, type, sense, gender, speech, decl,
|
|
translit_policy, first_person, nickname_translation,
|
|
since_ch, until_ch, status, allow_short, source,
|
|
ruby_reading, ruby_class, confidence, note
|
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
e.BookID, e.Src, e.Dst, e.Type, e.Sense, e.Gender, e.Speech, e.Decl,
|
|
e.TranslitPolicy, e.FirstPerson, e.NicknameTranslation,
|
|
e.SinceCh, e.UntilCh, e.Status, boolToInt(e.AllowShort), e.Source,
|
|
e.RubyReading, e.RubyClass, e.Confidence, e.Note)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
termID, err := res.LastInsertId()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, a := range e.Aliases {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO glossary_aliases (book_id, term_id, alias, alias_type)
|
|
VALUES (?, ?, ?, ?)`, e.BookID, termID, a.Alias, a.AliasType); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// B1: a term that existed with a non-empty dst and now renders differently
|
|
// (also non-empty) is an editorial change → append the revision journal row.
|
|
if old, ok := prior[[4]any{e.Src, e.Sense, e.SinceCh, e.UntilCh}]; ok && old != "" && e.Dst != "" && old != e.Dst {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO glossary_revisions (book_id, src, sense, old_dst, new_dst, reason)
|
|
VALUES (?, ?, ?, ?, ?, ?)`, e.BookID, e.Src, e.Sense, old, e.Dst, "seed-replace"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// GlossaryForBook returns every glossary entry for a book with its aliases, in a
|
|
// DETERMINISTIC order (src, sense, since_ch, until_ch, status, dst) with aliases sorted
|
|
// — the stable materialization the hot-path matcher and the F1 memoryVersion() consume.
|
|
func (s *Store) GlossaryForBook(bookID string) ([]GlossaryEntry, error) {
|
|
ctx, cancel := opContext()
|
|
defer cancel()
|
|
rows, err := s.r.QueryContext(ctx, `
|
|
SELECT id, src, dst, type, sense, gender, speech, decl,
|
|
translit_policy, first_person, nickname_translation,
|
|
since_ch, until_ch, status, allow_short, source,
|
|
ruby_reading, ruby_class, confidence, note
|
|
FROM glossary WHERE book_id = ?
|
|
ORDER BY src, sense, since_ch, until_ch, status, dst`, bookID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []GlossaryEntry
|
|
byID := map[int64]int{} // glossary.id → index in out
|
|
for rows.Next() {
|
|
e := GlossaryEntry{BookID: bookID}
|
|
var allowShort int
|
|
if err := rows.Scan(&e.ID, &e.Src, &e.Dst, &e.Type, &e.Sense, &e.Gender, &e.Speech, &e.Decl,
|
|
&e.TranslitPolicy, &e.FirstPerson, &e.NicknameTranslation,
|
|
&e.SinceCh, &e.UntilCh, &e.Status, &allowShort, &e.Source,
|
|
&e.RubyReading, &e.RubyClass, &e.Confidence, &e.Note); err != nil {
|
|
return nil, err
|
|
}
|
|
e.AllowShort = allowShort != 0
|
|
byID[e.ID] = len(out)
|
|
out = append(out, e)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
// Aliases in a second pass, ordered so each entry's alias list is deterministic.
|
|
arows, err := s.r.QueryContext(ctx, `
|
|
SELECT term_id, alias, alias_type FROM glossary_aliases
|
|
WHERE book_id = ? ORDER BY term_id, alias`, bookID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer arows.Close()
|
|
for arows.Next() {
|
|
var termID int64
|
|
var a GlossaryAlias
|
|
if err := arows.Scan(&termID, &a.Alias, &a.AliasType); err != nil {
|
|
return nil, err
|
|
}
|
|
if idx, ok := byID[termID]; ok {
|
|
out[idx].Aliases = append(out[idx].Aliases, a)
|
|
}
|
|
}
|
|
return out, arows.Err()
|
|
}
|
|
|
|
// GlossaryRevisionsForBook returns the editorial-time journal (B1) for a book,
|
|
// newest first, for the report / a human audit.
|
|
func (s *Store) GlossaryRevisionsForBook(bookID string) ([]GlossaryRevision, error) {
|
|
return queryAll(s.r, `
|
|
SELECT src, sense, old_dst, new_dst, editorial_ts, reason
|
|
FROM glossary_revisions WHERE book_id = ?
|
|
ORDER BY id DESC`,
|
|
func(rows *sql.Rows) (GlossaryRevision, error) {
|
|
var r GlossaryRevision
|
|
err := rows.Scan(&r.Src, &r.Sense, &r.OldDst, &r.NewDst, &r.EditorialTS, &r.Reason)
|
|
return r, err
|
|
}, bookID)
|
|
}
|
|
|
|
// UpsertRetrievalState writes (or overwrites) the per-chunk retrieval-state record.
|
|
// Overwrite is the resolve semantics: recomputing the deterministic matcher over the
|
|
// same chunk reproduces the same row (self-heals on resume).
|
|
func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
|
|
ctx, cancel := opContext()
|
|
defer cancel()
|
|
_, err := s.w.ExecContext(ctx, `
|
|
INSERT INTO retrieval_state (
|
|
book_id, chapter, chunk_idx, snapshot_id,
|
|
n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted,
|
|
embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
|
n_style_flags, style_detail, updated_at
|
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
|
|
ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET
|
|
snapshot_id = excluded.snapshot_id,
|
|
n_exact_hits = excluded.n_exact_hits,
|
|
n_sticky = excluded.n_sticky,
|
|
n_ambiguous_flagged = excluded.n_ambiguous_flagged,
|
|
n_spoiler_blocked = excluded.n_spoiler_blocked,
|
|
n_evicted = excluded.n_evicted,
|
|
embedding_tier_used = excluded.embedding_tier_used,
|
|
n_postcheck_miss = excluded.n_postcheck_miss,
|
|
postcheck_detail = excluded.postcheck_detail,
|
|
injected_ids = excluded.injected_ids,
|
|
n_style_flags = excluded.n_style_flags,
|
|
style_detail = excluded.style_detail,
|
|
updated_at = excluded.updated_at`,
|
|
rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID,
|
|
rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted,
|
|
rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs,
|
|
rs.NStyleFlags, rs.StyleDetail)
|
|
return err
|
|
}
|
|
|
|
// GetRetrievalState returns the record for a chunk, or nil if none.
|
|
func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*RetrievalState, error) {
|
|
ctx, cancel := opContext()
|
|
defer cancel()
|
|
rs := RetrievalState{BookID: bookID, Chapter: chapter, ChunkIdx: chunkIdx}
|
|
err := s.r.QueryRowContext(ctx, `
|
|
SELECT snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked,
|
|
n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
|
n_style_flags, style_detail
|
|
FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`,
|
|
bookID, chapter, chunkIdx).Scan(
|
|
&rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked,
|
|
&rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
|
|
&rs.NStyleFlags, &rs.StyleDetail)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &rs, nil
|
|
}
|
|
|
|
// RetrievalStatesForBook returns every retrieval-state record for a book, ordered for
|
|
// a stable report (the memory section of tmctl report).
|
|
func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error) {
|
|
return queryAll(s.r, `
|
|
SELECT chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged,
|
|
n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
|
n_style_flags, style_detail
|
|
FROM retrieval_state WHERE book_id = ?
|
|
ORDER BY chapter, chunk_idx`,
|
|
func(rows *sql.Rows) (RetrievalState, error) {
|
|
rs := RetrievalState{BookID: bookID}
|
|
err := rows.Scan(&rs.Chapter, &rs.ChunkIdx, &rs.SnapshotID, &rs.NExactHits, &rs.NSticky,
|
|
&rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed,
|
|
&rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
|
|
&rs.NStyleFlags, &rs.StyleDetail)
|
|
return rs, err
|
|
}, bookID)
|
|
}
|
|
|
|
func boolToInt(b bool) int {
|
|
if b {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|