package store import ( "context" "fmt" ) // migrations are applied in order; schema_version records the highest applied // version so re-runs are no-ops. Every step is also idempotent (CREATE TABLE // IF NOT EXISTS) so a half-applied database still converges. Append-only: // never edit an earlier migration (дисциплина vojo). var migrations = []string{ // v1: the Phase-0 operational schema. ` -- Snapshot контекста джобы (Р6, implementation-notes §3.2): материализованная -- запись; snapshot_id входит в request-hash, так что волатильность контекста -- заморожена на джобу. payload — JSON (версии промптов per-role, модели, -- версия чанкера, style sheet, резюме на момент старта). CREATE TABLE IF NOT EXISTS snapshots ( snapshot_id TEXT PRIMARY KEY, brief_hash TEXT NOT NULL, payload TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); -- Durable jobs: джоба = глава×стадия (Р6). Гранулярность СОХРАННОСТИ — чанк -- (checkpoints), джоба лишь группирует их и несёт snapshot_id. CREATE TABLE IF NOT EXISTS jobs ( id INTEGER PRIMARY KEY, book_id TEXT NOT NULL, chapter INTEGER NOT NULL, stage TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending', -- pending|running|done|failed snapshot_id TEXT NOT NULL REFERENCES snapshots(snapshot_id), created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')), UNIQUE (book_id, chapter, stage) ); -- Чанк-чекпоинты: сырой ответ LLM персистится атомарно вместе с settle -- (одна транзакция — SettleWithCheckpoint), ключ — request_hash от -- детерминированного рендера (§3.1). kill -9 теряет максимум один -- in-flight вызов. Найденный чекпоинт на resume = вызов не повторяется и -- не оплачивается второй раз. CREATE TABLE IF NOT EXISTS checkpoints ( request_hash TEXT PRIMARY KEY, job_id INTEGER NOT NULL REFERENCES jobs(id), chunk_idx INTEGER NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, -- регенерации Фазы 1: attempt входит в request-hash stage TEXT NOT NULL, role TEXT NOT NULL, model_requested TEXT NOT NULL, model_actual TEXT NOT NULL, response_text TEXT NOT NULL, usage_json TEXT NOT NULL, cost_usd REAL NOT NULL, finish_reason TEXT NOT NULL DEFAULT '', provider_request_id TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS checkpoints_job_idx ON checkpoints (job_id, chunk_idx); -- Дневной/книжный ledger с резервированием (порт семантики vojo spend): -- потолки считают committed + reserved на допуске, поэтому burst -- конкурентных вызовов не проскакивает потолок (TOCTOU-дисциплина). CREATE TABLE IF NOT EXISTS spend ( book_id TEXT NOT NULL, date TEXT NOT NULL, -- UTC YYYY-MM-DD committed_usd REAL NOT NULL DEFAULT 0, reserved_usd REAL NOT NULL DEFAULT 0, PRIMARY KEY (book_id, date) ); -- request_log: одна строка на LLM-вызов (Р7: per-книга/глава/чанк/стадия/ -- роль/модель, $, latency, cache-поля). Телеметрия, не источник денег — -- деньги живут в spend/checkpoints. CREATE TABLE IF NOT EXISTS request_log ( id INTEGER PRIMARY KEY, ts TEXT NOT NULL DEFAULT (datetime('now')), trace_id TEXT NOT NULL DEFAULT '', book_id TEXT NOT NULL, chapter INTEGER NOT NULL DEFAULT 0, chunk_idx INTEGER NOT NULL DEFAULT 0, stage TEXT NOT NULL DEFAULT '', role TEXT NOT NULL DEFAULT '', model_requested TEXT NOT NULL DEFAULT '', model_actual TEXT NOT NULL DEFAULT '', request_hash TEXT NOT NULL DEFAULT '', prompt_tokens INTEGER NOT NULL DEFAULT 0, cached_tokens INTEGER NOT NULL DEFAULT 0, cache_creation_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, reasoning_tokens INTEGER NOT NULL DEFAULT 0, cost_usd REAL NOT NULL DEFAULT 0, latency_ms INTEGER NOT NULL DEFAULT 0, finish_reason TEXT NOT NULL DEFAULT '', tm_hit INTEGER NOT NULL DEFAULT 0, -- ответ взят из чекпоинта, вызова не было degraded TEXT NOT NULL DEFAULT '', err TEXT NOT NULL DEFAULT '', ok INTEGER NOT NULL DEFAULT 1 ); CREATE INDEX IF NOT EXISTS request_log_ts_idx ON request_log (ts); `, // v2: per-chunk×stage disposition (Веха 2, D2). Резолв ПОВЕРХ append-only // checkpoints (keyed by request_hash-с-attempt) — не колонка на checkpoints: // на resume строка пересобирается из чекпоинтов, поэтому её потеря на kill -9 // самолечится (classify по тексту чекпоинта, без ре-биллинга). jobs.status // контентом НЕ расширяется — failed остаётся инфра-сбоем; плохой чанк живёт // здесь как disposition, а цикл продолжается. ` CREATE TABLE IF NOT EXISTS chunk_status ( book_id TEXT NOT NULL, chapter INTEGER NOT NULL, chunk_idx INTEGER NOT NULL, stage TEXT NOT NULL, snapshot_id TEXT NOT NULL, -- под каким снапшотом резолвнут (stale после --resnapshot игнорируется) content_hash TEXT NOT NULL DEFAULT '',-- сигнатура отрендеренных msgs (исходник НЕ в снапшоте): правка src делает позиционную строку недействительной, а не подаёт устаревший перевод disposition TEXT NOT NULL, -- ok | flagged | skipped flag_reason TEXT NOT NULL DEFAULT '', -- '' когда ok; Go-константа FlagReason attempts INTEGER NOT NULL DEFAULT 0, final_hash TEXT NOT NULL DEFAULT '', -- request_hash авторитетного чекпоинта (ok-путь подаёт его текст дальше) cost_usd REAL NOT NULL DEFAULT 0, -- сумма по ВСЕМ попыткам (F3-честно: ретраи учтены) detail TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (book_id, chapter, chunk_idx, stage) ); CREATE INDEX IF NOT EXISTS chunk_status_book_idx ON chunk_status (book_id, disposition); `, // v3: escalation flag on checkpoints (Веха 2.5, D12 single-hop). The fallback // draft (a deterministic content-failure routed to another model ONCE) is tagged // so the book's escalation.budget_usd can be enforced by summing ONLY escalation // spend, independent of the primary translation cost. On the checkpoint (written // in the same tx as the settle), NOT request_log, so the sum is durable across // resume / kill -9 (telemetry is fire-and-forget; money-path budgeting is not). `ALTER TABLE checkpoints ADD COLUMN escalation INTEGER NOT NULL DEFAULT 0;`, // v4: ruby/furigana readings captured on ingest (шаг 3a, 04-unhappy §4 / D9). A // ja epub's basereading carries the AUTHOR's reading of a // name/term; epub-v1 drops inline markup (02-mvp:25), which would silently kill // this layer, so ingest EXTRACTS the readings into this table instead. It is NOT // injected into any prompt (that is memory v2 + determinism-load-bearing) — this // step only CAPTURES + PERSISTS; memory v2 (шаг 4) consumes it into a glossary // name-lock (base→dst via reading, first_chapter→since_ch — D7 schema). PK // (book_id, base, reading): the same name may carry several readings across a // book (variant yomi), each a distinct row. first_chapter = the MIN chapter the // pair appears in (a glossary "since"); occurrences = the pair's full-book count // (a memory-v2 confidence signal). The persist is idempotent AND a full replace: // ingest aggregates each (base,reading) once over the WHOLE book, and persistRuby // DELETEs the book's set before re-inserting (store.ReplaceRubyReadings), so // re-ingesting the same source re-derives identical rows and a source edit // converges every column — including DROPPING a pair the edit removed (never a // lingering phantom, never a MIN-pin/increment that drifts on resume). ` CREATE TABLE IF NOT EXISTS ruby_readings ( book_id TEXT NOT NULL, base TEXT NOT NULL, -- the ruby BODY: the kanji/base surface form reading TEXT NOT NULL, -- the reading (furigana) captured for it first_chapter INTEGER NOT NULL,-- 1-based full-book MIN chapter the pair appears in (glossary "since"; idempotent REPLACE) occurrences INTEGER NOT NULL DEFAULT 0, -- full-book count (idempotent REPLACE, not increment) created_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (book_id, base, reading) ); CREATE INDEX IF NOT EXISTS ruby_readings_book_idx ON ruby_readings (book_id); `, // v5: the memory bank v2 glossary (шаг 4, registry architecture/06 §Гейты + D7). // The DETERMINISTIC substrate the hot-path matcher, the injection surface and the // post-check are built on. Rows are the frozen-at-job-start injectable pool whose // approved subset the F1 memoryVersion() hashes into the snapshot. Like ruby_readings // this is "dumb storage" (aggregation/classification live in the pipeline); the // pipeline REPLACES a book's whole glossary from its deterministic inputs (seed file // + classified ruby), so re-seeding is idempotent and a seed edit converges every // column. No cross-table FK (mirrors the ruby/chunkstatus dumb-storage style): both // child tables are scoped by book_id and replaced in the same transaction, term_id is // a logical link (glossary.id is a fresh autoincrement each replace and is NEVER hashed // into memoryVersion — only content columns are, ORDER BY-stable). ` CREATE TABLE IF NOT EXISTS glossary ( id INTEGER PRIMARY KEY, book_id TEXT NOT NULL, src TEXT NOT NULL, -- source key (raw; normalized in Go for matching, §3.6) dst TEXT NOT NULL DEFAULT '', -- approved Russian translation (LEMMA/base form); '' for a ruby candidate with no dst yet type TEXT NOT NULL DEFAULT '', -- name|term|title|… alias-graph node type (B3) sense TEXT NOT NULL DEFAULT '', -- A3 polysemy disambiguator (part of the uniqueness key) gender TEXT NOT NULL DEFAULT '', -- male|female|hidden|'' (C2; hidden = gender-avoidant rendering until reveal) speech TEXT NOT NULL DEFAULT '', -- ты/вы register note (series-bible-lite; the transition JOURNAL is deferred) decl TEXT NOT NULL DEFAULT '', -- JSON {"invariant":bool,"forms":[...]}: the dst declension forms the post-check accepts (E1/E2 safety, decl = grammaticality mechanism, not a market feature) translit_policy TEXT NOT NULL DEFAULT '', -- B5 western-name-via-katakana (field only in v1; validator = Phase 2) first_person TEXT NOT NULL DEFAULT '', -- D7 first_person field (field only) nickname_translation TEXT NOT NULL DEFAULT '', -- D7 (keep in v1: a column + a "translate once" lock) since_ch INTEGER NOT NULL DEFAULT 0, -- spoiler window start; 0 = valid from the beginning (C1 hard reject-gate below since_ch) until_ch INTEGER NOT NULL DEFAULT 0, -- spoiler window end; 0 = no end (valid forever) status TEXT NOT NULL DEFAULT 'auto',-- auto|draft|approved (term status machine; only approved is CONFIRMED-injected) allow_short INTEGER NOT NULL DEFAULT 0, -- rare escape hatch: permit a below-min_key_len key (A3 single-key ban override) source TEXT NOT NULL DEFAULT '', -- provenance: seed|ruby|auto (who created the row) ruby_reading TEXT NOT NULL DEFAULT '', -- ruby-seeded rows: the furigana reading (bridge to Polivanov dst validation, B6 Phase 2 — NOT a dst) ruby_class TEXT NOT NULL DEFAULT '', -- name|gloss|ambiguous (ruby classifier §8: phonetic-name → name-lock candidate, unrelated reading → gloss/footnote, else flag) confidence INTEGER NOT NULL DEFAULT 0, -- ruby occurrences count = confidence prior (NOT a direct promotion to approved) note TEXT NOT NULL DEFAULT '', -- B2: at most one row per (source key, sense, spoiler window). since_ch/until_ch are -- NOT NULL with default 0, so the SQLite "NULLs are distinct in UNIQUE" gotcha cannot -- silently admit a duplicate. A term whose approved rendering changes across a spoiler -- boundary is two rows with different windows (legitimately distinct), not a collision. UNIQUE (book_id, src, sense, since_ch, until_ch) ); CREATE INDEX IF NOT EXISTS glossary_book_idx ON glossary (book_id, status); CREATE TABLE IF NOT EXISTS glossary_aliases ( id INTEGER PRIMARY KEY, book_id TEXT NOT NULL, term_id INTEGER NOT NULL, -- logical link to glossary.id within the SAME replace tx (no hard FK — dumb storage, scoped+replaced by book_id) alias TEXT NOT NULL, -- an alternative source surface (raw; normalized in Go for matching) alias_type TEXT NOT NULL DEFAULT '', -- имя|цзы|хао|прозвище|титул (B3 alias-graph node type) UNIQUE (book_id, term_id, alias) ); CREATE INDEX IF NOT EXISTS glossary_aliases_term_idx ON glossary_aliases (term_id); CREATE INDEX IF NOT EXISTS glossary_aliases_book_idx ON glossary_aliases (book_id); -- B1 editorial-time axis, ORTHOGONAL to the spoiler window (since_ch/until_ch is the -- STORY timeline; this is the EDITORIAL timeline). Append-only journal of approved-dst -- changes so a stale first-translation-wins rendering is DETECTABLE (DelTA PNR generates -- stale/drift; this is the substrate that catches it). Not injected → not in any hash; -- datetime('now') here is fine (telemetry, like request_log), never feeds memoryVersion. CREATE TABLE IF NOT EXISTS glossary_revisions ( id INTEGER PRIMARY KEY, book_id TEXT NOT NULL, src TEXT NOT NULL, sense TEXT NOT NULL DEFAULT '', old_dst TEXT NOT NULL DEFAULT '', new_dst TEXT NOT NULL, editorial_ts TEXT NOT NULL DEFAULT (datetime('now')), reason TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS glossary_revisions_book_idx ON glossary_revisions (book_id, src); -- Per-chunk retrieval-state (registry gate #4 / A1/F2/G2/E1): observability of memory -- degradation FROM DAY ONE — the mechanism that converts silent degradation into loud. -- Recomputed deterministically each run (the matcher is $0), so it self-heals on resume; -- NOT wire-load-bearing (it records what the injection did, it does not change the request). CREATE TABLE IF NOT EXISTS retrieval_state ( book_id TEXT NOT NULL, chapter INTEGER NOT NULL, chunk_idx INTEGER NOT NULL, snapshot_id TEXT NOT NULL, n_exact_hits INTEGER NOT NULL DEFAULT 0, -- entries matched by an exact key/alias in this chunk n_sticky INTEGER NOT NULL DEFAULT 0, -- entries carried by scene-inertia (A5), not re-matched here n_ambiguous_flagged INTEGER NOT NULL DEFAULT 0, -- injected AMBIGUOUS (auto/draft) → "unverified" + forced post-check (A2) n_spoiler_blocked INTEGER NOT NULL DEFAULT 0, -- matched but hard-rejected by the since_ch/until_ch window (C1) n_evicted INTEGER NOT NULL DEFAULT 0, -- dropped by the token budget (F2 — logged, never silent) embedding_tier_used INTEGER NOT NULL DEFAULT 0, -- always 0 in v1 (no embeddings on the hot path, Р3) n_postcheck_miss INTEGER NOT NULL DEFAULT 0, -- E1 flags: a confirmed src matched but no accepted dst form is in the output postcheck_detail TEXT NOT NULL DEFAULT '', -- JSON [{src,dst}] of the misses, for the report / human injected_ids TEXT NOT NULL DEFAULT '', -- JSON of exact-matched entity keys (audit + sticky reconstruction) updated_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (book_id, chapter, chunk_idx) ); `, // v6: surface single-hop escalation on the chunk_status RESOLVE (D15.3 status/redrive). // The fallback fact lived only on the checkpoint (escalation flag, v3); folding it into // chunk_status lets `tmctl status` project escalation-on-flagged-resume WITHOUT re-reading // checkpoints (the deferred Milestone-2.5 finding #6 — the escalation column the status // read-model needs). Derivable from checkpoints, so a lost/rebuilt row self-heals on the // next resolve. escalation_model is the fallback that answered (empty when not escalated). ` ALTER TABLE chunk_status ADD COLUMN escalated INTEGER NOT NULL DEFAULT 0; ALTER TABLE chunk_status ADD COLUMN escalation_model TEXT NOT NULL DEFAULT ''; `, // v7: the cheap deterministic style flaggers (dialogue-dash, yofikator, translit-interjection // blocklist, 万/億 magnitude gate — cheapgates.go) ride the same per-chunk observability row as // the glossary post-check. n_style_flags is the total hit count; style_detail is the JSON // breakdown for the report. Recomputed deterministically each run (self-heals on resume), NOT // wire-load-bearing (observability, never a disposition). ` ALTER TABLE retrieval_state ADD COLUMN n_style_flags INTEGER NOT NULL DEFAULT 0; ALTER TABLE retrieval_state ADD COLUMN style_detail TEXT NOT NULL DEFAULT ''; `, } // migrate runs all pending migrations on the write pool, one transaction per // step, recording each in schema_version. func (s *Store) migrate(ctx context.Context) error { if _, err := s.w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY)`); err != nil { return fmt.Errorf("store: schema_version: %w", err) } var current int if err := s.w.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_version`).Scan(¤t); err != nil { return fmt.Errorf("store: read version: %w", err) } for v := current; v < len(migrations); v++ { tx, err := s.w.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("store: begin migration %d: %w", v+1, err) } if _, err := tx.ExecContext(ctx, migrations[v]); err != nil { _ = tx.Rollback() return fmt.Errorf("store: apply migration %d: %w", v+1, err) } if _, err := tx.ExecContext(ctx, `INSERT INTO schema_version (version) VALUES (?)`, v+1); err != nil { _ = tx.Rollback() return fmt.Errorf("store: record migration %d: %w", v+1, err) } if err := tx.Commit(); err != nil { return fmt.Errorf("store: commit migration %d: %w", v+1, err) } } return nil }