textmachine/backend/internal/store/migrate.go

package store

import (
	"context"
	"database/sql"
	"fmt"
)

// migrations are applied in order; schema_version records the highest applied
// version so re-runs are no-ops. Every step is also idempotent (CREATE TABLE
// IF NOT EXISTS) so a half-applied database still converges. Append-only:
// never edit an earlier migration (vojo discipline).
var migrations = []string{
	// v1: the Phase-0 operational schema.
	`
	-- Snapshot контекста джобы (Р6, implementation-notes §3.2): материализованная
	-- запись; snapshot_id входит в request-hash, так что волатильность контекста
	-- заморожена на джобу. payload — JSON (версии промптов per-role, модели,
	-- версия чанкера, style sheet, резюме на момент старта).
	CREATE TABLE IF NOT EXISTS snapshots (
		snapshot_id TEXT PRIMARY KEY,
		brief_hash  TEXT NOT NULL,
		payload     TEXT NOT NULL,
		created_at  TEXT NOT NULL DEFAULT (datetime('now'))
	);

	-- Durable jobs: джоба = глава×стадия (Р6). Гранулярность СОХРАННОСТИ — чанк
	-- (checkpoints), джоба лишь группирует их и несёт snapshot_id.
	CREATE TABLE IF NOT EXISTS jobs (
		id          INTEGER PRIMARY KEY,
		book_id     TEXT NOT NULL,
		chapter     INTEGER NOT NULL,
		stage       TEXT NOT NULL,
		status      TEXT NOT NULL DEFAULT 'pending', -- pending|running|done|failed
		snapshot_id TEXT NOT NULL REFERENCES snapshots(snapshot_id),
		created_at  TEXT NOT NULL DEFAULT (datetime('now')),
		updated_at  TEXT NOT NULL DEFAULT (datetime('now')),
		UNIQUE (book_id, chapter, stage)
	);

	-- Чанк-чекпоинты: сырой ответ LLM персистится атомарно вместе с settle
	-- (одна транзакция — SettleWithCheckpoint), ключ — request_hash от
	-- детерминированного рендера (§3.1). kill -9 теряет максимум один
	-- in-flight вызов. Найденный чекпоинт на resume = вызов не повторяется и
	-- не оплачивается второй раз.
	CREATE TABLE IF NOT EXISTS checkpoints (
		request_hash        TEXT PRIMARY KEY,
		job_id              INTEGER NOT NULL REFERENCES jobs(id),
		chunk_idx           INTEGER NOT NULL,
		attempt             INTEGER NOT NULL DEFAULT 0, -- регенерации Фазы 1: attempt входит в request-hash
		stage               TEXT NOT NULL,
		role                TEXT NOT NULL,
		model_requested     TEXT NOT NULL,
		model_actual        TEXT NOT NULL,
		response_text       TEXT NOT NULL,
		usage_json          TEXT NOT NULL,
		cost_usd            REAL NOT NULL,
		finish_reason       TEXT NOT NULL DEFAULT '',
		provider_request_id TEXT NOT NULL DEFAULT '',
		created_at          TEXT NOT NULL DEFAULT (datetime('now'))
	);
	CREATE INDEX IF NOT EXISTS checkpoints_job_idx ON checkpoints (job_id, chunk_idx);

	-- Дневной/книжный ledger с резервированием (порт семантики vojo spend):
	-- потолки считают committed + reserved на допуске, поэтому burst
	-- конкурентных вызовов не проскакивает потолок (TOCTOU-дисциплина).
	CREATE TABLE IF NOT EXISTS spend (
		book_id      TEXT NOT NULL,
		date         TEXT NOT NULL, -- UTC YYYY-MM-DD
		committed_usd REAL NOT NULL DEFAULT 0,
		reserved_usd  REAL NOT NULL DEFAULT 0,
		PRIMARY KEY (book_id, date)
	);

	-- request_log: одна строка на LLM-вызов (Р7: per-книга/глава/чанк/стадия/
	-- роль/модель, $, latency, cache-поля). Телеметрия, не источник денег —
	-- деньги живут в spend/checkpoints.
	CREATE TABLE IF NOT EXISTS request_log (
		id                  INTEGER PRIMARY KEY,
		ts                  TEXT NOT NULL DEFAULT (datetime('now')),
		trace_id            TEXT NOT NULL DEFAULT '',
		book_id             TEXT NOT NULL,
		chapter             INTEGER NOT NULL DEFAULT 0,
		chunk_idx           INTEGER NOT NULL DEFAULT 0,
		stage               TEXT NOT NULL DEFAULT '',
		role                TEXT NOT NULL DEFAULT '',
		model_requested     TEXT NOT NULL DEFAULT '',
		model_actual        TEXT NOT NULL DEFAULT '',
		request_hash        TEXT NOT NULL DEFAULT '',
		prompt_tokens       INTEGER NOT NULL DEFAULT 0,
		cached_tokens       INTEGER NOT NULL DEFAULT 0,
		cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
		completion_tokens   INTEGER NOT NULL DEFAULT 0,
		reasoning_tokens    INTEGER NOT NULL DEFAULT 0,
		cost_usd            REAL NOT NULL DEFAULT 0,
		latency_ms          INTEGER NOT NULL DEFAULT 0,
		finish_reason       TEXT NOT NULL DEFAULT '',
		tm_hit              INTEGER NOT NULL DEFAULT 0, -- ответ взят из чекпоинта, вызова не было
		degraded            TEXT NOT NULL DEFAULT '',
		err                 TEXT NOT NULL DEFAULT '',
		ok                  INTEGER NOT NULL DEFAULT 1
	);
	CREATE INDEX IF NOT EXISTS request_log_ts_idx ON request_log (ts);
	`,
	// v2: per-chunk×stage disposition (Milestone 2, D2). The resolve sits ON TOP OF
	// append-only checkpoints (keyed by request_hash-with-attempt) — not a column on
	// checkpoints: on resume the row is rebuilt from checkpoints, so its loss on kill -9
	// self-heals (classify from the checkpoint's text, without re-billing). jobs.status
	// is NOT extended with content — failed stays an infra failure; a bad chunk lives
	// here as a disposition, and the loop continues.
	`
	CREATE TABLE IF NOT EXISTS chunk_status (
		book_id     TEXT NOT NULL,
		chapter     INTEGER NOT NULL,
		chunk_idx   INTEGER NOT NULL,
		stage       TEXT NOT NULL,
		snapshot_id TEXT NOT NULL,            -- под каким снапшотом резолвнут (stale после --resnapshot игнорируется)
		content_hash TEXT NOT NULL DEFAULT '',-- сигнатура отрендеренных msgs (исходник НЕ в снапшоте): правка src делает позиционную строку недействительной, а не подаёт устаревший перевод
		disposition TEXT NOT NULL,            -- ok | flagged | skipped
		flag_reason TEXT NOT NULL DEFAULT '', -- '' когда ok; Go-константа FlagReason
		attempts    INTEGER NOT NULL DEFAULT 0,
		final_hash  TEXT NOT NULL DEFAULT '', -- request_hash авторитетного чекпоинта (ok-путь подаёт его текст дальше)
		cost_usd    REAL NOT NULL DEFAULT 0,  -- сумма по ВСЕМ попыткам (F3-честно: ретраи учтены)
		detail      TEXT NOT NULL DEFAULT '',
		updated_at  TEXT NOT NULL DEFAULT (datetime('now')),
		PRIMARY KEY (book_id, chapter, chunk_idx, stage)
	);
	CREATE INDEX IF NOT EXISTS chunk_status_book_idx ON chunk_status (book_id, disposition);
	`,
	// v3: escalation flag on checkpoints (Milestone 2.5, D12 single-hop). The fallback
	// draft (a deterministic content-failure routed to another model ONCE) is tagged
	// so the book's escalation.budget_usd can be enforced by summing ONLY escalation
	// spend, independent of the primary translation cost. On the checkpoint (written
	// in the same tx as the settle), NOT request_log, so the sum is durable across
	// resume / kill -9 (telemetry is fire-and-forget; money-path budgeting is not).
	`ALTER TABLE checkpoints ADD COLUMN escalation INTEGER NOT NULL DEFAULT 0;`,
	// v4: ruby/furigana readings captured on ingest (step 3a, 04-unhappy §4 / D9). A
	// ja epub's <ruby>base<rt>reading</rt></ruby> carries the AUTHOR's reading of a
	// name/term; epub-v1 drops inline markup (02-mvp:25), which would silently kill
	// this layer, so ingest EXTRACTS the readings into this table instead. It is NOT
	// injected into any prompt (that is memory v2 + determinism-load-bearing) — this
	// step only CAPTURES + PERSISTS; memory v2 (step 4) consumes it into a glossary
	// name-lock (base→dst via reading, first_chapter→since_ch — D7 schema). PK
	// (book_id, base, reading): the same name may carry several readings across a
	// book (variant yomi), each a distinct row. first_chapter = the MIN chapter the
	// pair appears in (a glossary "since"); occurrences = the pair's full-book count
	// (a memory-v2 confidence signal). The persist is idempotent AND a full replace:
	// ingest aggregates each (base,reading) once over the WHOLE book, and persistRuby
	// DELETEs the book's set before re-inserting (store.ReplaceRubyReadings), so
	// re-ingesting the same source re-derives identical rows and a source edit
	// converges every column — including DROPPING a pair the edit removed (never a
	// lingering phantom, never a MIN-pin/increment that drifts on resume).
	`
	CREATE TABLE IF NOT EXISTS ruby_readings (
		book_id       TEXT NOT NULL,
		base          TEXT NOT NULL,   -- the ruby BODY: the kanji/base surface form
		reading       TEXT NOT NULL,   -- the <rt> reading (furigana) captured for it
		first_chapter INTEGER NOT NULL,-- 1-based full-book MIN chapter the pair appears in (glossary "since"; idempotent REPLACE)
		occurrences   INTEGER NOT NULL DEFAULT 0, -- full-book count (idempotent REPLACE, not increment)
		created_at    TEXT NOT NULL DEFAULT (datetime('now')),
		PRIMARY KEY (book_id, base, reading)
	);
	CREATE INDEX IF NOT EXISTS ruby_readings_book_idx ON ruby_readings (book_id);
	`,
	// v5: the memory bank v2 glossary (step 4, registry architecture/06 §Гейты + D7).
	// The DETERMINISTIC substrate the hot-path matcher, the injection surface and the
	// post-check are built on. Rows are the frozen-at-job-start injectable pool whose
	// approved subset the F1 memoryVersion() hashes into the snapshot. Like ruby_readings
	// this is "dumb storage" (aggregation/classification live in the pipeline); the
	// pipeline REPLACES a book's whole glossary from its deterministic inputs (seed file
	// + classified ruby), so re-seeding is idempotent and a seed edit converges every
	// column. No cross-table FK (mirrors the ruby/chunkstatus dumb-storage style): both
	// child tables are scoped by book_id and replaced in the same transaction, term_id is
	// a logical link (glossary.id is a fresh autoincrement each replace and is NEVER hashed
	// into memoryVersion — only content columns are, ORDER BY-stable).
	`
	CREATE TABLE IF NOT EXISTS glossary (
		id            INTEGER PRIMARY KEY,
		book_id       TEXT NOT NULL,
		src           TEXT NOT NULL,               -- source key (raw; normalized in Go for matching, §3.6)
		dst           TEXT NOT NULL DEFAULT '',    -- approved Russian translation (LEMMA/base form); '' for a ruby candidate with no dst yet
		type          TEXT NOT NULL DEFAULT '',    -- name|term|title|… alias-graph node type (B3)
		sense         TEXT NOT NULL DEFAULT '',    -- A3 polysemy disambiguator (part of the uniqueness key)
		gender        TEXT NOT NULL DEFAULT '',    -- male|female|hidden|'' (C2; hidden = gender-avoidant rendering until reveal)
		speech        TEXT NOT NULL DEFAULT '',    -- DEPRECATED (pack-19): superseded by the address_pairs journal (v13); never read, never hashed
		decl          TEXT NOT NULL DEFAULT '',    -- JSON {"invariant":bool,"forms":[...]}: the dst declension forms the post-check accepts (E1/E2 safety, decl = grammaticality mechanism, not a market feature)
		translit_policy TEXT NOT NULL DEFAULT '',  -- B5 western-name-via-katakana (field only in v1; validator = Phase 2)
		first_person  TEXT NOT NULL DEFAULT '',    -- D7 first_person field (field only)
		nickname_translation TEXT NOT NULL DEFAULT '', -- D7 (keep in v1: a column + a "translate once" lock)
		since_ch      INTEGER NOT NULL DEFAULT 0,  -- spoiler window start; 0 = valid from the beginning (C1 hard reject-gate below since_ch)
		until_ch      INTEGER NOT NULL DEFAULT 0,  -- spoiler window end; 0 = no end (valid forever)
		status        TEXT NOT NULL DEFAULT 'auto',-- auto|draft|approved (term status machine; only approved is CONFIRMED-injected)
		allow_short   INTEGER NOT NULL DEFAULT 0,  -- rare escape hatch: permit a below-min_key_len key (A3 single-key ban override)
		source        TEXT NOT NULL DEFAULT '',    -- provenance: seed|ruby|auto (who created the row)
		ruby_reading  TEXT NOT NULL DEFAULT '',    -- ruby-seeded rows: the furigana reading (bridge to Polivanov dst validation, B6 Phase 2 — NOT a dst)
		ruby_class    TEXT NOT NULL DEFAULT '',    -- name|gloss|ambiguous (ruby classifier §8: phonetic-name → name-lock candidate, unrelated reading → gloss/footnote, else flag)
		confidence    INTEGER NOT NULL DEFAULT 0,  -- ruby occurrences count = confidence prior (NOT a direct promotion to approved)
		note          TEXT NOT NULL DEFAULT '',
		-- B2: at most one row per (source key, sense, spoiler window). since_ch/until_ch are
		-- NOT NULL with default 0, so the SQLite "NULLs are distinct in UNIQUE" gotcha cannot
		-- silently admit a duplicate. A term whose approved rendering changes across a spoiler
		-- boundary is two rows with different windows (legitimately distinct), not a collision.
		UNIQUE (book_id, src, sense, since_ch, until_ch)
	);
	CREATE INDEX IF NOT EXISTS glossary_book_idx ON glossary (book_id, status);

	CREATE TABLE IF NOT EXISTS glossary_aliases (
		id         INTEGER PRIMARY KEY,
		book_id    TEXT NOT NULL,
		term_id    INTEGER NOT NULL,          -- logical link to glossary.id within the SAME replace tx (no hard FK — dumb storage, scoped+replaced by book_id)
		alias      TEXT NOT NULL,             -- an alternative source surface (raw; normalized in Go for matching)
		alias_type TEXT NOT NULL DEFAULT '',  -- name|zi|hao|nickname|title (B3 alias-graph node type)
		UNIQUE (book_id, term_id, alias)
	);
	CREATE INDEX IF NOT EXISTS glossary_aliases_term_idx ON glossary_aliases (term_id);
	CREATE INDEX IF NOT EXISTS glossary_aliases_book_idx ON glossary_aliases (book_id);

	-- B1 editorial-time axis, ORTHOGONAL to the spoiler window (since_ch/until_ch is the
	-- STORY timeline; this is the EDITORIAL timeline). Append-only journal of approved-dst
	-- changes so a stale first-translation-wins rendering is DETECTABLE (DelTA PNR generates
	-- stale/drift; this is the substrate that catches it). Not injected → not in any hash;
	-- datetime('now') here is fine (telemetry, like request_log), never feeds memoryVersion.
	CREATE TABLE IF NOT EXISTS glossary_revisions (
		id           INTEGER PRIMARY KEY,
		book_id      TEXT NOT NULL,
		src          TEXT NOT NULL,
		sense        TEXT NOT NULL DEFAULT '',
		old_dst      TEXT NOT NULL DEFAULT '',
		new_dst      TEXT NOT NULL,
		editorial_ts TEXT NOT NULL DEFAULT (datetime('now')),
		reason       TEXT NOT NULL DEFAULT ''
	);
	CREATE INDEX IF NOT EXISTS glossary_revisions_book_idx ON glossary_revisions (book_id, src);

	-- Per-chunk retrieval-state (registry gate #4 / A1/F2/G2/E1): observability of memory
	-- degradation FROM DAY ONE — the mechanism that converts silent degradation into loud.
	-- Recomputed deterministically each run (the matcher is $0), so it self-heals on resume;
	-- NOT wire-load-bearing (it records what the injection did, it does not change the request).
	CREATE TABLE IF NOT EXISTS retrieval_state (
		book_id             TEXT NOT NULL,
		chapter             INTEGER NOT NULL,
		chunk_idx           INTEGER NOT NULL,
		snapshot_id         TEXT NOT NULL,
		n_exact_hits        INTEGER NOT NULL DEFAULT 0,  -- entries matched by an exact key/alias in this chunk
		n_sticky            INTEGER NOT NULL DEFAULT 0,  -- entries carried by scene-inertia (A5), not re-matched here
		n_ambiguous_flagged INTEGER NOT NULL DEFAULT 0,  -- injected AMBIGUOUS (auto/draft) → "unverified" + forced post-check (A2)
		n_spoiler_blocked   INTEGER NOT NULL DEFAULT 0,  -- matched but hard-rejected by the since_ch/until_ch window (C1)
		n_evicted           INTEGER NOT NULL DEFAULT 0,  -- dropped by the token budget (F2 — logged, never silent)
		embedding_tier_used INTEGER NOT NULL DEFAULT 0,  -- always 0 in v1 (no embeddings on the hot path, Р3)
		n_postcheck_miss    INTEGER NOT NULL DEFAULT 0,  -- E1 flags: a confirmed src matched but no accepted dst form is in the output
		postcheck_detail    TEXT NOT NULL DEFAULT '',    -- JSON [{src,dst}] of the misses, for the report / human
		injected_ids        TEXT NOT NULL DEFAULT '',    -- JSON of exact-matched entity keys (audit + sticky reconstruction)
		updated_at          TEXT NOT NULL DEFAULT (datetime('now')),
		PRIMARY KEY (book_id, chapter, chunk_idx)
	);
	`,
	// v6: surface single-hop escalation on the chunk_status RESOLVE (D15.3 status/redrive).
	// The fallback fact lived only on the checkpoint (escalation flag, v3); folding it into
	// chunk_status lets `tmctl status` project escalation-on-flagged-resume WITHOUT re-reading
	// checkpoints (the deferred Milestone-2.5 finding #6 — the escalation column the status
	// read-model needs). Derivable from checkpoints, so a lost/rebuilt row self-heals on the
	// next resolve. escalation_model is the fallback that answered (empty when not escalated).
	`
	ALTER TABLE chunk_status ADD COLUMN escalated INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE chunk_status ADD COLUMN escalation_model TEXT NOT NULL DEFAULT '';
	`,
	// v7: the cheap deterministic style flaggers (dialogue-dash, yofikator, translit-interjection
	// blocklist, 万/億 magnitude gate — cheapgates.go) ride the same per-chunk observability row as
	// the glossary post-check. n_style_flags is the total hit count; style_detail is the JSON
	// breakdown for the report. Recomputed deterministically each run (self-heals on resume), NOT
	// wire-load-bearing (observability, never a disposition).
	`
	ALTER TABLE retrieval_state ADD COLUMN n_style_flags INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE retrieval_state ADD COLUMN style_detail  TEXT NOT NULL DEFAULT '';
	`,
	// v8 (D39 layer 4): the disposition-gated suppressor's LOUD record. n_trust_gated_suppress counts
	// the times a longer but LOWER-trust glossary key (draft/ambiguous) was REFUSED from suppressing a
	// nested HIGHER-trust key (approved/confirmed) — the term-drift code root, converted from a silent
	// drop into a visible signal (research/13 §7). A nonzero count means the seed has a draft term
	// nesting over an approved one to reconcile. trust_gate_detail is the JSON of the refused pairs.
	// Recomputed deterministically each run (self-heals on resume), NOT wire-load-bearing.
	`
	ALTER TABLE retrieval_state ADD COLUMN n_trust_gated_suppress INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE retrieval_state ADD COLUMN trust_gate_detail      TEXT NOT NULL DEFAULT '';
	`,
	// v9 (WS4): the banknote channel's per-chunk telemetry (integration point 10) — the accepted line
	// count and the parse-fail / truncation flags of the translator draft's ⟦TM-BANK-v1⟧ block, riding
	// the same per-chunk observability row as the glossary post-check and the cheap style flaggers.
	// Recomputed deterministically each run (self-heals on resume), NOT wire-load-bearing (observability,
	// never a disposition). Zero for every channel-off / non-banknote chunk (the common case).
	`
	ALTER TABLE retrieval_state ADD COLUMN n_banknote_lines    INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE retrieval_state ADD COLUMN banknote_parse_fail INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE retrieval_state ADD COLUMN banknote_truncated  INTEGER NOT NULL DEFAULT 0;
	`,
	// v10 (pack-13 point-9, research/21 §1.10): the CostSource marker on the telemetry row. When a paid
	// 2xx settles at the RESERVATION ESTIMATE (a billed decode failure, or a paid-but-zero-usage 2xx) the
	// row's cost_usd is an estimate, not provider-reported. Without a flag those rows are indistinguishable
	// from real zero-token calls and skew token-based COGS analytics. estimated=1 makes the estimated share
	// queryable; est_tokens is a DISPLAY-ONLY fertility (est_out) estimate of the output tokens (never the
	// char/4 CJK-mine). Additive columns, default 0 → every prior row and the WIRE are unchanged (the pack
	// invariant); an estimated row keeps its real settled cost_usd — the flag/estimate are display-only.
	`
	ALTER TABLE request_log ADD COLUMN estimated  INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE request_log ADD COLUMN est_tokens INTEGER NOT NULL DEFAULT 0;
	`,
	// v11 (mini-run findings Д1 + Д4, 25.07): two facts the mini-run proved are produced and then lost.
	//
	//   • chunk_status.first_flag_reason — the disposition reason of the FIRST attempt, kept when a later
	//     attempt (a regenerate or the single-hop escalation) recovered the chunk. Without it a recovered
	//     row is written `ok` with an empty flag_reason and the primary failure vanishes from every
	//     durable surface: the mini-run's echo (1 of 20 drafts, cjk_artifact, escalation-recovered) was
	//     reported as echo_draft=0.0%. Since the echo rate is the only numeric watchman of the DeepSeek
	//     echo mine (D18/D19), losing it hides the class it exists to watch. Re-derived every run from the
	//     stored checkpoints (self-heals on resume); observability only, never a verdict, never wire.
	//
	//   • retrieval_state.banknote_detail — the PARSED ⟦TM-BANK-v1⟧ entries (src/dst/type) of the chunk,
	//     the WHAT channel D39.36 found is parsed and thrown into `_`. It mirrors the three *_detail
	//     JSON columns already on this row (postcheck / style / trust-gate): re-derived deterministically
	//     each run from the raw checkpoint, self-healing, never wire-load-bearing. The bank-mining stop
	//     reads it to attach the model's PROPOSED dst to the owner's signature map — proposals stay
	//     status:auto, so delivering the WHAT grants no trust (the owner still signs).
	//
	// Both are additive with defaults ⇒ every prior row and the WIRE are unchanged.
	`
	ALTER TABLE chunk_status    ADD COLUMN first_flag_reason TEXT NOT NULL DEFAULT '';
	ALTER TABLE retrieval_state ADD COLUMN banknote_detail   TEXT NOT NULL DEFAULT '';
	`,
	// v12 (pack-20 / D39.42 п.4): the $0 observability channel of the UNVERIFIED wire.
	//
	// In the auto mode the bank legitimately carries renderings nobody signed, and the owner's decision is
	// that they reach the model. That makes a question answerable that never was: when an unsigned
	// rendering is actually PUT IN FRONT of the model, does the model go along with it? (⚠ The pair was
	// born measuring a wire that MARKED such a row ⟨проверить⟩ and invited the model to reject it; row 134
	// took the mark off, so it now measures agreement with a row shown as law — still the only channel
	// that says whether an unsigned rendering survives into the text, and no longer a measure of granted
	// judgement. The columns and their arithmetic are unchanged.)
	//
	//   • n_unverified_shown    — unsigned rows whose src fired in THIS chunk (the denominator);
	//   • n_unverified_followed — of those, the ones whose rendering appears in the output.
	//
	// ⚠ The question above is narrower than the pair answers, and the bullets are the exact rule: a STICKY
	// carry is put in front of the model and is NOT counted, because its src is in the previous chunk and
	// the output is not expected to answer for it (membank.PostcheckResult). "Fired here", not "shown".
	//
	// The pair is deliberately measured ON THE WIRE THAT SHOWED THE ROW: before pack-20 the ambiguous
	// counter was collected on the editor wire, whose block was CONFIRMED-only — it counted deviations
	// from something the editor was never shown. Neither number gates anything: an unverified row is a
	// candidate the model is entitled to reject (external-review major #1), so this is a measurement of
	// the channel, never a judgement of the output. Additive with defaults ⇒ prior rows and the WIRE are
	// unchanged.
	`
	ALTER TABLE retrieval_state ADD COLUMN n_unverified_shown    INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE retrieval_state ADD COLUMN n_unverified_followed INTEGER NOT NULL DEFAULT 0;
	`,
	// v13 (pack-19 / D39.55): the two D21 record types the bank was missing — the per-character VOICE
	// profile and the ordered ADDRESS pair. They are TABLES rather than glossary columns for one hard
	// reason: a character's term row and its voice profile share (src, sense, since_ch, until_ch), which
	// is the glossary UNIQUE key, so the profile could not coexist with the term it describes without
	// rebuilding that constraint — forbidden by the append-only discipline above.
	//
	// They are nevertheless BANK CONTENT, not a store beside the bank: their rows are replaced in the SAME
	// transaction as the glossary and fold into the SAME memory_version (conditionally — see
	// membank.ComputeVersionScopedIn). Giving them a snapshot key of their own would classify a signature
	// as moveOther and cost a whole edit wave instead of the touched units (pipeline/repin.go).
	//
	// address_pairs is the MATERIALIZATION of the ты/вы transition journal (D7-Amendment-1, D21 п.2), not a
	// second source of truth: a row IS a journal entry (register valid from since_ch to until_ch) and the
	// "current register at chapter N" is the projection membank computes, never a stored state. A ты↔вы
	// switch is a second row with a non-overlapping window, exactly like a spoiler handoff.
	//
	// Both reference characters by the STABLE key (src, sense) rather than glossary.id: that id is a fresh
	// autoincrement on every replace (see v5), so an id link would silently re-point after any re-seed.
	`
	CREATE TABLE IF NOT EXISTS voice_profiles (
		id              INTEGER PRIMARY KEY,
		book_id         TEXT NOT NULL,
		src             TEXT NOT NULL,              -- character identity, half 1 (the glossary term's src)
		sense           TEXT NOT NULL DEFAULT '',   -- character identity, half 2
		register        TEXT NOT NULL DEFAULT '',   -- one line: register + tone
		self_ref        TEXT NOT NULL DEFAULT '',   -- the character's own self-designation (D7 first_person)
		address_default TEXT NOT NULL DEFAULT '',   -- informal|formal|'' — the T/V default outside the pair registry
		lexicon_markers TEXT NOT NULL DEFAULT '',   -- JSON []string: characteristic words
		ng_lexicon      TEXT NOT NULL DEFAULT '',   -- JSON []string: words this character never says
		exemplars       TEXT NOT NULL DEFAULT '',   -- JSON []string: 3-5 approved target replies
		brightness      TEXT NOT NULL DEFAULT '',   -- injection priority hint (D21 п.1в: hypothesis, not a rule)
		since_ch        INTEGER NOT NULL DEFAULT 0, -- manner-change window, same axis as the glossary spoiler window
		until_ch        INTEGER NOT NULL DEFAULT 0,
		UNIQUE (book_id, src, sense, since_ch, until_ch)
	);
	CREATE INDEX IF NOT EXISTS voice_profiles_book_idx ON voice_profiles (book_id);

	CREATE TABLE IF NOT EXISTS address_pairs (
		id              INTEGER PRIMARY KEY,
		book_id         TEXT NOT NULL,
		speaker_src     TEXT NOT NULL,
		speaker_sense   TEXT NOT NULL DEFAULT '',
		addressee_src   TEXT NOT NULL,
		addressee_sense TEXT NOT NULL DEFAULT '',
		register        TEXT NOT NULL,              -- informal|formal — ABSTRACT; the surfaces are target data
		form            TEXT NOT NULL DEFAULT '',   -- the address form («молодой господин», a title)
		closeness       TEXT NOT NULL DEFAULT '',   -- one word of hierarchy/closeness (why the register is what it is)
		since_ch        INTEGER NOT NULL DEFAULT 0, -- a ты<->вы switch is a STORY event: a second row, new window
		until_ch        INTEGER NOT NULL DEFAULT 0,
		UNIQUE (book_id, speaker_src, speaker_sense, addressee_src, addressee_sense, since_ch, until_ch)
	);
	CREATE INDEX IF NOT EXISTS address_pairs_book_idx ON address_pairs (book_id);
	`,
	// v14 (pack-19): the per-chunk observability of the two new $0 flaggers, riding the same
	// recomputed-every-run row as the glossary post-check and the cheap style gates.
	//
	//   • n_voice_flags / voice_detail   — the deterministic T/V + voice-marker flagger (gates.voice);
	//   • n_spoiler_leaks / spoiler_leak_detail — a rendering the spoiler window REJECTED for this
	//     chapter that appeared in the output anyway (the reveal half of D21 п.3).
	//
	// Neither is ever a disposition, and both are re-derived deterministically each run (self-healing on
	// resume), so they are observability exactly like their siblings. Additive with defaults ⇒ every prior
	// row and the WIRE are unchanged.
	`
	ALTER TABLE retrieval_state ADD COLUMN n_voice_flags       INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE retrieval_state ADD COLUMN voice_detail        TEXT NOT NULL DEFAULT '';
	ALTER TABLE retrieval_state ADD COLUMN n_spoiler_leaks     INTEGER NOT NULL DEFAULT 0;
	ALTER TABLE retrieval_state ADD COLUMN spoiler_leak_detail TEXT NOT NULL DEFAULT '';
	`,
	// v15 (row 103): the run-event OUTBOX. A line of `events.jsonl` is a projection of a row committed
	// here — that is the ratified form of the seam (D39.106 §2), and it is what lets the money event ride
	// the very transaction that settled the money. The number is assigned inside that transaction, so a
	// rollback leaves no hole in a stream whose reader treats a hole as fatal; the exact bytes are kept,
	// so a retry after a failed journal write re-projects the identical line rather than a fresher one.
	//
	// It is a BUFFER, not an archive: outbox.go drops every other run's rows at open, so a book that has
	// been resumed a hundred times carries one run's worth of them.
	`
	CREATE TABLE IF NOT EXISTS events_outbox (
		engine_run_id TEXT    NOT NULL,   -- the PROCESS's id (trace id); seq restarts at 1 per process
		seq           INTEGER NOT NULL,
		-- Non-empty for an event that must be announced ONCE for the life of the book however many
		-- processes it takes (today: one per unit per wave). Those rows OUTLIVE their run — they are the
		-- emission ledger, not a buffer — which is what makes a crash between a unit's disposition and
		-- its announcement recoverable instead of a count lost for good.
		once_key      TEXT    NOT NULL DEFAULT '',
		line          TEXT    NOT NULL,   -- the NDJSON line, verbatim, without its newline
		created_at    TEXT    NOT NULL DEFAULT (datetime('now')),
		PRIMARY KEY (engine_run_id, seq)
	);
	CREATE UNIQUE INDEX IF NOT EXISTS events_outbox_once ON events_outbox (once_key) WHERE once_key <> '';
	`,

	// v16 — the bank-stop memory (the D39.144 flag model): every normalized surface a --verify-bank stop
	// has already PRESENTED to the owner — each presented cluster's src and aliases. The stop fires only
	// on a cluster whose WHOLE surface set is outside this set (pipeline/mining.go hasUnpresentedCluster —
	// one shared member means "seen"; keyed on the representative alone, a cluster would pay one extra
	// stop every time it grows or re-ranks a member), so
	// the flag pays with one stop per novelty instead of re-stopping on everything still undecided. Book
	// state, like the reject list, and under the same growth law: memory ⊆ the book's mined vocabulary,
	// and it is never trimmed — a trim is a re-stop through the back door.
	`
	CREATE TABLE IF NOT EXISTS bank_stop_presented (
		book_id TEXT NOT NULL,
		surface TEXT NOT NULL,   -- text.NormalizeSourceKey form, the key the emission compares surfaces in
		PRIMARY KEY (book_id, surface)
	);
	`,
	// v17 (row 417): WHAT EACH WAVE WAS SHOWN — the axis retrieval_state does not have.
	//
	// The two waves select over DIFFERENT banks: the draft over the BASE bank with mined rows excluded, the
	// editor over the ENRICHED one. retrieval_state is keyed (book_id, chapter, chunk_idx), one row per
	// chunk, so their traces land on the same row and the editor's half is simply lost — the unit merge
	// writes the editor's post-check onto the LEADER's DRAFT row and leaves injected_ids, n_exact_hits and
	// n_evicted carrying the draft's numbers (waverun.go mergeUnitRetrievalState). The editor wave's
	// EVICTIONS were recorded nowhere at all. So "did the bank reach the editor's prompt" — the question
	// the paid run of 11.09 could not answer about four terms whose shipped rendering was not the bank's —
	// was unanswerable from the schema, not from a missing counter.
	//
	// This is a SEPARATE table rather than a wider key on the old one, and the reason is not migration risk
	// (a neighbouring table bumps the version just the same): changing a PRIMARY KEY REWRITES the rows that
	// exist, and every stored run would have to be re-derived to mean what the new key says. An additive
	// table leaves them alone and answers the new question for every run from here on.
	//
	// Recomputed deterministically each run like its neighbour ($0 — the matcher is free), so it self-heals
	// on resume and is never wire-load-bearing: it records what the injection DID, it does not change the
	// request.
	`
	CREATE TABLE IF NOT EXISTS wave_selection (
		book_id             TEXT    NOT NULL,
		chapter             INTEGER NOT NULL,
		chunk_idx           INTEGER NOT NULL, -- the draft chunk, or the edit unit's LEADER chunk
		wave                TEXT    NOT NULL, -- draft | edit — the axis the old key is missing
		snapshot_id         TEXT    NOT NULL,
		n_exact_hits        INTEGER NOT NULL DEFAULT 0,
		n_sticky            INTEGER NOT NULL DEFAULT 0,
		n_ambiguous_flagged INTEGER NOT NULL DEFAULT 0,
		n_spoiler_blocked   INTEGER NOT NULL DEFAULT 0,
		n_evicted           INTEGER NOT NULL DEFAULT 0,
		injected_srcs       TEXT    NOT NULL DEFAULT '', -- JSON array of the SOURCE surfaces this wave showed
		evicted_srcs        TEXT    NOT NULL DEFAULT '', -- JSON array of the surfaces the token budget dropped
		updated_at          TEXT    NOT NULL DEFAULT (datetime('now')),
		PRIMARY KEY (book_id, chapter, chunk_idx, wave)
	);
	CREATE INDEX IF NOT EXISTS wave_selection_book_idx ON wave_selection (book_id, wave);
	`,
	// v18 (row 422): WHAT ATE THE BUDGET. `reasoning_tokens` means «thinking billed ON TOP of the
	// completion», which is 0 by definition on a subset-billing provider — so on DeepSeek, the only
	// provider the shipping configs call, it has carried 0 for the entire life of the project, and a
	// reader asking «did thinking eat max_tokens» got a zero that looks measured and is not.
	//
	// This column answers the other question: how much of the completion WE ALREADY PAID FOR was
	// thinking rather than the answer. It is the provider's own number (DeepSeek reports it in
	// completion_tokens_details on every reply), it prices nothing, and it is the evidence behind the
	// cold run of 11.09, where three calls bought 8496/8496/16000 completion tokens and returned zero
	// characters of text.
	//
	// ⚠ NULLABLE ON PURPOSE, against the NOT NULL DEFAULT 0 of every column around it. A subset
	// provider that reports the field and a call that genuinely did not think both produce 0, and a
	// provider that reports nothing must not be spelled the same way — that is the defect this
	// migration exists to remove, and a DEFAULT 0 would re-introduce it on the first row. NULL = the
	// question was not answered; every row written before this column is NULL for exactly that reason.
	`
	ALTER TABLE request_log ADD COLUMN reasoning_in_completion INTEGER;
	`,
}

// DESIGN NOTE (D21.10 → BUILT by pack-19 / D39.55; kept as the record of what each type is FOR).
//
// research/15 (D21) ratified three memory MECHANISMS extending the bank. Two are now schema (v13); the
// third turned out to need none.
//
//   • voice_profile (D21.1) — BUILT as `voice_profiles` (v13), keyed by the STABLE (src, sense) rather
//     than the reserved (book_id, term_id): glossary.id is a fresh autoincrement per replace. Still
//     FROZEN per-job (self-populating exemplars = mid-run memory append = book re-pay under D8/D15.1);
//     the condition for lifting it — D15.2 landing — has NOT happened (verdictSnapshotID/guard_hash do
//     not exist in this engine, only in comments).
//
//   • address_pair (D21.2) — BUILT as `address_pairs` (v13). The reserved shape guessed a derived
//     projection over a separate `speech_transitions` journal; the built one collapses the two, because a
//     windowed row IS a journal entry and the "register at chapter N" projection is computed, never
//     stored. So there is still exactly one source of truth, with one table instead of two.
//     `glossary.speech` (the static v5 note) is superseded and DEPRECATED — see its column comment.
//
//   • reveal_ch + pre/post-reveal aliases (D21.3) — NOT built, and deliberately: the pre/post rendering
//     pair is already expressible as two glossary rows with NON-OVERLAPPING windows (the UNIQUE key
//     admits them, membank.windowsOverlap permits them, spoilerBlocked picks exactly one), and aliases
//     inherit their row's window, so a phase tag would be a third way to say what two already say. What
//     was genuinely missing is the LEAK CHECK — a rejected rendering appearing in the output anyway —
//     which is membank.SpoilerLeaks over the selection's Rejected set, no schema at all. Scope amended
//     by D39.55 (form, not intent).

// SchemaHead is the schema version this binary migrates a project database to. It is the version every
// read-only open demands (OpenReadOnly) and the version `tmctl migrate` brings a project to.
func SchemaHead() int { return len(migrations) }

// SchemaMismatchError is a project database whose recorded schema version is not this binary's. It is a
// TYPE rather than a message because it is the one open failure a caller can repair by itself, and it
// could not act on the difference while every failure arrived as the same text and the same exit 1.
//
// The engine's read-only opens never migrate — only a write open does — so upgrading the binary leaves
// every existing book unreadable until something writes to it. The platform calls `tmctl status --json`
// before each spawn and for the money, that call refuses the older schema, and the write command that
// would have migrated the project never comes: the deploy deadlock of backlog row 174.
//
// The numbers therefore travel BOTH in the type (for a Go caller) and in the message (for the shell),
// where they carry the stable token
//
//	schema_mismatch found=<N> expected=<M>
//
// Found < Expected is the direction `tmctl migrate` repairs — "caught it, migrated, retried" instead of
// stopping the world. Found > Expected is a binary OLDER than the project, which no command here can fix
// (the operator upgrades tmctl), so a caller that blindly re-ran the migration would loop forever.
type SchemaMismatchError struct {
	Path     string
	Found    int
	Expected int
}

func (e *SchemaMismatchError) Error() string {
	repair := "the database is NEWER than this tmctl — update the binary (an old build must neither read nor write a newer schema)"
	if e.Stale() {
		repair = "the database is older than this tmctl — run `tmctl migrate --config <book.yaml>` (read-only opens never migrate)"
	}
	return fmt.Sprintf("store: schema_mismatch found=%d expected=%d path=%s — %s", e.Found, e.Expected, e.Path, repair)
}

// Stale reports whether the database is BEHIND the binary — the one direction a migration repairs.
func (e *SchemaMismatchError) Stale() bool { return e.Found < e.Expected }

// Migration is the schema transition a write open performed: From is the version the project database
// was recorded at, To the version it is at now. From == To is a database that was already at head.
type Migration struct {
	From int
	To   int
}

// Migrate applies a project database's pending migrations and reports the transition — the WRITE open
// (exclusive flock · migration chain · stale-reservation recovery) with no run behind it, which is all
// `tmctl migrate` is (backlog row 174). It sends nothing, spends nothing and holds the flock only for
// as long as the migration takes.
//
// A database already at head is a no-op; one that does not exist is CREATED at head, exactly as the
// first touch of a project by any other write command creates it. A database a NEWER binary wrote is
// refused (SchemaMismatchError) before anything of this process reaches the file.
//
// beforeApply, when non-nil, runs INSIDE the project's exclusive lock, after the pending set is known
// and before the first step is applied — and only when there IS a step to apply. That is where a caller
// takes its restore point, and the placement is the whole point of the parameter: a backup taken before
// the lock is a full copy written for a migration that a live run may then refuse (litter on every
// retry of an exit-12), and one taken after the steps is not a restore point at all. An error from it
// aborts the migration with nothing applied.
func Migrate(path string, beforeApply func(Migration) error) (Migration, error) {
	s, err := open(path, beforeApply)
	if err != nil {
		return Migration{}, err
	}
	applied := s.applied
	if err := s.Close(); err != nil {
		return applied, fmt.Errorf("store: close %s after migrating: %w", path, err)
	}
	return applied, nil
}

// schemaVersion reads the recorded schema version through db.
//
// A database that exists but has no schema_version TABLE — a fresh file, or a process killed between
// creating one and committing the first migration — is version 0, which is what it is: nothing has been
// applied. That is asked as a separate question rather than read off a failed SELECT, so a file that is
// not a database at all still fails loudly instead of reading as an empty project.
func schemaVersion(ctx context.Context, db *sql.DB) (int, error) {
	var recorded int
	if err := db.QueryRowContext(ctx,
		`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'schema_version'`).Scan(&recorded); err != nil {
		return 0, err
	}
	if recorded == 0 {
		return 0, nil
	}
	var current int
	if err := db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_version`).Scan(&current); err != nil {
		return 0, err
	}
	return current, nil
}

// migrate runs all pending migrations on the write pool, one transaction per
// step, recording each in schema_version.
//
// Every phase gets its own opTimeout budget, and the caller's seam gets NONE. That is not tidiness: the
// seam exists to take a restore point, whose cost scales with the book, and while it shared one deadline
// with the steps a large project made the migration fail on the copy it had just paid for (see open).
func (s *Store) migrate(path string, beforeApply func(Migration) error) error {
	current, err := s.schemaBaseline()
	if err != nil {
		return err
	}
	// A project a NEWER binary wrote is REFUSED rather than opened. The loop below would silently do
	// nothing with it — there is no step left to apply — and the caller would then read and, worse,
	// WRITE a schema through code that predates it. OpenReadOnly has always refused that direction
	// ("reading a newer schema with old code is unsafe"); the write path used to accept it, which is
	// the more dangerous half of the same fact. It is refused here, BEFORE recoverReservations, so a
	// refused open leaves the file exactly as it found it.
	if current > len(migrations) {
		return &SchemaMismatchError{Path: path, Found: current, Expected: len(migrations)}
	}
	// The caller's seam (Migrate): the lock is held, the pending set is known, nothing has been applied,
	// and no deadline of this package is running — the work is the caller's and so is its bounding.
	if beforeApply != nil && current < len(migrations) {
		if err := beforeApply(Migration{From: current, To: len(migrations)}); err != nil {
			return err
		}
	}
	for v := current; v < len(migrations); v++ {
		if err := s.applyStep(v); err != nil {
			return err
		}
	}
	s.applied = Migration{From: current, To: len(migrations)}
	return nil
}

// schemaBaseline makes sure the version table exists and reports what is recorded in it.
func (s *Store) schemaBaseline() (int, error) {
	ctx, cancel := opContext()
	defer cancel()
	if _, err := s.w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY)`); err != nil {
		return 0, fmt.Errorf("store: schema_version: %w", err)
	}
	current, err := schemaVersion(ctx, s.w)
	if err != nil {
		return 0, fmt.Errorf("store: read version: %w", err)
	}
	return current, nil
}

// applyStep applies migrations[v] and records v+1 in ONE transaction (DDL is transactional in SQLite),
// so a process that dies mid-step leaves the database at its previous version rather than half-applied.
func (s *Store) applyStep(v int) error {
	ctx, cancel := opContext()
	defer cancel()
	tx, err := s.w.BeginTx(ctx, nil)
	if err != nil {
		return fmt.Errorf("store: begin migration %d: %w", v+1, err)
	}
	if _, err := tx.ExecContext(ctx, migrations[v]); err != nil {
		_ = tx.Rollback()
		return fmt.Errorf("store: apply migration %d: %w", v+1, err)
	}
	if _, err := tx.ExecContext(ctx, `INSERT INTO schema_version (version) VALUES (?)`, v+1); err != nil {
		_ = tx.Rollback()
		return fmt.Errorf("store: record migration %d: %w", v+1, err)
	}
	if err := tx.Commit(); err != nil {
		return fmt.Errorf("store: commit migration %d: %w", v+1, err)
	}
	return nil
}