diff --git a/backend/README.md b/backend/README.md index 173aeaf..c4caa7f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -6,7 +6,7 @@ | Пакет | Что делает | Ключевые файлы | |---|---|---| -| `cmd/tmctl` | CLI: `translate --config book.yaml` / `report` (status/redrive — в работе, D15.3) | `main.go` | +| `cmd/tmctl` | CLI: `translate` / `report` / `status` (read-only проекция N/M+паспорта глав+деньги, `--json`) / `redrive` (переатака флагнутых: `--chapter/--chunk/--reason/--dry-run`, D15.3) | `main.go`, `internal/pipeline/status.go` | | `internal/llm` | OpenAI-совместимый транспорт + retry/backoff, **capability-слой** (budget_field/temperature/reasoning per-модель), failover (написан, НЕ подключён — D4: только local-роли), нативный Anthropic = DEPRECATED-референс под будущий Gemini | `httpllm.go`, `capability.go`, `failover.go` | | `internal/ledger` | Цены по usage (вкл. reasoning/cache-поля), `PriceForResponse` по фактической модели | `pricing.go` | | `internal/store` | SQLite (modernc, CGO-free), идемпотентные миграции v1–v5, reserve/settle+checkpoint, chunk_status, глоссарий, ruby, retrieval_state, request_log | `ledger.go`, `migrate.go`, `glossary.go` | @@ -16,7 +16,7 @@ ## Инварианты — ЛОМАТЬ НЕЛЬЗЯ (каждый закреплён тестами) -1. **Деньги:** reserve → call → **settle+checkpoint одной транзакцией**; `committed == SUM(checkpoints)` переживает kill -9 (`kill9_test.go`); цена — по фактически ответившей модели; потолки книга/день считают committed+reserved. +1. **Деньги:** reserve → call → **settle+checkpoint одной транзакцией**; `committed == SUM(checkpoints)` переживает kill -9 (`kill9_test.go`); цена — по фактически ответившей модели; потолки книга/день считают committed+reserved. **Одно исключение — явный `tmctl redrive`** (D15.3): удаляет чекпоинты сброшенных стадий, НЕ возвращая committed (реально потраченные деньги остаются) → после redrive `committed ≥ SUM(checkpoints)` — безопасное направление (потолок не занижает), задокументировано в `ResetChunkStages`. 2. **Snapshot-дисциплина:** всё, что влияет на wire-байты ИЛИ на вердикты (capability, память content-hash, context-assembly, версии classify/coverage/chunker/maxtok, эскалационные оверрайды), свёрнуто в `snapshotID`; правка = громкий `--resnapshot`. ⚠ Любой resnapshot = переоплата книги (D15) — content-addressed reuse спроектировать до онгоинга. 3. **Эхо-мина DeepSeek:** thinking у deepseek НИКОГДА не отключать — `reasoning:"off"` там осознанный no-op; `echoes_when_thinking_off` + рекурсивный скан extra_body валят конфиг fail-fast. У grok наоборот: off = ЯВНЫЙ `reasoning_effort:"none"`. 4. **Порядок classify:** refusal-blacklist / CJK-echo — ДО length/empty (усечённый отказ не превращается в платный ретрай); retry-flagged только {length, empty} на той же модели; loop-чек ПЕРЕД удвоением max_tokens. @@ -24,6 +24,12 @@ 6. **Детерминизм:** рендер — чистая функция snapshot; сортировки перед записью; никакого map-order в выводе; length-prefixed хэши. Сорс-чанкер lossless (fuzz-тесты). 7. **Нейтральность адаптера:** `internal/llm` не знает про перевод (src/target/coverage) — контент-вердикты живут в раннере (disposition-слой, post-settle). +## Подготовка ja-книги (чек-лист сида) + +- **Kana-написания имён = matchable.** Ruby-чтения (`鈴木すずき`) захватываются на инжесте и авто-подцепляются как alias соответствующей ручной записи (D16.4, `attachRubyAliasesToManual`), поэтому кана-форма имени, у которого есть кандзи+фуригана, матчится автоматически. **Сверх этого — руками:** имя, которое встречается ТОЛЬКО каной (без кандзи-написания с ruby), не даст авто-alias → внеси кана-написание в сид как `aliases:` вручную, иначе кана-упоминание молча не заматчится («тихо пусто»). +- **Двойные чтения (強敵→とも) — под флагом.** Авто-alias ставится для формы «all-Han база + all-kana чтение»; семантическое двойное чтение неотличимо оффлайн (нужен yomi-словарь, B6). Такой alias фаерится AMBIGUOUS (короткий/коллизионный) и проходит post-check, но длинное двойное чтение может дать CONFIRMED-ложь — при подозрении убери его из сида. Полная дизамбигуация — Фаза 2 (B6/Палладий-Поливанов). +- **Kana-precision ≥4 не замерена** (minKeyLenPhonetic=3 — консервативный дефолт): кана-ключи НЕ проверяются на границу слова (в отличие от латиницы/кириллицы — D16.3), т.к. у каны нет сегментации (как у Han) — иначе ломается частый кейс «имя+частица» (すずきは). ≥4-кана-компаунд-precision — задача полигона (расширение E1) + токенизатор B6. + ## Как гонять ```bash diff --git a/backend/cmd/tmctl/main.go b/backend/cmd/tmctl/main.go index 6977b60..6591096 100644 --- a/backend/cmd/tmctl/main.go +++ b/backend/cmd/tmctl/main.go @@ -4,6 +4,7 @@ package main import ( "bufio" "context" + "encoding/json" "errors" "flag" "fmt" @@ -38,7 +39,7 @@ func main() { func run() error { if len(os.Args) < 2 { - return fmt.Errorf("usage: tmctl --config book.yaml") + return fmt.Errorf("usage: tmctl --config book.yaml") } cmd, args := os.Args[1], os.Args[2:] @@ -49,6 +50,11 @@ func run() error { fs := flag.NewFlagSet(cmd, flag.ContinueOnError) cfgPath := fs.String("config", "", "path to book.yaml") resnapshot := fs.Bool("resnapshot", false, "re-pin existing jobs to the current config snapshot (пере-перевод оплаченных чанков — явное согласие)") + asJSON := fs.Bool("json", false, "status: emit the projection as JSON (stable disposition/flag_reason enums) for CI/IDE") + chapter := fs.Int("chapter", -1, "redrive: restrict to this chapter (default: any)") + chunk := fs.Int("chunk", -1, "redrive: restrict to this chunk index within the chapter (default: any)") + reason := fs.String("reason", "", "redrive: restrict to this flag_reason (default: any)") + dryRun := fs.Bool("dry-run", false, "redrive: report what would be re-attacked without touching anything") if err := fs.Parse(args); err != nil { return err } @@ -73,8 +79,14 @@ func run() error { return translate(ctx, *cfgPath, *resnapshot) case "report": return report(*cfgPath) + case "status": + return status(ctx, *cfgPath, *asJSON) + case "redrive": + return redrive(ctx, *cfgPath, pipeline.RedriveSelector{ + Chapter: *chapter, ChunkIdx: *chunk, Reason: *reason, DryRun: *dryRun, + }) default: - return fmt.Errorf("unknown command %q (want translate|report)", cmd) + return fmt.Errorf("unknown command %q (want translate|report|status|redrive)", cmd) } } @@ -222,6 +234,126 @@ func report(cfgPath string) error { return nil } +// status prints the READ-ONLY progress projection (D15.3): 0 LLM, 0 replay. --json emits the +// StatusReport with stable disposition/flag_reason enums for CI/IDE; the default is a human +// dashboard (unit counts, per-chapter quality passports, money, secondary ETA). +func status(ctx context.Context, cfgPath string, asJSON bool) error { + r, err := pipeline.NewRunner(cfgPath, obs.NewLogger()) + if err != nil { + return err + } + defer r.Close() + + rep, err := r.Status(ctx) + if err != nil { + return err + } + + if asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(rep) + } + + snap := rep.Snapshot + if snap == "" { + snap = "—" + } else if len(snap) > 12 { + snap = snap[:12] + } + drift := "" + if rep.SnapshotDrift { + drift += " ⚠ SNAPSHOT-DRIFT (несколько снапшотов в chunk_status — конфиг менялся; --resnapshot)" + } + if rep.ConfigDrift { + drift += " ⚠ CONFIG-DRIFT (текущий конфиг рендерит другой снапшот — правка после прогона; translate потребует --resnapshot = переоплата книги)" + } + fmt.Printf("=== СТАТУС: %s (snapshot %s)%s ===\n", rep.BookID, snap, drift) + fmt.Printf("Прогресс: %d/%d чанков (%.1f%%) — done=%d in_progress=%d flagged=%d pending=%d\n", + rep.Done, rep.TotalChunks, rep.PercentDone, rep.Done, rep.InProgress, rep.Flagged, rep.Pending) + fmt.Printf("Эскалаций: %d · post-check промахов (confirmed): %d\n", rep.Escalations, rep.PostcheckMisses) + ceil := "" + if rep.BookCeilingUSD > 0 { + ceil = fmt.Sprintf(" (потолок $%.2f — %.1f%%)", rep.BookCeilingUSD, rep.CeilingPct) + } + fmt.Printf("Деньги: committed=$%.6f reserved=$%.6f%s · прогноз книги ~$%.6f\n", + rep.CommittedUSD, rep.ReservedUSD, ceil, rep.ProjectedBookUSD) + if rep.ETASeconds > 0 { + fmt.Printf("ETA: ~%.0fс (по throughput fresh-вызовов; вторично)\n", rep.ETASeconds) + } + + if len(rep.Chapters) > 0 { + fmt.Printf("\n%-5s %-9s %-10s %-6s %-4s %-16s %10s\n", "ch", "done/all", "verdict", "flag", "esc", "worst_flag", "cost_usd") + for _, p := range rep.Chapters { + fmt.Printf("%-5d %d/%-7d %-10s %-6d %-4d %-16s %10.6f\n", + p.Chapter, p.ChunksDone, p.ChunksTotal, p.Verdict, p.ChunksFlagged, p.Escalations, + dashIfEmpty(p.WorstFlagReason), p.CostUSD) + } + } + // flagged≠failed (jobs.status.failed = infra-only, D12): flagged chunks are a + // "clean run, attention needed" signal, re-drivable with `tmctl redrive`. + if rep.Flagged > 0 { + fmt.Printf("\n%d флагнутых чанк(ов) ждут внимания — переатаковать: tmctl redrive --config %s [--chapter N --chunk M --reason R]\n", + rep.Flagged, cfgPath) + return &pipeline.CompletedWithFlags{Flagged: rep.Flagged, Total: rep.TotalChunks} + } + return nil +} + +// dashIfEmpty renders "" as "—" for a table cell. +func dashIfEmpty(s string) string { + if s == "" { + return "—" + } + return s +} + +// redrive re-attacks the FLAGGED chunks matching the selector (D15.3): it resets their terminal +// flag (chunk_status + checkpoints of the flagged/skipped stages) and re-runs the durable loop +// with a FRESH retry/escalation budget, never touching DispOK work. On --dry-run it only reports +// the plan. Redrive-calls bill normally; money already spent on the discarded attempts stays +// committed (honest). Requires the current config to render the same snapshot (no --resnapshot). +func redrive(ctx context.Context, cfgPath string, sel pipeline.RedriveSelector) error { + r, err := pipeline.NewRunner(cfgPath, obs.NewLogger()) + if err != nil { + return err + } + defer r.Close() + + summary, res, err := r.Redrive(ctx, sel) + if err != nil { + return err + } + + fmt.Printf("=== REDRIVE: %s ===\n", r.Book.BookID) + if len(summary.Targets) == 0 { + fmt.Println("Флагнутых чанков под селектор не найдено — нечего переатаковать.") + return nil + } + fmt.Printf("Целей: %d флагнутых чанк(ов) — сброс терминального флага + свежий retry/escalation-бюджет (DispOK не тронут):\n", len(summary.Targets)) + for _, t := range summary.Targets { + fmt.Printf(" ch%d/chunk%d (%s) → стадии: %s\n", t.Chapter, t.ChunkIdx, t.FlagReason, strings.Join(t.Stages, ", ")) + } + if summary.DryRun { + fmt.Println("[dry-run: ничего не сброшено, вызовов не сделано]") + return nil + } + fmt.Println("[сброшено; перезапуск durable-цикла — DispOK чанки резюмируются за $0, сброшенные переатакуются]") + fmt.Println() + + if res != nil { + fmt.Printf("ИТОГО re-run (этот запуск): $%.6f — чанков %d, флагов %d\n", res.TotalUSD, len(res.Chunks), res.Flagged) + committed, reserved, serr := r.Store.SpentUSD(res.BookID) + if serr == nil { + fmt.Printf("Ledger книги: committed=$%.6f reserved=$%.6f\n", committed, reserved) + } + if res.Flagged > 0 { + return &pipeline.CompletedWithFlags{Flagged: res.Flagged, Total: len(res.Chunks)} + } + } + return nil +} + // loadDotEnv reads KEY=VALUE lines into the environment without overriding // already-set variables. Ключи — из backend/.env (gitignored); секреты никогда // не попадают в конфиги/репо. diff --git a/backend/configs/pipeline-c1.yaml b/backend/configs/pipeline-c1.yaml index cf1622b..e6103a6 100644 --- a/backend/configs/pipeline-c1.yaml +++ b/backend/configs/pipeline-c1.yaml @@ -46,7 +46,10 @@ stages: # models.yaml) — омиссия у grok = low. Прод: только чистый xAI-ключ без data-sharing. model: grok-4.3 prompt: ../prompts/editor.md - prompt_version: v0-draft + # D1-уточнение №3: коммит 6a68dfd сделал editor.md монолингвальным (убрал {{text}}) → + # сменил SHA, но лейбл остался v0-draft (один лейбл на два SHA). Бамп на v1-monolingual; + # осознанно меняет снапшот (боевых прогонов ещё не было). + prompt_version: v1-monolingual temperature: 0.4 reasoning: "off" diff --git a/backend/configs/pipeline-c2.yaml b/backend/configs/pipeline-c2.yaml index 3611684..fe9702d 100644 --- a/backend/configs/pipeline-c2.yaml +++ b/backend/configs/pipeline-c2.yaml @@ -45,7 +45,10 @@ stages: # (Стадия select выше — judge/glm-5, Gemini-слот Фазы 2; её НЕ трогаем.) model: grok-4.3 prompt: ../prompts/editor.md - prompt_version: v0-draft + # D1-уточнение №3: editor.md стал монолингвальным (6a68dfd убрал {{text}}) → SHA сменился, + # лейбл v0-draft остался (один лейбл на два SHA). Бамп на v1-monolingual (тот же editor.md, + # что в C1). Стадию select выше НЕ трогаем. + prompt_version: v1-monolingual temperature: 0.4 reasoning: "off" diff --git a/backend/docs/D15.2-content-addressed-resume-spec.md b/backend/docs/D15.2-content-addressed-resume-spec.md new file mode 100644 index 0000000..80c76ae --- /dev/null +++ b/backend/docs/D15.2-content-addressed-resume-spec.md @@ -0,0 +1,160 @@ +# D15.2 — Спека: content-addressed переиспользование чекпоинтов (дизайн, НЕ код) + +**Статус:** проект спеки бэкенд-сессии, сдан на утверждение оркестратору. Реализация — Ф1.5, +**после** ратификации. Зона: спека написана в `backend/docs/` (зона бэкенда); при утверждении +оркестратор переносит/сворачивает в `docs/architecture/03-implementation-notes.md` новым разделом. + +**Гейтит:** онгоинг-режим (`add-chapters`, сегмент «ИИ-фабрик», Р9) — D15.2. Текущий +all-or-nothing resnapshot **приемлем для статичной приёмочной книги Ф1 (D15.1) — НЕ трогать до +приёмки**; эта спека — для онгоинга. + +--- + +## 1. Проблема (верифицирована D15 / 07 §4.4) + +`snapshotID` вшит в `RequestHash` (`render.go` — поле `snapshotID` в length-prefixed хеше). Значит +**любое** изменение снапшота (флип coverage-гейта, toggle `postcheck_gate`, бамп `classifierVersion`, +append approved-терминов через `memoryVersion`, правка инертной ручки) после `--resnapshot` +пере-оплачивает **всю книгу**, включая **байт-идентичные** запросы (воспроизведено: 2→4 оплаченных +вызова на wire-идентичных телах). Механизм честного reuse существует (`msgsContentHash`, +`render.go`), но заперт условием `cs.SnapshotID == snapID` (`runner.go`, resume fast-path). + +Лгущие комментарии `runner.go`/`coverage.go` про «re-classify from checkpoint for FREE» уже +исправлены в этом пакете (Задача 1a): сегодня это правда ТОЛЬКО внутри неизменного снапшота. +Цель спеки — **сделать это правдой и через смену снапшота**, без потери snapshot-гарантий. + +## 2. Ключевое наблюдение: снапшот смешивает ДВА разных класса входов + +`snapshotID` сегодня сворачивает вместе: + +**(A) Wire-определяющие входы** — меняют БАЙТЫ, отправленные провайдеру, значит меняют сам +ответ, значит требуют нового вызова: +- `brief_hash`, `chunker_version`, `memory_version`, `context_assembly` — **все они уже входят в + `msgsContentHash`**, потому что рендер (метаданные книги в шаблоне, нарезка → `ch.Text`, + инъекция глоссария как ОТДЕЛЬНОЕ сообщение) целиком материализуется в `msgs`. Это + belt-and-suspenders: фактический per-chunk рендер — источник истины. +- `model`, `temperature`, `reasoning`, `jsonOnly`, `maxTokens` — уже ОТДЕЛЬНЫЕ поля `RequestHash` + (не через снапшот). +- `capability` (budget-поле / temp-режим / reasoning-контроль), `model_extra`, + провайдер-оверрайды (local temp/max_tokens/model), эскалационные capability/оверрайды — **НЕ в + `msgs` и НЕ отдельные поля**: меняют тело запроса, но сегодня заходят в ключ только через + снапшот. Это единственная wire-релевантная часть снапшота, не покрытая иначе. +- `estimator_version`, `max_tokens_policy` — влияют на `maxTokens`, а он уже отдельное поле; сами + ВЕРСИИ избыточны (фактический `maxTokens` в ключе). + +**(B) Вердикт-определяющие входы** — меняют, как ответ КЛАССИФИЦИРУЕТСЯ, но НЕ байты запроса: +- `classifier_version` (порядок/пороги intrinsic classify), `coverage` (enabled + пороги + + `coverage_gate_version`), `postcheck_gate` (on/off; сам АЛГОРИТМ post-check — в `memory_version`, + но он влияет на вердикт, не на wire… см. §6 нюанс). + +Сегодня (B) сидит в `RequestHash` → вердикт-only правка промахивается по всем чекпоинтам и +пере-оплачивает книгу. Это и есть дефект. + +## 3. Дизайн: расщепить снапшот на wire-часть и вердикт-часть + +- **`wireSnapshotID`** = хеш ТОЛЬКО (A)-входов, НЕ покрытых `msgs`/прямыми полями: резолвнутая + `capability`, `model_extra`, провайдер-оверрайды (temp/max_tokens/model), эскалационные + capability/extra/оверрайды. (Всё остальное wire-определяющее уже в `msgsContentHash` или в + прямых полях `RequestHash`.) +- **`verdictSnapshotID`** = хеш (B)-входов: `classifier_version`, coverage-конфиг+версия, + `postcheck_gate`-состояние. + +**`RequestHash` (ключ чекпоинта) = f(bookID, chapter, chunkIdx, attempt, stage, role, model, temp, +reasoning, jsonOnly, maxTokens, `wireSnapshotID`, msgs).** — убираем `snapshotID`, добавляем +`wireSnapshotID`. Вердикт-версии из ключа чекпоинта **уходят**. + +Полный `snapshotID` (для `jobs.snapshot_id`, `snapshots`-таблицы, отчётности) остаётся как +`hash(wireSnapshotID, verdictSnapshotID)` — но перестаёт быть частью ключа вызова. + +### Что тогда re-bill-ится, а что бесплатно (операторский контракт) + +| Изменение | Класс | Эффект | +|---|---|---| +| Флип `coverage.enabled`, правка порогов coverage; бамп `classifier_version`; toggle/пороги `postcheck_gate` | (B) вердикт | **$0** — `wireSnapshotID`+`msgs` не меняются → чекпоинт ХИТ → только **re-classify** | +| Append approved-терминов | (A) через `msgs` | Re-bill **только тех чанков, где новый термин ФАЕРИТСЯ** (их инъекция → `msgs` меняются); чанки, где не фаерится — **$0** (идентичная инъекция → идентичный `msgs`) | +| Правка book-brief, меняющая метаданные части глав | (A) через `msgs` | Re-bill только затронутых чанков | +| Правка промпт-шаблона, смена модели/temp/reasoning, правка `capability`, ре-чанкинг, смена estimator/maxtok-политики (меняет фактический `maxTokens`) | (A) wire | Re-bill (честно: тело/ответ реально другие) | + +**Онгоинг-выигрыш (флагман «ИИ-фабрик»):** подтверждение пачки терминов на границе тома больше не +пере-покупает готовые главы — только те чанки, где термин действительно матчится. Это прямо снимает +D15-мину онгоинга. + +## 4. Resume: когда байт-идентичный запрос легально переиспользуется + +Reuse чекпоинта легален ⟺ **полная wire-идентичность** совпадает: `msgsContentHash` И не-msgs +wire-параметры (`wireSnapshotID` + прямые поля model/temp/reasoning/maxTokens). Это ровно то, что +даёт новый `RequestHash` — совпал ключ ⇒ совпал wire ⇒ переиспользуем оплаченный ответ (то же +основание, что у сегодняшнего same-snapshot resume; провайдер недетерминирован при temp>0, но +чекпоинт — валидный ответ на ЭТОТ wire). + +**Вердикт-различие НИКОГДА не блокирует reuse** — доказуемо wire-нейтрально (см. §5). + +## 5. Re-classify по новым порогам (и почему это безопасно, F1) + +Машинерия re-classify **уже есть**: на checkpoint-hit `runAttempt` вызывает `classifyOutput` над +текстом чекпоинта (не доверяет старому вердикту). Единственная правка — чтобы чекпоинт ХИТИЛ при +вердикт-only смене (§3 убирает вердикт-версии из ключа). Тогда: + +- **Fast-path `resumeFromChunkStatus`** (короткое замыкание до `runAttempt`): доверять строке + `chunk_status` только когда совпали `content_hash` И `wireSnapshotID` И `verdictSnapshotID`. При + СМЕНЕ `verdictSnapshotID` — **не** отдавать старый вердикт: проваливаться в `runAttempt`, где + чекпоинт хитит по wire-идентичности → re-classify бесплатно → re-upsert `chunk_status` с новым + вердиктом. Нужна колонка `chunk_status.verdict_snapshot` (в дополнение к `content_hash`). +- **Вердикт-флип на re-classify:** `flagged→ok` (ослабили гейт) → текст чекпоинта отдаётся + бесплатно. `ok→flagged` (ужесточили) → чанк флагается; если это влечёт НОВЫЙ wire-запрос (retry + на attempt+1, или эскалация на другую модель) — он честно оплачивается (это реально новая работа, + не re-pay старой). Ретраи/эскалация уже сидят на своих осях ключа (`attempt`, модель эскалации). + +**Ось безопасности (F1 — никакого тихого расходящегося re-pay):** reuse — precondition на ТОЧНОЕ +совпадение wire-идентичности (`msgsContentHash` — сильнейший гард: точные байты промпта+инъекции; +плюс не-msgs wire-параметры). Вердикт-смена меняет только post-settle классификацию, НИКОГДА не +запрос, поэтому re-classify матчнутого чекпоинта не может отдать расходящийся ПЕРЕВОД — только другой +ВЕРДИКТ на том же переводе (искомый бесплатный re-verdict). Опасный кейс — reuse чекпоинта, чей wire +на новом конфиге был бы ДРУГИМ — исключён требованием точного wire-match. + +## 6. Нюансы и открытые вопросы к оркестратору + +1. **`memory_version` содержит `memoryMatchVersion` (алгоритм матчера).** Смена алгоритма матчера + меняет per-chunk ВЫБОР (какие записи инъектятся) → инъекция → `msgs` → корректно re-bill только + затронутых чанков. Значит `memory_version` из wire-ключа можно убрать (полагаться на `msgs`). + **НО:** post-check-алгоритм тоже версионируется `memoryMatchVersion`, а post-check — ВЕРДИКТ, не + wire. Дилемма: одна версия покрывает и wire-часть матчера, и вердикт-часть post-check. Развязка: + при реализации расщепить `memoryMatchVersion` на `memoryInjectVersion` (wire, покрыт `msgs`) и + `memoryPostcheckVersion` (вердикт, в `verdictSnapshotID`). До расщепления — держать + `memory_version` в wire-части (консервативно: лишний re-bill безопаснее тихого stale-вердикта). +2. **Гранулярность гейта.** Сегодня resnapshot-гейт — per-JOB (`jobs.snapshot_id`, глава×стадия). + Для онгоинга нужен per-CHUNK (`content_hash`) первичный гейт; job-снапшот демотируется до + грубого advisory. Раннер уже делает per-chunk `content_hash`-чек (`runStage`) — спека делает его + ПЕРВИЧНЫМ. Job-снапшот-mismatch перестаёт быть fail-loud стопом; заменяется на per-chunk решение. +3. **Экономика показа (D5 retro-patch).** Append-термин, фаерящийся в старых главах, ЗАКОННО их + re-bill-ит (термин должен примениться) — но оператору показать стоимость (`tmctl status` + projected-дельта или dry-run «сколько чанков затронет этот append»). Это UI-хвост, не F1. +4. **Миграция ключей.** Смена формулы `RequestHash` (убрать `snapshotID`, добавить `wireSnapshotID`) + — это смена ФОРМАТА ключа (как v1→v2 сейчас). Все существующие чекпоинты промахнутся ОДИН раз при + переходе на новую формулу. Приемлемо (одноразово, на границе Ф1.5); зафиксировать бампом + `tm-request-v2`→`v3` и задокументировать как осознанный one-time re-pin. +5. **`AMBIGUOUS-BILLED` / F3.** Ортогонально этой спеке (F3 — техдолг at-most-once); content-addressed + reuse не меняет F3-инвариант. + +## 7. Минимальный план реализации (Ф1.5, после утверждения) + +1. Расщепить `snapshotID()` → `wireSnapshotID()` + `verdictSnapshotID()`; `snapshotID` = их хеш. +2. `RequestHash`: `snapshotID` → `wireSnapshotID` (бамп `tm-request-v3`). +3. `chunk_status`: добавить `verdict_snapshot` (миграция), писать/читать; fast-path доверяет строке + только при совпадении `content_hash`+`wireSnapshotID`+`verdict_snapshot`, иначе проваливается в + `runAttempt` (re-classify бесплатно). +4. Job-снапшот-mismatch: демотировать из fail-loud стопа в per-chunk `content_hash`-решение. +5. (Опц.) расщепить `memoryMatchVersion` на inject/postcheck версии (§6.1). +6. Тесты: (а) вердикт-only смена → $0 re-classify, вердикт обновлён; (б) append-термин → re-bill + ТОЛЬКО фаерящихся чанков, остальные $0; (в) wire-смена (промпт/модель/capability) → честный + re-bill; (г) F1: никакой матч не отдаёт расходящийся текст (mutation: ослабить wire-match → + тест ловит stale-serve). + +--- + +**Резюме для оркестратора:** дефект — `snapshotID` в ключе вызова смешивает wire- и вердикт-входы. +Фикс — расщепить; ключ чекпоинта держать на wire-идентичности (по факту это `msgsContentHash` + +не-msgs wire-параметры), вердикт-версии убрать из ключа и re-classify-ить на resume бесплатно. +Безопасность (F1) сохраняется: reuse — на точном wire-match, вердикт-смена доказуемо wire-нейтральна. +Онгоинг-мина D15 снимается: append-термин re-bill-ит только реально затронутые чанки. Реализация — +малая и хирургическая (машинерия re-classify уже есть), но гейтит онгоинг Ф1.5 и требует ратификации. diff --git a/backend/internal/config/additive_reasoning_test.go b/backend/internal/config/additive_reasoning_test.go new file mode 100644 index 0000000..6db0ac1 --- /dev/null +++ b/backend/internal/config/additive_reasoning_test.go @@ -0,0 +1,96 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// additive_reasoning_test.go is the D13.6 fail-fast gate: a stage that THINKS +// (reasoning low|medium|high) on an ADDITIVE-billing provider (xAI — reasoning bills on top +// of completion) must declare reasoning_max_tokens, or LoadPipeline fails loud — the same +// structural discipline as the echo mine. Reverting the check in LoadPipeline makes the +// "armed" cases below stop failing. +func TestReasoningBufferFailFast(t *testing.T) { + today := time.Now().UTC().Format("2006-01-02") + dir := t.TempDir() + // A prompt template LoadPipeline can stat + split. + promptPath := filepath.Join(dir, "p.md") + if err := os.WriteFile(promptPath, []byte("sys\n---USER---\n{{text}}"), 0o644); err != nil { + t.Fatal(err) + } + modelsPath := filepath.Join(dir, "models.yaml") + if err := os.WriteFile(modelsPath, []byte(fmt.Sprintf(` +prices_checked: %q +default_model: additive-model +providers: + xai: + kind: openai + base_url: http://x + reasoning: additive + ds: + kind: openai + base_url: http://y + reasoning: subset +models: + additive-model: { provider: xai, price: { input_per_m: 1, output_per_m: 2 } } + subset-model: { provider: ds, price: { input_per_m: 1, output_per_m: 2 } } +`, today)), 0o644); err != nil { + t.Fatal(err) + } + models, err := LoadModels(modelsPath) + if err != nil { + t.Fatalf("models load: %v", err) + } + + load := func(stage string) error { + pp := filepath.Join(dir, "pipe.yaml") + body := fmt.Sprintf("core: C1\nversion: 1\ndefaults: { max_output_ratio: 2, min_max_tokens: 512 }\nstages:\n%s", stage) + if err := os.WriteFile(pp, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + _, e := LoadPipeline(pp, models) + return e + } + stage := func(model, reasoning string, buf int, escalateTo string) string { + s := fmt.Sprintf(" - { name: s, role: translator, model: %s, prompt: %s, prompt_version: v, reasoning: %q", model, promptPath, reasoning) + if buf > 0 { + s += fmt.Sprintf(", reasoning_max_tokens: %d", buf) + } + if escalateTo != "" { + s += fmt.Sprintf(", escalate_to: %s", escalateTo) + } + return s + " }" + } + + cases := []struct { + name string + stage string + wantErr bool + }{ + {"think-on additive no buffer → fail", stage("additive-model", "high", 0, ""), true}, + {"think-on additive with buffer → ok", stage("additive-model", "medium", 4000, ""), false}, + {"think-on subset no buffer → ok", stage("subset-model", "high", 0, ""), false}, + {"off additive no buffer → ok", stage("additive-model", "off", 0, ""), false}, + // finding #4: "" on an additive provider falls to the grok default (thinks) → MUST fail. + {"empty-reasoning additive no buffer → fail", stage("additive-model", "", 0, ""), true}, + {"empty-reasoning additive with buffer → ok", stage("additive-model", "", 4000, ""), false}, + {"empty-reasoning subset no buffer → ok", stage("subset-model", "", 0, ""), false}, + {"escalate_to additive think-on no buffer → fail", stage("subset-model", "high", 0, "additive-model"), true}, + {"escalate_to additive empty no buffer → fail", stage("subset-model", "", 0, "additive-model"), true}, + {"escalate_to additive think-on with buffer → ok", stage("subset-model", "high", 4000, "additive-model"), false}, + } + for _, c := range cases { + err := load(c.stage) + if c.wantErr { + if err == nil || !strings.Contains(err.Error(), "reasoning_max_tokens") { + t.Errorf("%s: want an error mentioning reasoning_max_tokens, got %v", c.name, err) + } + } else if err != nil { + t.Errorf("%s: want ok, got %v", c.name, err) + } + } +} diff --git a/backend/internal/config/models.go b/backend/internal/config/models.go index 64a121b..1aebb35 100644 --- a/backend/internal/config/models.go +++ b/backend/internal/config/models.go @@ -247,6 +247,35 @@ func (m *Models) providerPermissive(modelName string) bool { return m.Providers[mod.Provider].Permissive } +// providerReasoning returns a model's provider billing semantic ("subset" | "additive" | ""; +// D3/D6.2). Unknown model → "" (no additive assumption). +func (m *Models) providerReasoning(modelName string) string { + mod, ok := m.Models[modelName] + if !ok { + return "" + } + return m.Providers[mod.Provider].Reasoning +} + +// AdditiveReasoningTokens is the reasoning-token buffer to ADD to a reservation for a call to +// modelName at effort `reasoning` (D13.6): the stage's declared reasoning_max_tokens when the +// provider bills reasoning ADDITIVELY (xAI — on top of completion) AND the effort MAY think, +// else 0 (a subset provider folds reasoning into maxTokens). "May think" = any effort except an +// explicit "off": low|medium|high think, and an OMITTED/"" effort falls to the provider default +// (xAI-grok = thinking ON) — so "" must reserve the buffer too (self-review finding #4). The +// runner passes this to ledger.EstimateUSD so the reservation covers the overshoot instead of +// blinding the ceiling. LoadPipeline fail-fasts if the buffer is missing for an additive +// non-"off" stage, so a >0 declaredBuffer is guaranteed here for that case. +func (m *Models) AdditiveReasoningTokens(modelName, reasoning string, declaredBuffer int) int { + if declaredBuffer <= 0 || reasoning == "off" { + return 0 + } + if m.providerReasoning(modelName) == "additive" { + return declaredBuffer + } + return 0 +} + // APIKey resolves a provider's key from the environment ("" if the provider // declares no env var — the local backend). func (p Provider) APIKey() string { diff --git a/backend/internal/config/pipeline.go b/backend/internal/config/pipeline.go index 4531d55..0d0982e 100644 --- a/backend/internal/config/pipeline.go +++ b/backend/internal/config/pipeline.go @@ -64,6 +64,12 @@ type Stage struct { PromptVersion string `yaml:"prompt_version"` Temperature float64 `yaml:"temperature"` Reasoning string `yaml:"reasoning"` // "", off, low, medium, high + // ReasoningMaxTokens is the explicit reasoning-token BUFFER reserved for this stage + // when it thinks (reasoning low|medium|high) on an ADDITIVE-billing provider (xAI — + // reasoning bills on top of completion, D13.6). Required (>0) in exactly that case + // (fail-fast in LoadPipeline); ignored for subset providers / reasoning-off. It only + // sizes the reservation (EstimateUSD), so it is NOT part of the snapshot/wire. + ReasoningMaxTokens int `yaml:"reasoning_max_tokens"` // EscalateTo is the SINGLE-HOP fallback model tried ONCE when this stage's // output is a deterministic content-failure another model might fix (echo / // excision / refusal — D12). Empty = no escalation (e.g. the editor is pinned @@ -222,6 +228,24 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) { default: bad("stage %q: reasoning must be off|low|medium|high, got %q", st.Name, st.Reasoning) } + // D13.6: on an ADDITIVE-billing provider (xAI — reasoning bills ON TOP of completion) a + // stage that MAY think needs a reserved reasoning_max_tokens buffer, else the reservation + // under-budgets and the spend ceiling is blind to the overshoot pricing.go only + // acknowledges. "May think" = any effort EXCEPT an explicit reasoning:"off" (which sends + // the provider's disable — off_effort:"none" for grok): low|medium|high think, and an + // OMITTED/"" effort falls to the provider DEFAULT, which for xAI-grok is "low" (thinking + // ON, additive) — so "" must NOT escape the gate (self-review finding #4: the earlier + // thinks-only check let it through). Subset providers never need a buffer. This is the + // structural gate D6.2 requires before think-ON is allowed on grok (the contrast the task + // draws with the echo mine). Checks the primary AND the escalate_to model (same effort). + if st.Reasoning != "off" && st.ReasoningMaxTokens <= 0 { + if models.providerReasoning(st.Model) == "additive" { + bad("stage %q: reasoning=%q on additive-billing provider (model %q) requires reasoning_max_tokens>0 — xAI bills reasoning on top of completion and thinks by default unless reasoning is explicitly \"off\"; without a reserved buffer the spend ceiling is blind to the overshoot (D6.2/D13.6)", st.Name, st.Reasoning, st.Model) + } + if st.EscalateTo != "" && models.providerReasoning(st.EscalateTo) == "additive" { + bad("stage %q: reasoning=%q with escalate_to on an additive-billing provider (model %q) requires reasoning_max_tokens>0 (D6.2/D13.6)", st.Name, st.Reasoning, st.EscalateTo) + } + } switch st.Channel { case "", "sfw", "adult": default: diff --git a/backend/internal/ledger/pricing.go b/backend/internal/ledger/pricing.go index 749d2c9..08f9bc3 100644 --- a/backend/internal/ledger/pricing.go +++ b/backend/internal/ledger/pricing.go @@ -106,14 +106,24 @@ func CostUSD(price ModelPrice, u llm.Usage) float64 { // misses the cache — or, worse, is entirely WRITTEN to the cache (Anthropic // write ×1.25/×2 дороже input) — and the completion runs to maxTokens. // Deliberately pessimistic: реальная цена не должна превышать резерв, иначе -// settle пробивает потолок, пропущенный на допуске. Известный остаточный -// случай: additive reasoning-токены (xAI) биллятся ПОВЕРХ completion и оценкой -// не покрыты — допустимый овершут ≤ одной резервации, как у донора. -func EstimateUSD(price ModelPrice, promptTokens, maxTokens int) float64 { +// settle пробивает потолок, пропущенный на допуске. +// +// reasoningBudgetTokens (D13.6) reserves an EXPLICIT allowance for providers that +// bill reasoning ADDITIVELY (xAI: reasoning tokens land ON TOP of completion, not +// inside maxTokens — the overshoot pricing.go used to only acknowledge). It is +// priced at OutputPerM (reasoning bills as output) and MUST be > 0 for a think-ON +// stage on such a provider (config fail-fasts otherwise — pipeline.go); it is 0 for +// subset-billing providers (deepseek/zai/kimi: reasoning ⊆ completion ≤ maxTokens, +// already covered) and for reasoning-off stages. This closes the D6.2 block on +// think-ON grok: the ceiling is no longer blind to the additive reasoning spend. +func EstimateUSD(price ModelPrice, promptTokens, maxTokens, reasoningBudgetTokens int) float64 { inPerM := price.InputPerM if price.CacheWritePerM > inPerM { inPerM = price.CacheWritePerM } + if reasoningBudgetTokens < 0 { + reasoningBudgetTokens = 0 + } return float64(promptTokens)*inPerM/1_000_000 + - float64(maxTokens)*price.OutputPerM/1_000_000 + float64(maxTokens+reasoningBudgetTokens)*price.OutputPerM/1_000_000 } diff --git a/backend/internal/ledger/pricing_test.go b/backend/internal/ledger/pricing_test.go index f7f9dcd..6ec7ef6 100644 --- a/backend/internal/ledger/pricing_test.go +++ b/backend/internal/ledger/pricing_test.go @@ -92,7 +92,7 @@ func TestPricerNeverZero(t *testing.T) { func TestEstimateUSDIsPessimistic(t *testing.T) { price := ModelPrice{InputPerM: 2.0, CachedPerM: 0.2, OutputPerM: 10.0} - est := EstimateUSD(price, 1_000, 2_000) + est := EstimateUSD(price, 1_000, 2_000, 0) approx(t, est, 1_000*2.0/1e6+2_000*10.0/1e6, "estimate") // Реальный вызов с кэш-хитом обязан стоить не больше оценки — иначе // settle раздует committed выше потолка, пропущенного на допуске. @@ -101,3 +101,22 @@ func TestEstimateUSDIsPessimistic(t *testing.T) { t.Fatalf("real cost %v exceeds reservation estimate %v", real, est) } } + +// TestEstimateUSDAdditiveReasoningBuffer covers D13.6: the additive-reasoning buffer is +// reserved as output tokens, so an xAI-style think-ON call whose reasoning lands on top of +// completion no longer overshoots the reservation and blinds the ceiling. +func TestEstimateUSDAdditiveReasoningBuffer(t *testing.T) { + price := ModelPrice{InputPerM: 2.0, OutputPerM: 10.0} + base := EstimateUSD(price, 1_000, 2_000, 0) + withBuf := EstimateUSD(price, 1_000, 2_000, 3_000) + // The buffer adds exactly 3_000 output-priced tokens over the base estimate. + approx(t, withBuf-base, 3_000*10.0/1e6, "reasoning buffer term") + // A real additive call (completion 2_000 + reasoning 2_500 ≤ buffer 3_000) stays within. + real := CostUSD(price, llm.Usage{PromptTokens: 1_000, CompletionTokens: 2_000, ReasoningTokens: 2_500}) + if real > withBuf { + t.Fatalf("additive real cost %v exceeds buffered reservation %v", real, withBuf) + } + if real <= base { + t.Fatalf("without the buffer the reservation %v would be blind to the reasoning spend (real %v)", base, real) + } +} diff --git a/backend/internal/llm/capability.go b/backend/internal/llm/capability.go index 56e5912..64167d8 100644 --- a/backend/internal/llm/capability.go +++ b/backend/internal/llm/capability.go @@ -1,6 +1,6 @@ package llm -// capability.go is the per-MODEL wire-shape layer (03-decisions-log D3.1). +// capability.go is the per-MODEL wire-shape layer (05-decisions-log D3.1). // OpenAI-compatible providers share an ENDPOINT but not a BODY: Kimi accepts // only temperature:1 (else 400), gpt-5-mini needs max_completion_tokens WITHOUT // temperature (else 400), Gemini 3.1 Pro 400s if thinking is disabled. The diff --git a/backend/internal/pipeline/coverage.go b/backend/internal/pipeline/coverage.go index 292e276..263b82f 100644 --- a/backend/internal/pipeline/coverage.go +++ b/backend/internal/pipeline/coverage.go @@ -26,11 +26,15 @@ import ( // coverageGateVersion versions the segmentation + metric of this gate. It is folded // into the job snapshot (like chunkerVersion/estimatorVersion), so a change to the -// algorithm — or turning the gate ON — is a loud --resnapshot that re-evaluates every -// chunk from its checkpoint for FREE (no re-bill), never a silent divergence between -// an old checkpoint's stored verdict and a new gate. The segmentation is pinned 1:1 -// to eval/refusal_bench.py (D12 Q1); any change is Python-first, in lock-step, and -// bumps this constant. +// algorithm — or turning the gate ON — is a loud --resnapshot, never a silent divergence +// between an old checkpoint's stored verdict and a new gate. HONEST COST (D15): because +// the snapshotID is baked into every call's RequestHash (render.go), that --resnapshot +// re-bills the WHOLE book — every checkpoint misses on the new snapshot, even wire-identical +// requests whose verdict merely needs re-classifying (the content-addressed reuse that +// could make them free is gated on an unchanged snapshot). Acceptable for the static +// acceptance book (loud, no divergence); the reason content-addressed checkpoint reuse is +// required before ongoing (D15.2). The segmentation is pinned 1:1 to eval/refusal_bench.py +// (D12 Q1); any change is Python-first, in lock-step, and bumps this constant. const coverageGateVersion = "coverage-v1-naive-split-lower-bound" // oracleDefaultCorridorLow is refusal_bench.py's fallback len_ratio LOWER bound for a diff --git a/backend/internal/pipeline/memnorm.go b/backend/internal/pipeline/memnorm.go index 44c6e24..4dd556f 100644 --- a/backend/internal/pipeline/memnorm.go +++ b/backend/internal/pipeline/memnorm.go @@ -40,7 +40,7 @@ var trad2simpRaw string // editing/completing the data file also invalidates — drift-proof (D8): you cannot // forget to bump a version when you change the table, because the table's bytes ARE // the version. -const memoryNormAlgoVersion = "memnorm-v2-nfkc+stripignorable+trad+kana+lower/nfc+dash+stripignorable+lower+yofold" +const memoryNormAlgoVersion = "memnorm-v3-nfkc+apos+stripignorable+trad+kana+lower/nfc+dash+apos+stripignorable+lower+yofold" var ( // trad2simp is the parsed Traditional→Simplified single-char map (data/trad2simp.txt). @@ -106,6 +106,9 @@ func normalizeSourceKey(s string) string { var b strings.Builder b.Grow(len(s)) for _, r := range s { + if isApostropheVariant(r) { + r = '\'' // typographic apostrophe (O’Brien) folds to ASCII so a key matches either form + } if isIgnorableForMatch(r) { continue // zero-width / format / variation selector: invisible, never part of the key } @@ -145,6 +148,9 @@ func normalizeTargetForm(s string) string { if isDashVariant(r) { r = '-' } + if isApostropheVariant(r) { + r = '\'' // symmetric with the source side (д’Артаньян) — no false post-check MISS on a typographic apostrophe + } if isIgnorableForMatch(r) { continue // drop zero-width/format/VS (soft hyphen already folded to '-' above) } @@ -175,6 +181,19 @@ func isDashVariant(r rune) bool { return false } +// isApostropheVariant reports whether r is a typographic apostrophe/single-quote variant that +// should fold to ASCII '\” (Task-6 re-audit): U+2018/U+2019 curly single quotes, U+02BC +// modifier-letter apostrophe, U+FF07 fullwidth apostrophe. Smart-quote editors turn a source +// key O'Brien's apostrophe into U+2019, which NFKC/NFC do NOT fold to U+0027 — leaving the key +// silently un-matchable (тихо пусто) on en names. Folded symmetrically on both sides. +func isApostropheVariant(r rune) bool { + switch r { + case '‘', '’', 'ʼ', '＇': + return true + } + return false +} + // isIgnorableForMatch reports whether r is a default-ignorable / format code point that // must be DROPPED before matching. These survive NFKC/NFC unchanged yet are invisible or // carry no character identity, so leaving them in silently breaks a match — on the source diff --git a/backend/internal/pipeline/memnorm_test.go b/backend/internal/pipeline/memnorm_test.go index f812f98..232cbe4 100644 --- a/backend/internal/pipeline/memnorm_test.go +++ b/backend/internal/pipeline/memnorm_test.go @@ -3,6 +3,8 @@ package pipeline import ( "strings" "testing" + + "textmachine/backend/internal/store" ) // TestTradTableCoversKnownCases asserts the embedded trad→simp table covers every @@ -163,3 +165,22 @@ func TestNormalizeStripsDefaultIgnorable(t *testing.T) { t.Errorf("soft hyphen on the target side = %q, want %q (must fold to '-', not be stripped)", got, want) } } + +// TestApostropheFold covers the Task-6 re-audit finding: a typographic apostrophe (U+2019, +// smart-quote editors) folds to ASCII on BOTH sides, so a source key O'Brien matches either +// spelling (no тихо пусто) and the post-check does not false-MISS a д'Артаньян. Reverting the +// isApostropheVariant folds breaks these. +func TestApostropheFold(t *testing.T) { + if normalizeSourceKey("O’Brien") != normalizeSourceKey("O'Brien") { + t.Error("source: typographic and ASCII apostrophe must normalize identically") + } + if normalizeTargetForm("д’Артаньян") != normalizeTargetForm("д'Артаньян") { + t.Error("target: typographic and ASCII apostrophe must normalize identically") + } + // End-to-end: an ASCII-apostrophe key matches a chunk written with a typographic apostrophe. + e := gl("o'brien", "О'Брайен", "", "approved") + b := bankFrom([]store.GlossaryEntry{e}) + if _, ok := injMap(b.Select("mr o’brien arrived.", 1, nil, 0))["o'brien"]; !ok { + t.Error("typographic apostrophe in the chunk must match the ASCII-apostrophe key") + } +} diff --git a/backend/internal/pipeline/memory.go b/backend/internal/pipeline/memory.go index 1c7c304..5d1b128 100644 --- a/backend/internal/pipeline/memory.go +++ b/backend/internal/pipeline/memory.go @@ -32,7 +32,11 @@ import ( // ban, longest-match containment, spoiler gate, sticky, budget priority order). Folded // into memoryVersion() → a change is a loud --resnapshot (it shifts the injected bytes // → the wire → a resumed checkpoint's content). Sibling of memoryNormVersion. -const memoryMatchVersion = "memmatch-v2-perlang-minkey+collision-ambiguous+longest+spoiler+sticky+budget+postcheck-declaware" +// v3 (D16.2/D16.3): sticky now INHERITS the original match's disposition instead of +// recomputing it status-based (a collision-prone AMBIGUOUS carry no longer silently +// upgrades to CONFIRMED), and a spaced-phonetic (Latin/Cyrillic) source key is validated +// against a word boundary (approved "rose" no longer fires inside "roseanne"). +const memoryMatchVersion = "memmatch-v3-perlang-minkey+collision-ambiguous+longest+spoiler+sticky-inherit-disp+phonetic-srcboundary+budget+postcheck-declaware" // minKeyLenHan / minKeyLenPhonetic are the per-language single-key floors (A3, registry // "min_key_len per-язык"). An ideographic (Han) key is semantically distinct at 2 chars @@ -129,10 +133,15 @@ type pickedEntry struct { // memorySelection is the hot path's output for one chunk. type memorySelection struct { - injected []pickedEntry // in budget priority order (what the model sees) - rejected []pickedEntry // spoiler-window rejects (C1, logged) - evicted []pickedEntry // dropped by the token budget (F2, logged) - activeIDs map[string]bool // exact-matched ids (NOT sticky) → the next chunk's sticky_prev + injected []pickedEntry // in budget priority order (what the model sees) + rejected []pickedEntry // spoiler-window rejects (C1, logged) + evicted []pickedEntry // dropped by the token budget (F2, logged) + // activeIDs are the exact-matched ids (NOT sticky) → the next chunk's sticky_prev, + // each mapped to the DISPOSITION it fired with. Sticky carries this disposition + // forward instead of recomputing it (D16.2): a sticky carry has no firing key, so it + // cannot re-derive the collision-prone downgrade, and recomputing from status alone + // would silently upgrade a collision-prone AMBIGUOUS match to CONFIRMED. + activeIDs map[string]injectionDisposition } func (b *MemoryBank) Version() string { return b.version } @@ -286,9 +295,14 @@ func glossaryLineTokens(e *memoryEntry) int { // unbounded; otherwise records are kept in priority order while the cumulative injected // token cost stays within budget — the rest are EVICTED (dropped but logged, F2; the // budget may be underfilled, "лучше ничего, чем мусор"). -func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]bool, budgetTokens int) memorySelection { +func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]injectionDisposition, budgetTokens int) memorySelection { ntext := []rune(normalizeSourceKey(chunk)) occ := b.ac.matches(ntext) + // Source word-boundary for spaced-phonetic keys (D16.3): drop an occurrence of a Latin/ + // Cyrillic key that fired INSIDE a longer word of the same script ("rose" in "roseanne"). + // Closes the source-vs-target boundary asymmetry (the target side already has + // containsWholeWord). Han/kana are NOT boundary-checked (no word segmentation) — deliberate. + occ = b.suppressUnboundedPhonetic(occ, ntext) // Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when // the longer key belongs to a spoiler-VALID entry at this chapter (self-review #6). A // spoiler-blocked longer key must not suppress a valid shorter name nested inside it. @@ -305,7 +319,7 @@ func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]boo } } - sel := memorySelection{activeIDs: map[string]bool{}} + sel := memorySelection{activeIDs: map[string]injectionDisposition{}} hits := map[string]pickedEntry{} // id → picked (exact) // Iterate entries in their frozen (ORDER BY-stable) order — never map order. @@ -319,15 +333,21 @@ func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]boo sel.rejected = append(sel.rejected, pickedEntry{e, via, memReject}) continue } - hits[e.id] = pickedEntry{e, via, dispositionFor(e, via)} - sel.activeIDs[e.id] = true // exact-matched ids feed the next chunk's sticky (NOT sticky-carried) + disp := dispositionFor(e, via) + hits[e.id] = pickedEntry{e, via, disp} + sel.activeIDs[e.id] = disp // exact-matched ids feed the next chunk's sticky WITH the disposition they fired with (D16.2) } // Sticky scene-inertia: carry prior-chunk exact matches not re-matched here, unless - // the spoiler window blocks them (spoiler beats sticky). + // the spoiler window blocks them (spoiler beats sticky). The carry INHERITS the prior + // match's disposition (D16.2) — a sticky carry has no firing key, so it cannot re-derive + // the collision-prone downgrade; recomputing status-based would silently upgrade a + // collision-prone AMBIGUOUS match to CONFIRMED, past the post-check, into the editor + // constraints. An AMBIGUOUS carry stays AMBIGUOUS (⟨проверить⟩, excluded from the editor). for ei := range b.entries { e := &b.entries[ei] - if !stickyPrev[e.id] { + carryDisp, sticky := stickyPrev[e.id] + if !sticky { continue } if _, already := hits[e.id]; already { @@ -336,7 +356,7 @@ func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]boo if spoilerBlocked(e, chapter) { continue } - hits[e.id] = pickedEntry{e, "sticky", dispositionFor(e, "sticky")} + hits[e.id] = pickedEntry{e, "sticky", carryDisp} } // Deterministic priority order for the budget: confirmed>ambiguous, exact>sticky, @@ -387,7 +407,9 @@ func spoilerBlocked(e *memoryEntry, chapter int) bool { // AMBIGUOUS (unverified + forced post-check), because such a key may have fired inside an // unrelated word rather than on the entity (external-review major #2 — the A2 // "surface-collision → AMBIGUOUS" branch). allow_short is the author's explicit override. -// A sticky carry (via=="sticky") has no firing key, so it keeps its status-based disposition. +// Called ONLY for an EXACT match (a real firing key): a sticky carry INHERITS its prior +// disposition instead (D16.2), never routing through here, so it cannot silently upgrade a +// collision-prone match. The via!="sticky" guard is kept defensive. func dispositionFor(e *memoryEntry, via string) injectionDisposition { if via != "sticky" && !e.allowShort && collisionProneKey(via) { return memAmbiguous @@ -616,6 +638,67 @@ func (b *MemoryBank) suppressContained(ms []acMatch, chapter int) []acMatch { return kept } +// suppressUnboundedPhonetic drops an occurrence of a SPACED-SCRIPT phonetic key (Latin or +// Cyrillic — scripts that delimit words with whitespace/non-letters, exactly like the +// target-side containsWholeWord) when a neighbouring rune is a letter of the SAME script: +// the key fired INSIDE a longer word ("rose" in "roseanne"/"roses"), not on the entity +// (D16.3). This closes the source-vs-target boundary asymmetry the ≤3 collision-downgrade +// does not (a ≥4 phonetic key fires CONFIRMED with no boundary today). A cross-script +// neighbour (a Latin name abutting a Han char in unspaced source) is a valid boundary, so +// only a SAME-script letter suppresses — mirroring "не-буквенных или чужескриптовых". +// KANA and HAN are deliberately NOT boundary-checked: they have no word segmentation, so a +// letter-boundary rule would false-negative the ubiquitous kana name+particle case (すずきは) +// or contradict the long-key-CONFIRMED contract — their ≥4 precision is deferred to the +// kana-precision measurement + the B6 tokenizer (D16; flagged to the orchestrator). ntext is +// the already-normalized chunk. O(n) over the few matches in a chunk. +func (b *MemoryBank) suppressUnboundedPhonetic(ms []acMatch, ntext []rune) []acMatch { + var kept []acMatch + for _, m := range ms { + script := spacedPhoneticScript(b.ac.keys[m.keyIdx]) + if script != nil { + beforeBad := m.start > 0 && letterInScript(ntext[m.start-1], script) + afterBad := m.end < len(ntext) && letterInScript(ntext[m.end], script) + if beforeBad || afterBad { + continue // the key is inside a longer same-script word — not a whole-entity match + } + } + kept = append(kept, m) + } + return kept +} + +// spacedPhoneticScript returns the word-delimited alphabetic script a normalized key belongs +// to for the source word-boundary check — unicode.Latin or unicode.Cyrillic — or nil when the +// key carries a Han ideograph or kana (no word segmentation) or mixes the two spaced scripts. +// Digits/punctuation don't set the script but don't disqualify (so "o'brien" is still Latin). +func spacedPhoneticScript(normKey string) *unicode.RangeTable { + var script *unicode.RangeTable + for _, r := range normKey { + switch { + case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul): + return nil // an ideographic/kana anchor is never boundary-checked here + case unicode.In(r, unicode.Latin): + if script == unicode.Cyrillic { + return nil // mixed spaced scripts — do not boundary-check + } + script = unicode.Latin + case unicode.In(r, unicode.Cyrillic): + if script == unicode.Latin { + return nil + } + script = unicode.Cyrillic + } + } + return script +} + +// letterInScript reports whether r is a LETTER of the given spaced script — the "same-script +// letter" that breaks a word boundary. A digit, punctuation, space, or a letter of another +// script is a valid boundary and returns false. +func letterInScript(r rune, script *unicode.RangeTable) bool { + return unicode.IsLetter(r) && unicode.In(r, script) +} + // injectivityCollisions reports approved dst collisions (B2): two distinct source // terms mapped to the SAME dst (one Russian surface for two entities → the reader // cannot tell them apart), returned as human-readable strings for a load-time warning. diff --git a/backend/internal/pipeline/memory_test.go b/backend/internal/pipeline/memory_test.go index 7d36b1f..70a2b75 100644 --- a/backend/internal/pipeline/memory_test.go +++ b/backend/internal/pipeline/memory_test.go @@ -104,9 +104,10 @@ func TestHotPathSpecT1toT10(t *testing.T) { t.Error("T5 trad→simp: 魯鎮 did not match 鲁镇") } - // T6: pronominal chunk — sticky carries 四叔 from the previous chunk. + // T6: pronominal chunk — sticky carries 四叔 from the previous chunk, INHERITING the + // prior chunk's CONFIRMED disposition (D16.2: sticky carries the disposition forward). sishuID := "四叔\x1fchar\x1f0\x1f0" - sel = b.Select("他慢慢站起来，叹了口气。", 2, map[string]bool{sishuID: true}, budget) + sel = b.Select("他慢慢站起来，叹了口气。", 2, map[string]injectionDisposition{sishuID: memConfirmed}, budget) var stickySishu *pickedEntry for i := range sel.injected { if sel.injected[i].entry.src == "四叔" { @@ -399,6 +400,107 @@ func TestPerLanguageMinKeyAndCollisionDisposition(t *testing.T) { } } +// TestStickyInheritsDisposition covers D16.2: a collision-prone AMBIGUOUS exact match must +// carry into the next chunk's sticky AS AMBIGUOUS — never silently upgraded to CONFIRMED. A +// status-based recompute would upgrade it (past the post-check, into the editor constraints). +// Reverting the inherit change (recomputing via dispositionFor) makes the carry CONFIRMED here. +func TestStickyInheritsDisposition(t *testing.T) { + // An approved name whose only key is collision-prone (3 kana): an exact match fires + // AMBIGUOUS (the A2 collision downgrade), not CONFIRMED, despite the approved status. + suzuki := gl("すずき", "Судзуки", "", "approved") + b := bankFrom([]store.GlossaryEntry{suzuki}) + + sel1 := b.Select("すずきが来た。", 1, nil, 0) + if injMap(sel1)["すずき"] != memAmbiguous { + t.Fatalf("collision-prone approved key must fire AMBIGUOUS: %v", injMap(sel1)) + } + if len(sel1.activeIDs) != 1 { + t.Fatalf("want exactly one active id, got %d", len(sel1.activeIDs)) + } + for _, disp := range sel1.activeIDs { + if disp != memAmbiguous { + t.Fatalf("activeIDs must carry the AMBIGUOUS disposition, got %q", disp) + } + } + + // Pronominal chunk (no key): sticky carries すずき, and its disposition MUST stay AMBIGUOUS. + sel2 := b.Select("彼は立ち上がった。", 1, sel1.activeIDs, 0) + var carried *pickedEntry + for i := range sel2.injected { + if sel2.injected[i].entry.src == "すずき" { + carried = &sel2.injected[i] + } + } + if carried == nil || carried.via != "sticky" { + t.Fatalf("sticky did not carry すずき: %v", injMap(sel2)) + } + if carried.disp != memAmbiguous { + t.Errorf("D16.2: sticky carry upgraded disposition to %q, want AMBIGUOUS", carried.disp) + } +} + +// TestUnionStickyNewestWins covers self-review finding #6: unionSticky merges the sticky +// window so the MOST RECENT firing's disposition wins — an id that fired CONFIRMED via a clean +// key then AMBIGUOUS via a collision alias must carry AMBIGUOUS (no silent upgrade); a revert to +// keep-first would reintroduce the D16.2 upgrade. This exercises the merge that +// TestStickyInheritsDisposition (single-chunk activeIDs) bypasses. +func TestUnionStickyNewestWins(t *testing.T) { + id := "x\x1f\x1f0\x1f0" + // Oldest→newest CONFIRMED then AMBIGUOUS → newest (AMBIGUOUS) wins (the anti-upgrade case). + if got := unionSticky([]map[string]injectionDisposition{{id: memConfirmed}, {id: memAmbiguous}})[id]; got != memAmbiguous { + t.Errorf("newest firing must win: got %q, want ambiguous", got) + } + // Reversed → a re-established CONFIRMED (via a clean key in the newer chunk) wins. + if got := unionSticky([]map[string]injectionDisposition{{id: memAmbiguous}, {id: memConfirmed}})[id]; got != memConfirmed { + t.Errorf("a re-established CONFIRMED must win: got %q, want confirmed", got) + } + if unionSticky(nil) != nil { + t.Error("empty window → nil") + } +} + +// TestSourcePhoneticWordBoundary covers D16.3: a spaced-phonetic (Latin/Cyrillic) source key +// fires only as a WHOLE WORD, never inside a longer same-script word ("rose" in "roseanne"). +// A cross-script neighbour is a valid boundary. Reverting suppressUnboundedPhonetic fires +// "rose" inside "roseanne" and fails here. +func TestSourcePhoneticWordBoundary(t *testing.T) { + rose := gl("rose", "Роза", "", "approved") // 4 Latin → CONFIRMED, but boundary-checked + b := bankFrom([]store.GlossaryEntry{rose}) + + // Inside a longer Latin word → suppressed (both a suffix and a prefix collision). + if s := b.Select("roseanne walked in.", 1, nil, 0); len(s.injected) != 0 { + t.Errorf("latin key fired inside 'roseanne': %v", injMap(s)) + } + if s := b.Select("the roses bloomed.", 1, nil, 0); len(s.injected) != 0 { + t.Errorf("latin key fired inside 'roses': %v", injMap(s)) + } + // Standalone / at string edges → fires CONFIRMED. + if injMap(b.Select("a rose bloomed.", 1, nil, 0))["rose"] != memConfirmed { + t.Error("latin key must fire as a whole word") + } + if injMap(b.Select("rose.", 1, nil, 0))["rose"] != memConfirmed { + t.Error("latin key at string edges must fire") + } + // Cross-script neighbour (a Han char in unspaced source) is a valid boundary → fires. + if injMap(b.Select("我叫rose。", 1, nil, 0))["rose"] != memConfirmed { + t.Error("latin key abutting a cross-script (Han) neighbour must fire") + } +} + +// TestKanaNotSourceBoundaryChecked documents the D16.3 kana carve-out: kana has no word +// segmentation, so a kana key is NOT boundary-checked — it fires between kana neighbours (a +// name+particle case すずきは), matching the long-key-CONFIRMED contract. The ≥4-kana compound +// precision is deferred to the kana-precision measurement + B6 tokenizer (flagged). +func TestKanaNotSourceBoundaryChecked(t *testing.T) { + // A long kana key must still fire between kana neighbours (と…が), unaffected by the + // source boundary check (this mirrors the existing long-phonetic-CONFIRMED expectation). + name := gl("ながいなまえ", "Длинное имя", "", "approved") + b := bankFrom([]store.GlossaryEntry{name}) + if injMap(b.Select("私とながいなまえが会った。", 1, nil, 0))["ながいなまえ"] != memConfirmed { + t.Error("kana key must NOT be source-boundary-suppressed (name+particle recall)") + } +} + // TestOverlappingNonNestedKeysBothFire documents the eval-review low finding: two keys // that OVERLAP without nesting both fire (both surfaces are genuinely present in the // text). Accepted behavior (precision-side), asserted so a future change is deliberate. diff --git a/backend/internal/pipeline/memseed.go b/backend/internal/pipeline/memseed.go index e52eae4..6ada731 100644 --- a/backend/internal/pipeline/memseed.go +++ b/backend/internal/pipeline/memseed.go @@ -148,16 +148,32 @@ func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) { } out = append(out, e) } - // Overlapping spoiler windows for the SAME (src,sense) with DIFFERENT dst inject two - // contradictory renderings at a chapter in the overlap — silently (external-review - // minor #4). Fail loud. UNIQUE(src,sense,window) already blocks identical windows, so - // this catches the partial-overlap case. Iterate `out` in order (deterministic message). + // Overlapping spoiler windows for the SAME src inject CONTRADICTORY renderings at a + // chapter in the overlap — silently (external-review minor #4; D16.1). The deterministic + // matcher keys ONLY on src (it cannot disambiguate sense within a chunk), so BOTH rows + // fire together (keyOwners maps the shared key to both indices) and the post-check is + // guaranteed to miss one — with the gate ON, a deterministic per-chunk livelock. Fail + // loud. UNIQUE(src,sense,window) already blocks identical (src,sense,window), so this + // catches partial overlap and the different-SENSE case the old same-sense-only guard let + // through. Iterate `out` in order (deterministic message). for i := range out { for j := 0; j < i; j++ { a, b := out[i], out[j] - if a.Src == b.Src && a.Sense == b.Sense && a.Dst != b.Dst && windowsOverlap(a.SinceCh, a.UntilCh, b.SinceCh, b.UntilCh) { + if a.Src != b.Src || a.Dst == b.Dst || !windowsOverlap(a.SinceCh, a.UntilCh, b.SinceCh, b.UntilCh) { + continue + } + if a.Sense == b.Sense { problems = append(problems, fmt.Sprintf("term %q (sense %q): spoiler windows [%d,%d] and [%d,%d] overlap with different dst (%q vs %q) — a term has ONE rendering per chapter", a.Src, a.Sense, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh, b.Dst, a.Dst)) + continue + } + // Different sense, overlapping windows, different dst (D16.1 polysemy livelock). + // Scope to a MATCHABLE src: an A3-banned single-char src (道 — significantLen below + // the min-key floor, never in the automaton) is inert and cannot livelock, so its + // polysemy stays allowed. Both rows must be matchable for the contradiction to fire. + if srcKeyFires(a.Src, a.AllowShort) && srcKeyFires(b.Src, b.AllowShort) { + problems = append(problems, fmt.Sprintf("term %q: senses %q→%q and %q→%q have OVERLAPPING spoiler windows [%d,%d] and [%d,%d] — the deterministic matcher keys only on src and cannot pick a sense, so both inject as authoritative and the post-check is guaranteed to miss one (give them non-overlapping windows, merge the sense, or mark one status: auto)", + a.Src, b.Sense, b.Dst, a.Sense, a.Dst, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh)) } } } @@ -167,6 +183,98 @@ func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) { return out, nil } +// approvedSharedKeyCollisions detects the alias-generalization of the D16.1 polysemy livelock +// (Task-6 re-audit, confirmed by execution): a firing KEY (normalized src OR alias) shared by +// two approved terms with DIFFERENT dst and OVERLAPPING spoiler windows. The matcher keys only +// on the surface (keyOwners maps the shared key to BOTH terms), so both inject as authoritative +// and the post-check is guaranteed to miss one — a deterministic chunk livelock with the gate +// ON, exactly like same-src polysemy but reached via a shared прозвище (老赵 for two 赵). It does +// NOT skip same-src pairs: loadGlossarySeed's D16.1 check scopes its fail-loud to a MATCHABLE +// src (srcKeyFires), so a same-src pair whose src is single-key-BANNED (道) but that shares an +// ELIGIBLE alias (大道) slips D16.1 entirely — this catches it via the shared alias key (self- +// review finding #1). In the common case (an eligible shared src) loadGlossarySeed fails loud +// FIRST and seedGlossary never reaches here, so there is no double report. Returns human +// messages (deterministic order); seedGlossary fails loud on any. A shared key with the SAME +// dst (a legitimate merge) or NON-overlapping windows (a spoiler handoff) is not a contradiction. +func approvedSharedKeyCollisions(entries []store.GlossaryEntry) []string { + owners := map[string][]int{} // normalized firing key → indices of approved terms that own it + for i, e := range entries { + if e.Status != "approved" || strings.TrimSpace(e.Dst) == "" { + continue + } + for _, k := range entryFiringKeys(e) { + owners[k] = append(owners[k], i) + } + } + keys := make([]string, 0, len(owners)) + for k := range owners { + keys = append(keys, k) + } + sort.Strings(keys) + var out []string + seenPair := map[[2]int]bool{} + for _, k := range keys { + idxs := owners[k] + for a := 0; a < len(idxs); a++ { + for b := a + 1; b < len(idxs); b++ { + i, j := idxs[a], idxs[b] + ei, ej := entries[i], entries[j] + if ei.Dst == ej.Dst { + continue // same dst → both render identically, no contradiction (incl. an alias of one entry) + } + if !windowsOverlap(ei.SinceCh, ei.UntilCh, ej.SinceCh, ej.UntilCh) { + continue + } + pair := [2]int{i, j} + if seenPair[pair] { + continue + } + seenPair[pair] = true + out = append(out, fmt.Sprintf("firing key %q is shared by different terms %q→%q and %q→%q with overlapping spoiler windows — both inject as authoritative and the deterministic matcher cannot pick one (livelock; give non-overlapping windows, a more specific alias, or drop the shared surface)", + k, ei.Src, ei.Dst, ej.Src, ej.Dst)) + } + } + } + return out +} + +// entryFiringKeys returns the normalized keys of a term ELIGIBLE to enter the automaton (the +// same eligibility materializeMemory applies: src + aliases, each passing the per-language +// single-key floor or allow_short). Mirrors the matcher so the collision check sees exactly the +// keys that would fire. +func entryFiringKeys(e store.GlossaryEntry) []string { + surfaces := []string{e.Src} + for _, a := range e.Aliases { + surfaces = append(surfaces, a.Alias) + } + var keys []string + seen := map[string]bool{} + for _, s := range surfaces { + nk := normalizeSourceKey(s) + if nk == "" || seen[nk] { + continue + } + if significantLen(nk) >= minKeyLenFor(nk) || e.AllowShort { + seen[nk] = true + keys = append(keys, nk) + } + } + return keys +} + +// srcKeyFires reports whether src normalizes to a key eligible to ENTER the matcher's +// automaton — it passes the per-language single-key floor (minKeyLenFor), or allow_short +// overrides it. Mirrors materializeMemory's eligibility test. An ineligible src (a single- +// char Han like 道, banned by A3) never fires on the hot path, so a polysemy contradiction on +// it is inert and must not fail the seed load (D16.1 scoping). +func srcKeyFires(src string, allowShort bool) bool { + nk := normalizeSourceKey(src) + if nk == "" { + return false + } + return allowShort || significantLen(nk) >= minKeyLenFor(nk) +} + // windowsOverlap reports whether two spoiler windows share any chapter. since_ch 0 means // "from chapter 1"; until_ch 0 means "no end". func windowsOverlap(since1, until1, since2, until2 int) bool { @@ -252,6 +360,90 @@ func rubyToCandidates(readings []store.RubyReading, manualSrcs map[string]bool) return out } +// attachRubyAliasesToManual makes the kana READING of a manually-seeded name matchable +// (D16.4): a manual term 鈴木 that appears in a kana-written chunk as すずき would otherwise +// miss — rubyToCandidates SKIPS manual bases, so the reading is discarded, a "тихо пусто" +// recall hole on the ja acceptance book. For each name-shape reading (all-Han base + +// all-kana reading — the furigana-name form classifyRubyReading calls rubyClassName) whose +// base is a manual src, it appends the reading as an ALIAS of that manual entry IN PLACE, +// deduped. The reading then fires with the ordinary alias disposition — AMBIGUOUS when +// short/collision-prone (post-checked), so a possible double-reading (強敵→とも "friend") is +// never blindly trusted; full offline disambiguation is B6. Auto-candidates with no dst stay +// unmatchable (deliberate). Deterministic: readings are processed sorted, and GlossaryForBook +// re-sorts aliases on read, so the materialized bank is stable regardless of input order. +func attachRubyAliasesToManual(entries []store.GlossaryEntry, readings []store.RubyReading) (skipped []string) { + idx := map[string][]int{} + for i := range entries { + idx[entries[i].Src] = append(idx[entries[i].Src], i) + } + // owners of each FIRING key (src + eligible aliases) → term indices, for collision detection. + // A ruby reading is auto-derived (not curated), so if attaching it would collide with a + // DIFFERENT term's firing key with a different dst (two homophone names 鈴木/鈴樹 both read + // すずき), attaching would make approvedSharedKeyCollisions fail the whole book loud on an + // alias the operator cannot edit out of the seed. Instead SKIP it gracefully and LOG (not + // silent): the kana form of that homophone stays unmatchable until the seed disambiguates it + // explicitly (a curated manual alias — whose collisions DO fail loud, as an authoring error). + owners := map[string][]int{} + for i, e := range entries { + for _, k := range entryFiringKeys(e) { + owners[k] = append(owners[k], i) + } + } + collides := func(rk string, ti int) bool { + for _, oi := range owners[rk] { + if oi != ti && entries[oi].Dst != entries[ti].Dst { + return true + } + } + return false + } + sorted := append([]store.RubyReading(nil), readings...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Base != sorted[j].Base { + return sorted[i].Base < sorted[j].Base + } + return sorted[i].Reading < sorted[j].Reading + }) + for _, rr := range sorted { + targets, ok := idx[rr.Base] + if !ok { + continue // the reading's base is not a manual term + } + if classifyRubyReading(rr.Base, rr.Reading) != rubyClassName { + continue // only the all-kana furigana-NAME shape is a match surface; a kanji-bearing gloss/double-reading is a footnote, not an alias + } + rk := normalizeSourceKey(rr.Reading) + for _, ti := range targets { + e := &entries[ti] + if rr.Reading == e.Src || hasAliasSurface(e.Aliases, rr.Reading) { + continue + } + // Only an ELIGIBLE reading (passes the min-key floor or allow_short) can actually + // fire and thus collide; an inert short reading is attached freely. + eligible := rk != "" && (significantLen(rk) >= minKeyLenFor(rk) || e.AllowShort) + if eligible && collides(rk, ti) { + skipped = append(skipped, fmt.Sprintf("%q reading %q (homophone of another seeded term — kana left unmatchable; disambiguate in the seed if needed)", rr.Base, rr.Reading)) + continue + } + e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: rr.Reading, AliasType: "чтение"}) + if eligible { + owners[rk] = append(owners[rk], ti) // now this term owns rk → a later homophone collides and is skipped, not double-attached + } + } + } + return skipped +} + +// hasAliasSurface reports whether an alias with the given raw surface already exists on a term. +func hasAliasSurface(aliases []store.GlossaryAlias, surface string) bool { + for _, a := range aliases { + if a.Alias == surface { + return true + } + } + return false +} + // ruby classifier classes (deterministic HINTS for human promotion). const ( rubyClassName = "name_candidate" // all-Han base + all-kana reading: the furigana-name SHAPE diff --git a/backend/internal/pipeline/memseed_test.go b/backend/internal/pipeline/memseed_test.go index 19ab03c..25ac737 100644 --- a/backend/internal/pipeline/memseed_test.go +++ b/backend/internal/pipeline/memseed_test.go @@ -94,13 +94,50 @@ terms: `)); err == nil { t.Error("duplicate (src,sense,window) term must fail loud") } - // Same src, different SENSE → allowed (polysemy). + // Same src, different SENSE on a SINGLE-CHAR Han src (道): the src is A3-banned + // (significantLen 1 < min-key 2), so it never enters the automaton → its polysemy is + // inert and cannot livelock → still allowed (D16.1 fail-loud is scoped to a matchable src). if _, err := loadGlossarySeed(writeSeed(t, ` terms: - { src: 道, dst: путь, sense: way } - { src: 道, dst: дао, sense: dao } `)); err != nil { - t.Errorf("distinct senses should be allowed: %v", err) + t.Errorf("polysemy on a single-key-banned src is inert and must stay allowed: %v", err) + } +} + +// TestSeedPolysemyOverlapFailsLoud covers D16.1: two approved rows with the same MATCHABLE +// src, DIFFERENT senses, DIFFERENT dst and OVERLAPPING spoiler windows both inject as +// CONFIRMED (the matcher keys only on src) and the post-check is guaranteed to miss one — +// with the gate ON a deterministic chunk livelock. Fail loud. Reverting the different-sense +// branch in loadGlossarySeed passes the contradiction through and fails here. +func TestSeedPolysemyOverlapFailsLoud(t *testing.T) { + // Matchable src (神通, 2 Han ≥ min-key), different senses, default (overlapping) windows, + // different dst → livelock → fail loud. + if _, err := loadGlossarySeed(writeSeed(t, ` +terms: + - { src: 神通, dst: сила, sense: power } + - { src: 神通, dst: чудо, sense: miracle } +`)); err == nil { + t.Error("D16.1: polysemy on a matchable src with overlapping windows must fail loud") + } + // Same src+senses but NON-overlapping windows is a legitimate spoiler-boundary rendering + // change (only one sense can fire per chapter) → allowed. + if _, err := loadGlossarySeed(writeSeed(t, ` +terms: + - { src: 神通, dst: сила, sense: power, until_ch: 5 } + - { src: 神通, dst: чудо, sense: miracle, since_ch: 6 } +`)); err != nil { + t.Errorf("polysemy with non-overlapping windows should be allowed: %v", err) + } + // Same MATCHABLE src+sense with the SAME dst and overlapping windows → NOT a contradiction + // (both render identically), allowed. + if _, err := loadGlossarySeed(writeSeed(t, ` +terms: + - { src: 神通, dst: сила, sense: power, until_ch: 5 } + - { src: 神通, dst: сила, sense: might, since_ch: 3 } +`)); err != nil { + t.Errorf("same-dst overlap is not a contradiction: %v", err) } } @@ -194,6 +231,138 @@ func TestRubyToCandidates(t *testing.T) { } } +// TestApprovedSharedKeyCollisions covers the Task-6 re-audit finding: a firing key shared by +// two DIFFERENT approved terms with different dst + overlapping windows is the alias +// generalization of the D16.1 polysemy livelock. Reverting approvedSharedKeyCollisions (or its +// seedGlossary wiring) admits the contradictory injection. Legitimate cases (same dst, +// non-overlapping windows) and the same-src case (D16.1's job) must NOT be flagged here. +func TestApprovedSharedKeyCollisions(t *testing.T) { + al := func(a string) []store.GlossaryAlias { + return []store.GlossaryAlias{{Alias: a, AliasType: "прозвище"}} + } + // Shared alias 老赵 across two different approved terms, different dst, default (overlapping) + // windows → collision (livelock). + e1 := store.GlossaryEntry{Src: "赵甲", Dst: "Чжао Цзя", Status: "approved", Aliases: al("老赵")} + e2 := store.GlossaryEntry{Src: "赵乙", Dst: "Чжао И", Status: "approved", Aliases: al("老赵")} + if cols := approvedSharedKeyCollisions([]store.GlossaryEntry{e1, e2}); len(cols) == 0 { + t.Error("shared alias 老赵 across different terms must be flagged (livelock)") + } + // Same dst → a legitimate merge (both render identically), no contradiction. + e2same := e2 + e2same.Dst = "Чжао Цзя" + if cols := approvedSharedKeyCollisions([]store.GlossaryEntry{e1, e2same}); len(cols) != 0 { + t.Errorf("same-dst shared surface is a merge, not a collision: %v", cols) + } + // Non-overlapping windows → a spoiler-boundary handoff (only one fires per chapter) → allowed. + e1w, e2w := e1, e2 + e1w.UntilCh, e2w.SinceCh = 5, 6 + if cols := approvedSharedKeyCollisions([]store.GlossaryEntry{e1w, e2w}); len(cols) != 0 { + t.Errorf("non-overlapping windows are allowed: %v", cols) + } + // Self-review #1: a SUB-FLOOR src (道, single-key-banned) that shares an ELIGIBLE alias + // (大道) between different-dst rows slips loadGlossarySeed's src-scoped D16.1 check, so this + // guard must catch it via the shared alias key (livelock). + s1 := store.GlossaryEntry{Src: "道", Dst: "путь", Sense: "way", Status: "approved", Aliases: al("大道")} + s2 := store.GlossaryEntry{Src: "道", Dst: "дао", Sense: "dao", Status: "approved", Aliases: al("大道")} + if cols := approvedSharedKeyCollisions([]store.GlossaryEntry{s1, s2}); len(cols) == 0 { + t.Error("#1: sub-floor src sharing an eligible alias with a different dst must be flagged") + } + // The same sub-floor src with DISTINCT aliases is fine (道 does not fire, no shared key). + d1 := store.GlossaryEntry{Src: "道", Dst: "путь", Sense: "way", Status: "approved", Aliases: al("正道")} + d2 := store.GlossaryEntry{Src: "道", Dst: "дао", Sense: "dao", Status: "approved", Aliases: al("大道")} + if cols := approvedSharedKeyCollisions([]store.GlossaryEntry{d1, d2}); len(cols) != 0 { + t.Errorf("sub-floor src with distinct aliases has no shared firing key: %v", cols) + } + // A same-src pair with an ELIGIBLE shared src key IS a real collision in isolation (in the + // live flow loadGlossarySeed's D16.1 check fails loud first, so no double report). + p1 := store.GlossaryEntry{Src: "神通", Dst: "сила", Sense: "a", Status: "approved"} + p2 := store.GlossaryEntry{Src: "神通", Dst: "чудо", Sense: "b", Status: "approved"} + if cols := approvedSharedKeyCollisions([]store.GlossaryEntry{p1, p2}); len(cols) == 0 { + t.Error("an eligible shared src key is a real collision here too") + } + // An auto term sharing the key is not CONFIRMED, so no authoritative contradiction → allowed. + a1 := store.GlossaryEntry{Src: "钱甲", Dst: "Цянь А", Status: "approved", Aliases: al("老钱")} + a2 := store.GlossaryEntry{Src: "钱乙", Dst: "Цянь Б", Status: "auto", Aliases: al("老钱")} + if cols := approvedSharedKeyCollisions([]store.GlossaryEntry{a1, a2}); len(cols) != 0 { + t.Errorf("an auto term sharing a key is not a CONFIRMED contradiction: %v", cols) + } +} + +// TestRubyReadingBecomesManualAlias covers D16.4: a manual term's name-shape ruby reading +// (all-Han base + all-kana reading) becomes a matchable ALIAS so the kana spelling of a +// seeded name is found. A kanji-bearing gloss/double-reading is NOT attached (footnote, not +// a surface), and a reading whose base is not a manual term is left to the auto-candidate +// path. Reverting attachRubyAliasesToManual leaves the manual entry alias-less and fails here. +func TestRubyReadingBecomesManualAlias(t *testing.T) { + entries := []store.GlossaryEntry{ + {BookID: "b", Src: "鈴木", Dst: "Судзуки", Status: "approved", Source: "seed"}, + {BookID: "b", Src: "泣蟲", Dst: "плакса", Status: "approved", Source: "seed"}, + } + readings := []store.RubyReading{ + {BookID: "b", Base: "鈴木", Reading: "すずき", FirstChapter: 1, Occurrences: 5}, // name-shape → alias + {BookID: "b", Base: "泣蟲", Reading: "泣き虫", FirstChapter: 1, Occurrences: 2}, // gloss (carries kanji) → NOT an alias + {BookID: "b", Base: "田中", Reading: "たなか", FirstChapter: 1, Occurrences: 3}, // base not manual → skipped (auto path) + } + attachRubyAliasesToManual(entries, readings) + + if !hasAliasSurface(entries[0].Aliases, "すずき") { + t.Errorf("鈴木 did not gain its kana reading as an alias: %+v", entries[0].Aliases) + } + if hasAliasSurface(entries[1].Aliases, "泣き虫") { + t.Errorf("a kanji-bearing gloss reading must NOT become an alias: %+v", entries[1].Aliases) + } + // Idempotent: a second attach must not duplicate the alias. + attachRubyAliasesToManual(entries, readings) + n := 0 + for _, a := range entries[0].Aliases { + if a.Alias == "すずき" { + n++ + } + } + if n != 1 { + t.Errorf("ruby-alias attach not idempotent: すずき appears %d times", n) + } + // End-to-end: the kana spelling now matches through the materialized bank. + b := materializeMemory(entries, false) + if _, ok := injMap(b.Select("きのう、すずきに会った。", 1, nil, 0))["鈴木"]; !ok { + t.Error("kana spelling すずき did not match the manual 鈴木 via the reading alias") + } +} + +// TestRubyHomophoneSkippedNotFailLoud covers the self-review interaction: two homophone seeded +// names (鈴木/鈴樹 both read すずき, different dst) must NOT both auto-get the kana alias — that +// would make approvedSharedKeyCollisions fail the whole book loud on an alias the operator +// cannot edit out. The colliding reading is skipped+reported (graceful), and the resulting +// entry set passes the shared-key collision check. +func TestRubyHomophoneSkippedNotFailLoud(t *testing.T) { + entries := []store.GlossaryEntry{ + {BookID: "b", Src: "鈴木", Dst: "Судзуки", Status: "approved", Source: "seed"}, + {BookID: "b", Src: "鈴樹", Dst: "Судзуки-другой", Status: "approved", Source: "seed"}, + } + readings := []store.RubyReading{ + {BookID: "b", Base: "鈴木", Reading: "すずき", FirstChapter: 1, Occurrences: 8}, + {BookID: "b", Base: "鈴樹", Reading: "すずき", FirstChapter: 3, Occurrences: 4}, // homophone → collision + } + skipped := attachRubyAliasesToManual(entries, readings) + if len(skipped) != 1 { + t.Fatalf("want exactly one skipped homophone reading, got %d: %v", len(skipped), skipped) + } + // Exactly ONE term got the kana alias (the first, deterministically); the other was skipped. + got := 0 + for _, e := range entries { + if hasAliasSurface(e.Aliases, "すずき") { + got++ + } + } + if got != 1 { + t.Fatalf("want the kana alias on exactly one term, got %d", got) + } + // The resulting set must NOT trip the shared-key collision fail-loud (the whole point). + if cols := approvedSharedKeyCollisions(entries); len(cols) != 0 { + t.Errorf("homophone skip must avoid the fail-loud, got collisions: %v", cols) + } +} + // TestSeedSurroundingWhitespaceTrimmed covers the workflow-confirmed seed major: surrounding // whitespace on a keying surface passed the emptiness check but was stored raw, so the term // keyed WITH the space and could never match (silently inert), and a trailing-space sense diff --git a/backend/internal/pipeline/runner.go b/backend/internal/pipeline/runner.go index edda6eb..d258464 100644 --- a/backend/internal/pipeline/runner.go +++ b/backend/internal/pipeline/runner.go @@ -153,14 +153,21 @@ type contextSnap struct { } // coverageSnap freezes the excision coverage-gate config inside the snapshot, so -// enabling the gate OR tuning its thresholds is a loud --resnapshot that re-derives -// every chunk's verdict from its checkpoint for free (checkpoint-hit → re-classify, -// no re-bill), never a silent mismatch between an old checkpoint's stored disposition -// and a changed gate. Only folded when ENABLED: tweaking a disabled gate's bounds must -// not force a re-pin (the gate produces no verdicts while off). json.Marshal sorts the -// LenRatio map keys → deterministic render. Mirrors the estimator/max_tokens-policy -// discipline (render.go) for a NON-wire input that still determines a checkpoint's -// resolved verdict. +// enabling the gate OR tuning its thresholds is a loud --resnapshot, never a silent +// mismatch between an old checkpoint's stored disposition and a changed gate. HONEST +// COST (D15): the snapshotID is folded into every call's RequestHash (render.go), so a +// --resnapshot changes EVERY request_hash → every checkpoint misses → the whole book is +// RE-BILLED, even for byte-identical wire requests whose verdict merely needs +// re-classifying. There is no free "re-classify from the checkpoint" here: the +// content-addressed reuse that COULD make an unchanged wire request free (msgsContentHash, +// render.go) is gated on an UNCHANGED snapshot (runner.go), so it never fires across a +// resnapshot. This all-or-nothing re-pay is ACCEPTED for the static acceptance book (the +// gate is loud, divergences do not occur) but is exactly why content-addressed checkpoint +// reuse must be designed before the ongoing/append-chapters mode (D15.2). Only folded when +// ENABLED: tweaking a disabled gate's bounds must not force a re-pin (the gate produces no +// verdicts while off). json.Marshal sorts the LenRatio map keys → deterministic render. +// Mirrors the estimator/max_tokens-policy discipline (render.go) for a NON-wire input that +// still determines a checkpoint's resolved verdict. type coverageSnap struct { Enabled bool `json:"enabled"` Version string `json:"version,omitempty"` @@ -535,7 +542,7 @@ func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) { // the CURRENT chapter, reset at a chapter boundary (a new chapter is a scene change). // Rebuilt deterministically each run because translateChunk recomputes the ($0) // selection for EVERY chunk, resumed ones included — so sticky is resume-stable. - var stickyWin []map[string]bool + var stickyWin []map[string]injectionDisposition prevChapter := 0 for _, ch := range chunks { if ch.Chapter != prevChapter { @@ -562,15 +569,18 @@ func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) { return res, nil } -// unionSticky merges the recent chunks' exact-matched id sets into one sticky_prev. -func unionSticky(win []map[string]bool) map[string]bool { +// unionSticky merges the recent chunks' exact-matched ids into one sticky_prev, carrying +// each id's DISPOSITION (D16.2). When an id fired in several recent chunks, the MOST RECENT +// firing wins (win is ordered oldest→newest), so a re-established CONFIRMED match overrides +// an older collision-prone AMBIGUOUS one — and vice-versa, never silently upgrading. +func unionSticky(win []map[string]injectionDisposition) map[string]injectionDisposition { if len(win) == 0 { return nil } - out := map[string]bool{} + out := map[string]injectionDisposition{} for _, s := range win { - for id := range s { - out[id] = true + for id, disp := range s { + out[id] = disp } } return out @@ -599,10 +609,24 @@ func (r *Runner) seedGlossary(ctx context.Context) error { if err != nil { return err } + // D16.4: attach a manual term's kana ruby-reading as an alias (kana spelling matchable) + // BEFORE appending the auto-candidates, so it only touches the curated manual entries. A + // reading that would collide with a different seeded term (homophone) is skipped+logged, + // not attached (which would fail the book loud on an alias the operator cannot edit out). + if skipped := attachRubyAliasesToManual(entries, ruby); len(skipped) > 0 { + r.Log.WarnContext(ctx, "ruby kana-alias skipped as a homophone collision (kana form left unmatchable; disambiguate in the seed if needed)", + "skipped", strings.Join(skipped, "; ")) + } entries = append(entries, rubyToCandidates(ruby, manualSrcs)...) for i := range entries { entries[i].BookID = r.Book.BookID } + // Task-6 re-audit: fail loud on a firing key shared by two DIFFERENT approved terms with + // different dst + overlapping windows (the alias generalization of the D16.1 polysemy + // livelock) — checked over the FULL entry set (incl. ruby-attached aliases) before persisting. + if cols := approvedSharedKeyCollisions(entries); len(cols) > 0 { + return fmt.Errorf("pipeline: glossary shared-key collisions (A2 / D16.1 livelock class):\n - %s", strings.Join(cols, "\n - ")) + } if err := r.Store.ReplaceGlossary(r.Book.BookID, entries); err != nil { return err } @@ -675,7 +699,7 @@ func (r *Runner) persistRuby(readings []RubyReading) error { // persists the per-chunk retrieval-state (observability). Returns the chunk outcome, the // exact-matched entity ids (the next chunk's sticky_prev, A5), and an error only on an // infra failure. stickyPrev is the prior chunks' exact matches in this chapter. -func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, stickyPrev map[string]bool) (*ChunkOutcome, map[string]bool, error) { +func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, stickyPrev map[string]injectionDisposition) (*ChunkOutcome, map[string]injectionDisposition, error) { out := &ChunkOutcome{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: DispOK} prev := "" flagged := false @@ -686,7 +710,7 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st // included) so the sticky window and the injected bytes are resume-stable. var memSel memorySelection var translatorInjection, editorInjection string - activeIDs := map[string]bool{} + activeIDs := map[string]injectionDisposition{} if r.memory != nil { memSel = r.memory.Select(ch.Text, ch.Chapter, stickyPrev, r.Pipeline.Context.GlossaryTokenBudget) translatorInjection = renderGlossaryBlock(memSel.injected) @@ -999,6 +1023,7 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: st.Name, SnapshotID: snapID, ContentHash: contentHash, Disposition: string(disposition), FlagReason: string(last.cls.Reason), Attempts: attemptsMade, FinalHash: finalHash, CostUSD: cumCost, Detail: last.cls.Detail, + Escalated: escalated, EscalationModel: escModel, }); err != nil { return nil, fmt.Errorf("pipeline: record chunk_status: %w", err) } @@ -1084,7 +1109,11 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID for _, m := range msgs { promptEst += EstimateTokens(m.Content) } - estimate := ledger.EstimateUSD(price, promptEst, maxTokens) + // D13.6: reserve the additive reasoning buffer for the ACTUAL model being called (the + // escalate_to fallback may differ from st.Model and have its own provider). 0 for + // subset providers / reasoning-off — the ceiling then sees only completion, as before. + reasoningBudget := r.Models.AdditiveReasoningTokens(model, st.Reasoning, st.ReasoningMaxTokens) + estimate := ledger.EstimateUSD(price, promptEst, maxTokens, reasoningBudget) resv, verdict, err := r.Store.Reserve(r.Book.BookID, estimate, store.Ceilings{ BookUSD: r.Book.Ceilings.BookUSD, DayUSD: r.Book.Ceilings.DayUSD, @@ -1241,6 +1270,10 @@ func (r *Runner) resumeFromChunkStatus(ctx context.Context, st config.Stage, ch CostUSD: 0, CumCostUSD: cs.CostUSD, Disposition: Disposition(cs.Disposition), FlagReason: FlagReason(cs.FlagReason), Detail: cs.Detail, Attempts: cs.Attempts, + // Escalation attribution now travels on chunk_status (D15.3), so it is preserved + // across resume for BOTH an ok escalation AND a flagged-then-escalated chunk (the + // status read-model reads it without re-opening the checkpoint). + Escalated: cs.Escalated, EscalationModel: cs.EscalationModel, } if cs.Disposition == string(DispOK) { cp, err := r.Store.GetCheckpoint(cs.FinalHash) @@ -1260,12 +1293,6 @@ func (r *Runner) resumeFromChunkStatus(ctx context.Context, st config.Stage, ch sr.Model = cp.ModelActual sr.FinishReason = cp.FinishReason sr.Text = cp.ResponseText - // Preserve escalation attribution across resume (self-review): a chunk resolved - // via a fallback checkpoint is still an escalation, so a resumed run reports it. - if cp.Escalation { - sr.Escalated = true - sr.EscalationModel = cp.ModelRequested - } } r.Store.LogRequest(ctx, r.Log, store.RequestLog{ BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, diff --git a/backend/internal/pipeline/status.go b/backend/internal/pipeline/status.go new file mode 100644 index 0000000..9fca63b --- /dev/null +++ b/backend/internal/pipeline/status.go @@ -0,0 +1,430 @@ +package pipeline + +import ( + "context" + "fmt" + "sort" + + "textmachine/backend/internal/store" +) + +// status.go: the READ-ONLY progress projection (`tmctl status`) and the targeted re-attack +// of flagged chunks (`tmctl redrive`) — the D12 "прогресс — единственный реальный пробел" +// tail, ratified by D15.3. Status makes ZERO LLM calls and ZERO checkpoint replays: it reads +// the book manifest (a deterministic, $0 re-chunk of the source) and projects it against the +// stored chunk_status / spend / retrieval_state. Redrive resets the flagged chunks' terminal +// state (their chunk_status + checkpoints) and re-runs the durable loop, so the DispOK work +// resumes at $0 and only the reset stages re-attack with a fresh retry/escalation budget. + +// ChunkState is a chunk's resolved lifecycle state for the projection — a chunk-level view +// over its per-stage chunk_status rows. `skipped` is deliberately NOT a chunk state: it is a +// per-STAGE substate (the downstream stages of a flagged chunk), surfaced in the passport's +// stage counts, since the linear C1 core never skips a whole chunk. +type ChunkState string + +const ( + ChunkDone ChunkState = "done" // every pipeline stage resolved ok + ChunkFlagged ChunkState = "flagged" // a stage flagged (chunk not translated; ≠ infra failure) + ChunkInProgress ChunkState = "in_progress" // some stages resolved, not all, none flagged (an interrupted run) + ChunkPending ChunkState = "pending" // no stage attempted yet +) + +// ChapterPassport is the per-chapter quality passport (D12): unit counts, the worst flag +// reason, a pass|attention|fail verdict (exp07 chapter rule) and the chapter's spend. +type ChapterPassport struct { + Chapter int `json:"chapter"` + ChunksTotal int `json:"chunks_total"` + ChunksDone int `json:"chunks_done"` + ChunksFlagged int `json:"chunks_flagged"` + ChunksInFlight int `json:"chunks_in_progress"` + ChunksPending int `json:"chunks_pending"` + StagesSkipped int `json:"stages_skipped"` // per-stage skip count (downstream of a flag) + Escalations int `json:"escalations"` + PostcheckMisses int `json:"postcheck_misses"` // confirmed post-check misses (retrieval_state) + WorstFlagReason string `json:"worst_flag_reason,omitempty"` + Verdict string `json:"verdict"` // pass | attention | fail (exp07: 0/1/≥2 flagged chunks) + CostUSD float64 `json:"cost_usd"` +} + +// StatusReport is the whole-book read-model. +type StatusReport struct { + BookID string `json:"book_id"` + Snapshot string `json:"snapshot_id"` // the snapshot the stored rows were resolved under ("" = nothing run yet) + // SnapshotDrift is true when the stored rows carry more than one snapshot id (a config + // changed mid-book without a full --resnapshot re-pin) — a loud signal, not a silent skew. + SnapshotDrift bool `json:"snapshot_drift"` + // ConfigDrift is true when the CURRENT config renders a snapshot DIFFERENT from the one the + // stored rows were resolved under — a wire/verdict-affecting edit since the last run (e.g. a + // prompt-version bump, a gate flip) that `tmctl translate` would --resnapshot. Without this + // the operator sees "done/pass" and false confidence (finding #3). CurrentSnapshot is what + // the config renders now (empty when it could not be computed or matches). + ConfigDrift bool `json:"config_drift"` + CurrentSnapshot string `json:"current_snapshot,omitempty"` + + TotalChunks int `json:"total_chunks"` + Done int `json:"done"` + InProgress int `json:"in_progress"` + Flagged int `json:"flagged"` + Pending int `json:"pending"` + PercentDone float64 `json:"percent_done"` // 100·done/total, unit-count only (no synthetic time-bar) + + Escalations int `json:"escalations"` + PostcheckMisses int `json:"postcheck_misses"` + + CommittedUSD float64 `json:"committed_usd"` + ReservedUSD float64 `json:"reserved_usd"` + BookCeilingUSD float64 `json:"book_ceiling_usd,omitempty"` + CeilingPct float64 `json:"ceiling_pct,omitempty"` // 100·(committed+reserved)/book_ceiling + ProjectedBookUSD float64 `json:"projected_book_usd"` // committed extrapolated over the whole book + ETASeconds float64 `json:"eta_seconds,omitempty"` // secondary: mean fresh-call throughput × remaining + + Chapters []ChapterPassport `json:"chapters"` +} + +// flagReasonSeverity ranks flag reasons worst-first for the chapter's "worst flag" passport +// field. Lower = worse. Deterministic content-failures (refusal/echo/excision) outrank +// retryable budget symptoms (length/empty), which are the least alarming. +func flagReasonSeverity(reason string) int { + switch FlagReason(reason) { + case FlagHardRefusal, FlagSoftRefusal, FlagContentFilter, FlagHardBlock: + return 0 + case FlagCJKArtifact, FlagExcisionSuspect, FlagCoverageFail: + return 1 + case FlagLoopDegenerate: + return 2 + case FlagDecodeError: + return 3 + case FlagGlossaryMiss: + return 4 + case FlagLength, FlagEmpty: + return 5 + } + return 6 +} + +// bookChunks re-derives the book's chunk manifest — Ingest + SplitChunks — for the honest N/M +// denominator. Pure and $0 (no LLM); unlike TranslateBook it does NOT persist ruby or seed the +// glossary, so status stays a pure read. A source edit or a chunker-version change is reflected +// here (the manifest is CURRENT), which is exactly what makes a snapshot drift visible. +func (r *Runner) bookChunks() ([]Chunk, error) { + doc, err := Ingest(r.Book.SourceFile) + if err != nil { + return nil, err + } + return SplitChunks(doc.Chapters), nil +} + +// chunkKey identifies a chunk positionally. +type chunkKey struct { + chapter, chunkIdx int +} + +// resolveChunkState maps a chunk's per-stage rows to its chunk-level state, plus its cost, +// escalation flag and (when flagged) the flag reason. stagesTotal is the pipeline stage count. +func resolveChunkState(rows []store.ChunkStatus, stagesTotal int) (st ChunkState, reason string, cost float64, escalated bool, skipped int) { + if len(rows) == 0 { + return ChunkPending, "", 0, false, 0 + } + ok := 0 + for _, cs := range rows { + cost += cs.CostUSD + if cs.Escalated { + escalated = true + } + switch cs.Disposition { + case string(DispOK): + ok++ + case string(DispFlagged): + // The first (only) flagged stage decides the chunk; keep its reason. + st, reason = ChunkFlagged, cs.FlagReason + case string(DispSkipped): + skipped++ + } + } + if st == ChunkFlagged { + return st, reason, cost, escalated, skipped + } + if ok == stagesTotal { + return ChunkDone, "", cost, escalated, skipped + } + return ChunkInProgress, "", cost, escalated, skipped +} + +// Status builds the read-only progress projection. It opens no jobs, reserves nothing, makes +// no LLM call and replays no checkpoint — the D12 query-handler analogue. Safe to run whenever +// the project file is not held by a running tmctl (the same exclusive-lock rule as `report`). +func (r *Runner) Status(ctx context.Context) (*StatusReport, error) { + chunks, err := r.bookChunks() + if err != nil { + return nil, err + } + statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID) + if err != nil { + return nil, err + } + byChunk := map[chunkKey][]store.ChunkStatus{} + snapSeen := map[string]bool{} + for _, cs := range statuses { + byChunk[chunkKey{cs.Chapter, cs.ChunkIdx}] = append(byChunk[chunkKey{cs.Chapter, cs.ChunkIdx}], cs) + if cs.SnapshotID != "" { + snapSeen[cs.SnapshotID] = true + } + } + + // Per-chunk post-check misses (retrieval_state) — the glossary-consistency signal that is + // re-derived each run and NOT a chunk_status disposition (default flagger mode). + states, err := r.Store.RetrievalStatesForBook(r.Book.BookID) + if err != nil { + return nil, err + } + missByChunk := map[chunkKey]int{} + for _, rs := range states { + missByChunk[chunkKey{rs.Chapter, rs.ChunkIdx}] = rs.NPostcheckMiss + } + + stagesTotal := len(r.Pipeline.Stages) + // The post-check GATE (opt-in) flags a chunk at the CHUNK level AFTER the stage loop, so it + // is NEVER written as a chunk_status row (every stage stays DispOK). Without accounting for + // it here, status would report a gate-flagged chunk as done/pass while `tmctl translate` + // exits 2 (finding #2). When the gate is ON, promote an otherwise-done chunk with a CONFIRMED + // post-check miss to flagged/glossary_miss so status matches the run. + gateOn := r.Pipeline.Gates.Glossary.PostcheckGate + rep := &StatusReport{BookID: r.Book.BookID, TotalChunks: len(chunks)} + passports := map[int]*ChapterPassport{} + var chapterOrder []int + var processedCost float64 // spend of PROCESSED chunks (done+flagged) — the projection base (finding #7) + + for _, ch := range chunks { + p := passports[ch.Chapter] + if p == nil { + p = &ChapterPassport{Chapter: ch.Chapter, Verdict: "pass"} + passports[ch.Chapter] = p + chapterOrder = append(chapterOrder, ch.Chapter) + } + p.ChunksTotal++ + key := chunkKey{ch.Chapter, ch.ChunkIdx} + state, reason, cost, escalated, skipped := resolveChunkState(byChunk[key], stagesTotal) + p.CostUSD += cost + p.StagesSkipped += skipped + miss := missByChunk[key] + p.PostcheckMisses += miss + rep.PostcheckMisses += miss + if gateOn && state == ChunkDone && miss > 0 { + // Matches translateChunk: the gate flags only an otherwise-ok chunk. Not re-drivable + // (the miss re-derives from the unchanged glossary; a fix is a glossary edit = + // --resnapshot, not a redrive). + state, reason = ChunkFlagged, string(FlagGlossaryMiss) + } + if escalated { + p.Escalations++ + rep.Escalations++ + } + switch state { + case ChunkDone: + rep.Done++ + p.ChunksDone++ + case ChunkFlagged: + rep.Flagged++ + p.ChunksFlagged++ + if p.WorstFlagReason == "" || flagReasonSeverity(reason) < flagReasonSeverity(p.WorstFlagReason) { + p.WorstFlagReason = reason + } + case ChunkInProgress: + rep.InProgress++ + p.ChunksInFlight++ + case ChunkPending: + rep.Pending++ + p.ChunksPending++ + } + if state == ChunkDone || state == ChunkFlagged { + processedCost += cost // a fully-attempted chunk's cost feeds the projection + } + } + + // Chapter verdicts (exp07 chapter rule: 0 flagged = pass, 1 = attention, ≥2 = fail). + sort.Ints(chapterOrder) + for _, n := range chapterOrder { + p := passports[n] + switch { + case p.ChunksFlagged >= 2: + p.Verdict = "fail" + case p.ChunksFlagged == 1: + p.Verdict = "attention" + default: + p.Verdict = "pass" + } + rep.Chapters = append(rep.Chapters, *p) + } + + if rep.TotalChunks > 0 { + rep.PercentDone = 100 * float64(rep.Done) / float64(rep.TotalChunks) + } + if len(snapSeen) == 1 { + for s := range snapSeen { + rep.Snapshot = s + } + } else if len(snapSeen) > 1 { + rep.SnapshotDrift = true + } + + // Config-drift (finding #3): compute the CURRENT config's snapshot READ-ONLY (materialize + // the STORED glossary — no re-seed, no write) and compare to the single snapshot the stored + // rows carry. A wire/verdict-affecting config edit since the last run (a prompt bump, a gate + // flip — this very package bumps the editor prompt_version) is otherwise hidden behind + // "done/pass" until translate forces a --resnapshot. Only meaningful when a single snapshot + // is on record (multi-snapshot is already flagged as SnapshotDrift). Note: a change to the + // seed FILE that was not re-run is NOT detected here (the stored glossary is what status + // projects); it surfaces on the next translate's re-seed. + if rep.Snapshot != "" { + if rows, gerr := r.Store.GlossaryForBook(r.Book.BookID); gerr == nil { + r.memory = materializeMemory(rows, gateOn) + if curSnap, _, serr := r.snapshotID(); serr == nil && curSnap != rep.Snapshot { + rep.ConfigDrift = true + rep.CurrentSnapshot = curSnap + } + } + } + + // Money. + committed, reserved, err := r.Store.SpentUSD(r.Book.BookID) + if err != nil { + return nil, err + } + rep.CommittedUSD, rep.ReservedUSD = committed, reserved + rep.BookCeilingUSD = r.Book.Ceilings.BookUSD + if rep.BookCeilingUSD > 0 { + rep.CeilingPct = 100 * (committed + reserved) / rep.BookCeilingUSD + } + // Projected book cost: extrapolate the per-PROCESSED-chunk average over the whole book + // (done + flagged = a chunk fully attempted). Uses processedCost, NOT book committed: + // committed also carries partial spend on IN-PROGRESS chunks (excluded from the denominator), + // which would over-estimate (finding #7). The clean average × total is the honest estimate. + processed := rep.Done + rep.Flagged + if processed > 0 { + rep.ProjectedBookUSD = processedCost / float64(processed) * float64(rep.TotalChunks) + } + + // ETA (secondary): mean fresh-call throughput × remaining processing. No synthetic bar. + totalMS, freshCalls, err := r.Store.FreshCallLatencyMS(r.Book.BookID) + if err != nil { + return nil, err + } + remaining := rep.TotalChunks - processed + if processed > 0 && freshCalls > 0 && remaining > 0 { + meanMSPerChunk := float64(totalMS) / float64(processed) + rep.ETASeconds = meanMSPerChunk * float64(remaining) / 1000 + } + return rep, nil +} + +// RedriveSelector picks the flagged chunks to re-attack. -1 on Chapter/ChunkIdx means "any" +// (a valid chunk index is ≥0, a chapter ≥1, so -1 is an unambiguous "unset"); an empty Reason +// matches any flag_reason. DryRun reports what WOULD be reset without touching anything. +type RedriveSelector struct { + Chapter int + ChunkIdx int + Reason string + DryRun bool +} + +func (s RedriveSelector) matches(cs store.ChunkStatus) bool { + if s.Chapter >= 0 && cs.Chapter != s.Chapter { + return false + } + if s.ChunkIdx >= 0 && cs.ChunkIdx != s.ChunkIdx { + return false + } + if s.Reason != "" && cs.FlagReason != s.Reason { + return false + } + return true +} + +// RedriveTarget is one chunk selected for re-attack and the stages that were (or would be) reset. +type RedriveTarget struct { + Chapter int `json:"chapter"` + ChunkIdx int `json:"chunk_idx"` + FlagReason string `json:"flag_reason"` + Stages []string `json:"reset_stages"` +} + +// RedriveSummary reports the plan/outcome of the reset half of a redrive. +type RedriveSummary struct { + Targets []RedriveTarget `json:"targets"` + DryRun bool `json:"dry_run"` + ResetRun bool `json:"reset_run"` // the re-translate ran (false on dry-run / no targets) +} + +// Redrive re-attacks the FLAGGED chunks matching sel (D15.3 / D12 Step-Functions redrive). It +// NEVER touches DispOK work (D12): for each selected chunk it resets only the flagged stage and +// its downstream skipped stages — their chunk_status + checkpoints are deleted (ResetChunkStages), +// so the durable re-run re-derives them with a FRESH retry/escalation budget (a fresh provider +// call, not a deterministic replay of the flagged completion) while the upstream ok stages resume +// at $0. Money bills normally (reserve/settle+checkpoint); the previously-spent money on the +// discarded attempts stays committed (honest — it was really billed), so the ceilings remain +// accurate. Requires the CURRENT config to render the SAME snapshot the flagged rows carry (no +// --resnapshot): a config change makes the re-run fail loud with the resnapshot guidance instead +// of silently re-pricing the book. Returns the reset plan, and the re-run's BookResult (nil on +// dry-run or when nothing matched). +func (r *Runner) Redrive(ctx context.Context, sel RedriveSelector) (*RedriveSummary, *BookResult, error) { + statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID) + if err != nil { + return nil, nil, err + } + // Group by chunk and find the targets: a chunk with a FLAGGED stage matching the selector. + byChunk := map[chunkKey][]store.ChunkStatus{} + var order []chunkKey + for _, cs := range statuses { + k := chunkKey{cs.Chapter, cs.ChunkIdx} + if _, seen := byChunk[k]; !seen { + order = append(order, k) + } + byChunk[k] = append(byChunk[k], cs) + } + sort.Slice(order, func(i, j int) bool { + if order[i].chapter != order[j].chapter { + return order[i].chapter < order[j].chapter + } + return order[i].chunkIdx < order[j].chunkIdx + }) + + summary := &RedriveSummary{DryRun: sel.DryRun} + for _, k := range order { + rows := byChunk[k] + targeted := false + reason := "" + for _, cs := range rows { + if cs.Disposition == string(DispFlagged) && sel.matches(cs) { + targeted, reason = true, cs.FlagReason + break + } + } + if !targeted { + continue + } + // Reset the flagged stage AND its downstream skipped stages; keep upstream DispOK. + var stages []string + for _, cs := range rows { + if cs.Disposition == string(DispFlagged) || cs.Disposition == string(DispSkipped) { + stages = append(stages, cs.Stage) + } + } + sort.Strings(stages) + summary.Targets = append(summary.Targets, RedriveTarget{ + Chapter: k.chapter, ChunkIdx: k.chunkIdx, FlagReason: reason, Stages: stages, + }) + if !sel.DryRun { + if err := r.Store.ResetChunkStages(r.Book.BookID, k.chapter, k.chunkIdx, stages); err != nil { + return summary, nil, fmt.Errorf("pipeline: redrive reset ch%d/chunk%d: %w", k.chapter, k.chunkIdx, err) + } + } + } + + if sel.DryRun || len(summary.Targets) == 0 { + return summary, nil, nil + } + // Re-run the durable loop: reset chunks re-attack fresh; everything else resumes at $0. + res, err := r.TranslateBook(ctx) + summary.ResetRun = true + return summary, res, err +} diff --git a/backend/internal/pipeline/status_test.go b/backend/internal/pipeline/status_test.go new file mode 100644 index 0000000..725259a --- /dev/null +++ b/backend/internal/pipeline/status_test.go @@ -0,0 +1,262 @@ +package pipeline + +import ( + "context" + "math" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// TestStatusAndRedrive exercises the D15.3 read-model + targeted re-attack end-to-end against +// the mock provider ($0, no real LLM): a book with one flagged chapter and one ok chapter → +// status projects the honest N/M + per-chapter passports + money; redrive dry-run identifies +// the flagged chunk without touching it; a real redrive resets ONLY the flagged stages, re-runs +// with a fresh budget (ok work resumes at $0), and the previously-spent money stays committed. +func TestStatusAndRedrive(t *testing.T) { + rec := &reqRec{} + var mu sync.Mutex + refuse := true // ch1 draft refuses until flipped + srv := newJSONProvider(rec, func(body string) (string, string) { + mu.Lock() + r := refuse + mu.Unlock() + if r && strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) { + return "Извините, я не могу перевести это.", "stop" // soft refusal → flagged, non-retryable + } + return draftEdit(body) + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА", regenerate: 0}) + ctx := context.Background() + + // --- Run 1: ch1 flagged (soft_refusal), ch2 done. --- + r1 := newRunner(t, bookPath) + res1, err := r1.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + if res1.Flagged != 1 { + t.Fatalf("want 1 flagged chunk, got %d", res1.Flagged) + } + callsAfterRun1 := rec.count() // draft(ch1)=refusal, draft(ch2)+edit(ch2)=ok → 3 + if callsAfterRun1 != 3 { + t.Fatalf("want 3 provider calls after run1, got %d", callsAfterRun1) + } + + // --- STATUS projection (read-only). --- + rep, err := r1.Status(ctx) + if err != nil { + t.Fatal(err) + } + if rep.TotalChunks != 2 || rep.Done != 1 || rep.Flagged != 1 || rep.Pending != 0 || rep.InProgress != 0 { + t.Fatalf("status counts wrong: %+v", rep) + } + if math.Abs(rep.PercentDone-50) > 1e-9 { + t.Errorf("percent_done = %v, want 50", rep.PercentDone) + } + if len(rep.Chapters) != 2 { + t.Fatalf("want 2 chapter passports, got %d", len(rep.Chapters)) + } + if rep.Chapters[0].Verdict != "attention" || rep.Chapters[0].WorstFlagReason != string(FlagSoftRefusal) || rep.Chapters[0].ChunksFlagged != 1 { + t.Errorf("ch1 passport = %+v (want attention/soft_refusal/1 flagged)", rep.Chapters[0]) + } + if rep.Chapters[1].Verdict != "pass" || rep.Chapters[1].ChunksDone != 1 { + t.Errorf("ch2 passport = %+v (want pass/1 done)", rep.Chapters[1]) + } + wantCommitted := 3 * fakeCallUSD + if math.Abs(rep.CommittedUSD-wantCommitted) > 1e-9 { + t.Errorf("committed = %v, want %v", rep.CommittedUSD, wantCommitted) + } + // Projected extrapolates the per-processed-chunk average over the whole book (here both + // chunks are processed, so it equals committed). + if math.Abs(rep.ProjectedBookUSD-wantCommitted) > 1e-9 { + t.Errorf("projected = %v, want %v", rep.ProjectedBookUSD, wantCommitted) + } + r1.Close() + if rec.count() != callsAfterRun1 { + t.Fatalf("status made provider calls: %d -> %d", callsAfterRun1, rec.count()) + } + + // --- REDRIVE dry-run: identify the target, mutate nothing. --- + r2 := newRunner(t, bookPath) + sum, res, err := r2.Redrive(ctx, RedriveSelector{Chapter: -1, ChunkIdx: -1, Reason: string(FlagSoftRefusal), DryRun: true}) + if err != nil { + t.Fatal(err) + } + if res != nil || sum.ResetRun { + t.Error("dry-run must not re-run the book") + } + if len(sum.Targets) != 1 || sum.Targets[0].Chapter != 1 || sum.Targets[0].ChunkIdx != 0 { + t.Fatalf("dry-run targets = %+v", sum.Targets) + } + // The flagged stage AND its skipped downstream are both selected for reset. + if strings.Join(sum.Targets[0].Stages, ",") != "draft,edit" { + t.Errorf("dry-run reset stages = %v, want draft,edit", sum.Targets[0].Stages) + } + if cs, _ := r2.Store.GetChunkStatus("test-book", 1, 0, "draft"); cs == nil || cs.Disposition != "flagged" { + t.Fatalf("dry-run mutated chunk_status: %+v", cs) + } + r2.Close() + if rec.count() != callsAfterRun1 { + t.Fatal("dry-run made provider calls") + } + + // --- REDRIVE real: flip the provider to success, reset ch1, re-run. --- + mu.Lock() + refuse = false + mu.Unlock() + r3 := newRunner(t, bookPath) + defer r3.Close() + sum3, res3, err := r3.Redrive(ctx, RedriveSelector{Chapter: 1, ChunkIdx: 0, Reason: ""}) + if err != nil { + t.Fatal(err) + } + if !sum3.ResetRun || res3 == nil { + t.Fatal("real redrive should reset and re-run") + } + if res3.Flagged != 0 { + t.Fatalf("redrive should have fixed the flag, still %d flagged", res3.Flagged) + } + // Exactly two FRESH calls (ch1 draft + ch1 edit); ch2 resumes at $0. + if got := rec.count() - callsAfterRun1; got != 2 { + t.Fatalf("redrive should make exactly 2 fresh calls (ch1 draft+edit), made %d", got) + } + if cs, _ := r3.Store.GetChunkStatus("test-book", 1, 0, "draft"); cs == nil || cs.Disposition != "ok" { + t.Fatalf("redrive did not fix ch1 draft: %+v", cs) + } + // MONEY (documented): the discarded refusal's spend STAYS committed; redrive bills the two + // new calls on top → committed reflects total lifetime spend (5 calls), ceilings stay honest. + committed, _, _ := r3.Store.SpentUSD("test-book") + if math.Abs(committed-5*fakeCallUSD) > 1e-9 { + t.Errorf("committed after redrive = %v, want %v (3 original + 2 redrive)", committed, 5*fakeCallUSD) + } + // A clean status now: everything done. + rep2, err := r3.Status(ctx) + if err != nil { + t.Fatal(err) + } + if rep2.Done != 2 || rep2.Flagged != 0 { + t.Errorf("post-redrive status: done=%d flagged=%d, want 2/0", rep2.Done, rep2.Flagged) + } +} + +// TestStatusSurfacesGlossaryMissGate covers self-review finding #2: the post-check GATE flags a +// chunk at the CHUNK level (never a chunk_status row), so status must promote a gate-on chunk +// with a CONFIRMED post-check miss to flagged/glossary_miss — otherwise it reports done/pass +// while `tmctl translate` exits 2, contradicting the run. +func TestStatusSurfacesGlossaryMissGate(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isEditBody(body) { + return "ОТРЕДАКТИРОВАННЫЙ.", "stop" + } + return "Некто пошёл в библиотеку.", "stop" // drops the approved name → CONFIRMED post-check miss + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true}) + ctx := context.Background() + r := newRunner(t, bookPath) + defer r.Close() + res, err := r.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + if res.Flagged != 1 { + t.Fatalf("translate must flag the glossary_miss chunk: flagged=%d", res.Flagged) + } + rep, err := r.Status(ctx) + if err != nil { + t.Fatal(err) + } + if rep.Flagged != 1 || rep.Done != 0 { + t.Fatalf("status must match translate (surface glossary_miss): done=%d flagged=%d, want 0/1", rep.Done, rep.Flagged) + } + if len(rep.Chapters) != 1 || rep.Chapters[0].WorstFlagReason != string(FlagGlossaryMiss) { + t.Fatalf("passport must show glossary_miss: %+v", rep.Chapters) + } + if rep.Chapters[0].Verdict == "pass" { + t.Error("a chapter with a glossary_miss must not be 'pass'") + } +} + +// TestStatusDetectsConfigDrift covers self-review finding #3: after a wire-affecting config edit +// (a prompt_version bump) since the last run, status must flag ConfigDrift — the current config +// renders a different snapshot than the stored rows, which translate would --resnapshot (re-bill). +// Without it status shows a misleading "done/consistent". +func TestStatusDetectsConfigDrift(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, draftEdit) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0}) + ctx := context.Background() + + r1 := newRunner(t, bookPath) + if _, err := r1.TranslateBook(ctx); err != nil { + t.Fatal(err) + } + r1.Close() + + // No config change → no drift. + r2 := newRunner(t, bookPath) + rep, err := r2.Status(ctx) + if err != nil { + t.Fatal(err) + } + if rep.ConfigDrift { + t.Fatalf("an unchanged config must not drift (current=%s)", rep.CurrentSnapshot) + } + r2.Close() + + // Bump a wire-affecting field (draft prompt_version) → the current config renders a new + // snapshot → drift. + dir := filepath.Dir(bookPath) + body, err := os.ReadFile(filepath.Join(dir, "pipeline.yaml")) + if err != nil { + t.Fatal(err) + } + changed := strings.Replace(string(body), "prompt_version: v-test", "prompt_version: v-test2", 1) + if changed == string(body) { + t.Fatal("setup: prompt_version token not found to change") + } + writeFile(t, filepath.Join(dir, "pipeline.yaml"), changed) + + r3 := newRunner(t, bookPath) + defer r3.Close() + rep3, err := r3.Status(ctx) + if err != nil { + t.Fatal(err) + } + if !rep3.ConfigDrift || rep3.CurrentSnapshot == "" { + t.Errorf("a prompt_version bump since the run must surface as ConfigDrift: %+v", rep3) + } +} + +// TestRedriveNoTargets confirms a selector that matches nothing is a clean no-op (no re-run). +func TestRedriveNoTargets(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, draftEdit) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ГЛАВАОДИН\fГЛАВАДВА"}) + ctx := context.Background() + + r := newRunner(t, bookPath) + defer r.Close() + if _, err := r.TranslateBook(ctx); err != nil { + t.Fatal(err) + } + calls := rec.count() + // Nothing is flagged → redrive finds no targets and never re-runs. + sum, res, err := r.Redrive(ctx, RedriveSelector{Chapter: -1, ChunkIdx: -1}) + if err != nil { + t.Fatal(err) + } + if len(sum.Targets) != 0 || sum.ResetRun || res != nil { + t.Fatalf("redrive with no flagged chunks must be a no-op: %+v", sum) + } + if rec.count() != calls { + t.Fatal("no-target redrive made provider calls") + } +} diff --git a/backend/internal/store/chunkstatus.go b/backend/internal/store/chunkstatus.go index 56b83fc..e5e2b58 100644 --- a/backend/internal/store/chunkstatus.go +++ b/backend/internal/store/chunkstatus.go @@ -19,18 +19,20 @@ import ( // ChunkStatus is one (book, chapter, chunk, stage) disposition row. type ChunkStatus struct { - BookID string - Chapter int - ChunkIdx int - Stage string - SnapshotID string - ContentHash string // signature of the rendered msgs (the source is NOT in the snapshot); guards the resume fast-path against serving a stale translation after a source edit - Disposition string // ok | flagged | skipped - FlagReason string // "" when ok - Attempts int // number of attempts made for this chunk×stage - FinalHash string // request_hash of the authoritative checkpoint (ok path) - CostUSD float64 // sum across all attempts (F3-honest) - Detail string + BookID string + Chapter int + ChunkIdx int + Stage string + SnapshotID string + ContentHash string // signature of the rendered msgs (the source is NOT in the snapshot); guards the resume fast-path against serving a stale translation after a source edit + Disposition string // ok | flagged | skipped + FlagReason string // "" when ok + Attempts int // number of attempts made for this chunk×stage + FinalHash string // request_hash of the authoritative checkpoint (ok path) + CostUSD float64 // sum across all attempts (F3-honest) + Detail string + Escalated bool // a single-hop fallback was used for this chunk×stage (D12/D15.3 telemetry) + EscalationModel string // the fallback model that answered ("" when not escalated) } // UpsertChunkStatus writes (or overwrites) the disposition row. Overwrite is the @@ -42,20 +44,24 @@ func (s *Store) UpsertChunkStatus(cs ChunkStatus) error { _, err := s.w.ExecContext(ctx, ` INSERT INTO chunk_status ( book_id, chapter, chunk_idx, stage, snapshot_id, content_hash, - disposition, flag_reason, attempts, final_hash, cost_usd, detail, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now')) + disposition, flag_reason, attempts, final_hash, cost_usd, detail, + escalated, escalation_model, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now')) ON CONFLICT (book_id, chapter, chunk_idx, stage) DO UPDATE SET - snapshot_id = excluded.snapshot_id, - content_hash = excluded.content_hash, - disposition = excluded.disposition, - flag_reason = excluded.flag_reason, - attempts = excluded.attempts, - final_hash = excluded.final_hash, - cost_usd = excluded.cost_usd, - detail = excluded.detail, - updated_at = excluded.updated_at`, + snapshot_id = excluded.snapshot_id, + content_hash = excluded.content_hash, + disposition = excluded.disposition, + flag_reason = excluded.flag_reason, + attempts = excluded.attempts, + final_hash = excluded.final_hash, + cost_usd = excluded.cost_usd, + detail = excluded.detail, + escalated = excluded.escalated, + escalation_model = excluded.escalation_model, + updated_at = excluded.updated_at`, cs.BookID, cs.Chapter, cs.ChunkIdx, cs.Stage, cs.SnapshotID, cs.ContentHash, - cs.Disposition, cs.FlagReason, cs.Attempts, cs.FinalHash, cs.CostUSD, cs.Detail) + cs.Disposition, cs.FlagReason, cs.Attempts, cs.FinalHash, cs.CostUSD, cs.Detail, + boolToInt(cs.Escalated), cs.EscalationModel) return err } @@ -64,12 +70,16 @@ func (s *Store) GetChunkStatus(bookID string, chapter, chunkIdx int, stage strin ctx, cancel := opContext() defer cancel() cs := ChunkStatus{BookID: bookID, Chapter: chapter, ChunkIdx: chunkIdx, Stage: stage} + var escalated int err := s.r.QueryRowContext(ctx, ` - SELECT snapshot_id, content_hash, disposition, flag_reason, attempts, final_hash, cost_usd, detail + SELECT snapshot_id, content_hash, disposition, flag_reason, attempts, final_hash, cost_usd, detail, + escalated, escalation_model FROM chunk_status WHERE book_id = ? AND chapter = ? AND chunk_idx = ? AND stage = ?`, bookID, chapter, chunkIdx, stage).Scan( - &cs.SnapshotID, &cs.ContentHash, &cs.Disposition, &cs.FlagReason, &cs.Attempts, &cs.FinalHash, &cs.CostUSD, &cs.Detail) + &cs.SnapshotID, &cs.ContentHash, &cs.Disposition, &cs.FlagReason, &cs.Attempts, &cs.FinalHash, &cs.CostUSD, &cs.Detail, + &escalated, &cs.EscalationModel) + cs.Escalated = escalated != 0 if errors.Is(err, sql.ErrNoRows) { return nil, nil } @@ -86,7 +96,7 @@ func (s *Store) ChunkStatusesForBook(bookID string) ([]ChunkStatus, error) { defer cancel() rows, err := s.r.QueryContext(ctx, ` SELECT chapter, chunk_idx, stage, snapshot_id, content_hash, disposition, flag_reason, - attempts, final_hash, cost_usd, detail + attempts, final_hash, cost_usd, detail, escalated, escalation_model FROM chunk_status WHERE book_id = ? ORDER BY chapter, chunk_idx, stage`, bookID) if err != nil { @@ -96,11 +106,56 @@ func (s *Store) ChunkStatusesForBook(bookID string) ([]ChunkStatus, error) { var out []ChunkStatus for rows.Next() { cs := ChunkStatus{BookID: bookID} + var escalated int if err := rows.Scan(&cs.Chapter, &cs.ChunkIdx, &cs.Stage, &cs.SnapshotID, &cs.ContentHash, - &cs.Disposition, &cs.FlagReason, &cs.Attempts, &cs.FinalHash, &cs.CostUSD, &cs.Detail); err != nil { + &cs.Disposition, &cs.FlagReason, &cs.Attempts, &cs.FinalHash, &cs.CostUSD, &cs.Detail, + &escalated, &cs.EscalationModel); err != nil { return nil, err } + cs.Escalated = escalated != 0 out = append(out, cs) } return out, rows.Err() } + +// ResetChunkStages deletes the chunk_status rows AND the checkpoints for the given stages of +// ONE chunk, in a single transaction — the durable half of `tmctl redrive` (D15.3). After +// this, the next translate re-derives those stages: their chunk_status is gone (the runner +// re-runs instead of resuming the flag) and their checkpoints are gone (a FRESH provider call +// is made, not a deterministic replay of the flagged completion). Only the passed stages are +// touched — an upstream OK stage keeps its checkpoint and resumes at $0 (D12: never re-pay +// DispOK work). MONEY (documented, honest): the money already spent on the discarded attempts +// STAYS committed in `spend` (it was really billed at the provider), so the spend ceilings +// remain honest; only the resume cache and the escalation-budget accounting are reset, which +// is the "fresh retry/escalation budget" redrive grants an explicit operator command. Thus +// after a redrive committed(spend) >= SUM(checkpoints) — the safe direction (a ceiling never +// under-counts). Checkpoints are joined to jobs by (book, chapter, stage); request_log +// (append-only telemetry) is deliberately NOT touched, so the paid history is still auditable. +func (s *Store) ResetChunkStages(bookID string, chapter, chunkIdx int, stages []string) error { + if len(stages) == 0 { + return nil + } + ctx, cancel := opContext() + defer cancel() + tx, err := s.w.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, stage := range stages { + if _, err := tx.ExecContext(ctx, ` + DELETE FROM checkpoints + WHERE chunk_idx = ? AND job_id IN ( + SELECT id FROM jobs WHERE book_id = ? AND chapter = ? AND stage = ?)`, + chunkIdx, bookID, chapter, stage); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + DELETE FROM chunk_status + WHERE book_id = ? AND chapter = ? AND chunk_idx = ? AND stage = ?`, + bookID, chapter, chunkIdx, stage); err != nil { + return err + } + } + return tx.Commit() +} diff --git a/backend/internal/store/migrate.go b/backend/internal/store/migrate.go index ae1d215..6e5d64e 100644 --- a/backend/internal/store/migrate.go +++ b/backend/internal/store/migrate.go @@ -253,6 +253,16 @@ var migrations = []string{ PRIMARY KEY (book_id, chapter, chunk_idx) ); `, + // v6: surface single-hop escalation on the chunk_status RESOLVE (D15.3 status/redrive). + // The fallback fact lived only on the checkpoint (escalation flag, v3); folding it into + // chunk_status lets `tmctl status` project escalation-on-flagged-resume WITHOUT re-reading + // checkpoints (the deferred Milestone-2.5 finding #6 — the escalation column the status + // read-model needs). Derivable from checkpoints, so a lost/rebuilt row self-heals on the + // next resolve. escalation_model is the fallback that answered (empty when not escalated). + ` + ALTER TABLE chunk_status ADD COLUMN escalated INTEGER NOT NULL DEFAULT 0; + ALTER TABLE chunk_status ADD COLUMN escalation_model TEXT NOT NULL DEFAULT ''; + `, } // migrate runs all pending migrations on the write pool, one transaction per diff --git a/backend/internal/store/requestlog.go b/backend/internal/store/requestlog.go index 3a12dc9..b37c978 100644 --- a/backend/internal/store/requestlog.go +++ b/backend/internal/store/requestlog.go @@ -69,6 +69,19 @@ func (s *Store) LogRequest(ctx context.Context, log *slog.Logger, rl RequestLog) } } +// FreshCallLatencyMS sums the wall-clock latency of the book's FRESH provider calls (tm_hit=0 +// — a checkpoint replay has no answer latency to count) and their count. It is the throughput +// input for the `tmctl status` ETA (D15.3): mean fresh-call latency × remaining work, a +// throughput estimate, never a synthetic time-bar. Pure read, $0. +func (s *Store) FreshCallLatencyMS(bookID string) (totalMS int64, calls int, err error) { + ctx, cancel := opContext() + defer cancel() + err = s.r.QueryRowContext(ctx, + `SELECT COALESCE(SUM(latency_ms),0), COUNT(*) FROM request_log WHERE book_id = ? AND tm_hit = 0`, + bookID).Scan(&totalMS, &calls) + return +} + // RequestLogView is one request_log row for inspection (tmctl report). type RequestLogView struct { TS string