From be53cc82f15eddd6bde6a939f130016d8cb034d7 Mon Sep 17 00:00:00 2001 From: heaven Date: Fri, 11 Sep 2026 04:26:06 +0300 Subject: [PATCH] Freeze what the cold run will buy and what judges it: a pre-registered table of outcomes, the three-chapter slice, its digests, and the instruments. --- docs/experiments/24-door-to-file.md | 109 ++++++++++ eval/door_to_file/README.md | 27 +++ eval/door_to_file/collect.py | 310 ++++++++++++++++++++++++++++ eval/door_to_file/drive.sh | 232 +++++++++++++++++++++ eval/door_to_file/go.mod | 5 + eval/door_to_file/go.sum | 4 + eval/door_to_file/spread.py | 229 ++++++++++++++++++++ eval/door_to_file/stub/main.go | 137 ++++++++++++ eval/door_to_file/zeropipe/main.go | 221 ++++++++++++++++++++ 9 files changed, 1274 insertions(+) create mode 100644 docs/experiments/24-door-to-file.md create mode 100644 eval/door_to_file/README.md create mode 100644 eval/door_to_file/collect.py create mode 100755 eval/door_to_file/drive.sh create mode 100644 eval/door_to_file/go.mod create mode 100644 eval/door_to_file/go.sum create mode 100644 eval/door_to_file/spread.py create mode 100644 eval/door_to_file/stub/main.go create mode 100644 eval/door_to_file/zeropipe/main.go diff --git a/docs/experiments/24-door-to-file.md b/docs/experiments/24-door-to-file.md new file mode 100644 index 00000000..8e2285e8 --- /dev/null +++ b/docs/experiments/24-door-to-file.md @@ -0,0 +1,109 @@ +# Эксперимент 24. Холодный прогон «от двери до ФАЙЛА» — настоящая книга, настоящие деньги + +> **Пре-регистрация.** Всё, что ниже строки «ГРАНИЦА ФРИЗА», написано ДО первого платного вызова и +> закоммичено фриз-коммитом. Результаты дописываются ПОСЛЕ прогона, отдельной секцией, и ничего в +> пре-реге не правят: расхождение называется, а не подгоняется. + +**Зона:** полигон · **Пак:** `docs/POLYGON_COLD_RUN_A_SESSION_PROMPT.md` (редакция 2) · **Дата:** 11.09.2026 +**Предмет:** строка бэклога **16** — цена и проходимость книги, ДОВЕДЁННОЙ ДО КОНЦА, не измерены ни разу. + +## 0. Что именно проверяется + +Одна настоящая книга (蛊真人, главы 1–3, zh→ru) проходит весь путь: интейк → цена → холд → прогон → +остановка на подпись банка → решение человека через дверь правок → резюм → редакторская волна → +сборка выгрузки → скачанный файл → **человек читает первый абзац**. У каждого узла напечатано, ЧЕМ он +предъявлен. + +⚠ **Судьи на этом пути НЕТ.** Боевой `pipeline-c1.yaml` несёт две стадии — `draft` и `edit`; `role: judge` +живёт только в `c2`. `escalation.budget_usd: 0` ⇒ эскалация не исполняется; `gates.coverage.enabled: false` +⇒ из пяти гейтов работают четыре. Проверяемый путь: **черновик → редактор → четыре $0-гейта → выгрузка.** + +⚠ **О качестве перевода пак не судит вовсе:** 蛊真人 узнаётся моделями (претрейн), и любое суждение о +качестве было бы суждением о памяти модели. Чтение человеком отвечает на один вопрос — «это связный +русский текст, а не мусор и не обрывок». + +## 1. ГРАНИЦА ФРИЗА — что зафиксировано до первого цента + +| Предмет | Значение | +|---|---| +| Фриз-sha | `b0f5d8915c8553993f42e6e3a9d6fbd18adfa68e` | +| Фриз-дерево | локальный **клон** `~/tm-coldrun-a/freeze` (0 грязных строк при 13 в главном дереве) | +| Бинари | `tmctl` · `tmplatformd` · `tmplatformctl`, собраны из клона, у каждого `vcs.revision=b0f5d89…`, `vcs.modified=false`, строк `vcs.*` — 3 | +| Исходник (оригинал) | `books/gu-zhenren/coldrun-v16/guzhenren-ch1-10.gb18030.txt`, 57838 байт, sha256 `0b5f9b0266d8c32ac717a41211d2ed15d4f2a07f37a701fb686e255ab6c89f37` | +| Исходник (СРЕЗ, что покупается) | главы 1–3, 17564 байта, sha256 `ed870ba6065ce3eb4eec12e80238efbc2be8511f38334ab1d92d5693f4ef4df9` | +| Фактический pipeline-YAML | `~/tm-coldrun-a/mirror/cfg/pipeline-c1.yaml`, sha256 `a46b33ee65b7713b4fbf86f64c788ba6b11eb0a219170c145a6205088156ad49` — **побайтно равен** `backend/configs/pipeline-c1.yaml` фриза (`cmp`) | +| Артефакт контраста | `mining-contrast.zh.txt` = jieba 0.42.1 `dict.txt`, sha256 `7197c3211ddd98962b036cdf40324d1ea2bfaa12bd028e68faa70111a88e12a8`, 349 046 строк | +| Шаблон книги | `~/tm-coldrun-a/book-template-c1.yaml`, sha256 `f66e681ff55b2653fc87944f45d7214bac0cd209f42102f642267d1ef61d8057` | +| Потолки книги (шаблон) | `book_usd: 1.25` · `day_usd: 2.50` — машинный бэкстоп; сессия останавливается и пингует РАНЬШЕ, на факте $1.00 | +| Грант платной учётки | $2.50 (отдельная учётка, не дымовая — иначе леджер смешивает дымовые холды с боевыми) | + +**Объём заказа — вариант (а) пака:** режется ИСХОДНИК, грузится срез трёх глав. Довод сильнее, чем +«сверка честнее»: при варианте (б) семь незаказанных глав — это ДЫРЫ, дверь обязана отдать файл +`--partial` с пометкой (`D39.178`), и центральный критерий `exports.complete = true` недостижим +ПО ПОСТРОЕНИЮ. + +**Срез по $0-манифесту (`tmctl manifest --json`, читающий верб, ключей не требует):** + +| Глава | `units_total` | `chunks_total` | `expected_usd` | символов | +|---|---|---|---|---| +| 1 `第一节：纵身亡魔心仍不悔` | 2 | 2 | 0.032424 | 3287 | +| 2 `第二节：逆光阴五百年觉悟` | 1 | 2 | 0.028459 | 3051 | +| 3 `第三节：请一边玩蛋去` | 1 | 2 | 0.023237 | 2421 | +| **книга** | **4** | **6** | `expected_usd 2.08412008` · `book_once_usd 2.00` · `step_max_usd 0.13647876` | 8759 | + +⚠ `expected_usd` книги включает `book_once_usd` $2.00 — это ПОТОЛКИ контура банка +(`gates.terminology.budget_usd: 1` + `classify_budget_usd: 1`), а не смета. Смета текста — сумма глав, +**$0.084120**. Пинится **сумма `units_total` по заказанным главам = 4** (пин «≥1 глава с `units_total ≥ 2`» +из первой редакции пака охранял то, чего не бывает). + +## 2. Пре-рег-таблица: узел → что ДОЛЖНО произойти → чем предъявлено → что считается провалом + +| # | Узел | Ожидание (числом или строкой) | Чем предъявлено | Провал узла | +|---|---|---|---|---| +| 1 | Стенд — мой | слушатель на `127.0.0.1:8097` принадлежит МОЕМУ pid; на `11434` — заглушки нет вовсе (платная фаза) | `ss -ltnp` с pid | на порту чужой процесс | +| 2 | Код — из фриза | у трёх бинарей `vcs.revision=b0f5d89…`, `modified=false`, строк `vcs.*` > 0 | `go version -m` | любое из трёх не так ⇒ СТОП | +| 3 | Гость и деньги | учётка заведена `POST /auth/dev-login`; грант $2.50 одной строкой `credit_ledger.kind='grant'` | `tmplatformctl balance` + строка леджера | самореги нет, пополнения по HTTP нет — это ожидаемо | +| 4 | Интейк | `201`, `chapter_count = 3`, `character_count` ≈ 8759 | тело ответа + строка `books` | не 3 главы ⇒ СТОП до оплаты | +| 5 | Срез совпал | `units_total = 4`, по главам 2/1/1 | `chapters`+`units` в Postgres против таблицы §1 | расхождение ⇒ СТОП до оплаты | +| 6 | Цена и бонд | `term_consistency_funded: true`, `verdict: covers_all`, `affordable_chapters: 3`, `hold ≈ 2 241 629 µUSD` (= ⌈expected×1.25⌉ + step_max + бонд 2 000 000) | `GET /v0/books/{id}/run-options` | `false` ⇒ СТОП до оплаты (тихая потеря консолидации терминов) | +| 7 | Старт | `202`, `status: translating`, `ordered_chapters: 3`; строка `runs` с `bond_funded = t` | ответ + строка `runs` | отказ старта ⇒ СТОП и пинг | +| 8 | Ключи живы | первый платный вызов не отказан по авторизации | `request_log.err` пусто на первой строке | отказ провайдера ⇒ СТОП и пинг (не моя поломка) | +| 9 | Кадры | `events.jsonl` несёт `hello · progress · unit_done · spend · bank_stop · finished`; `run_attempts.last_seq` растёт | счёт кадров по типам + `last_seq` | кадров нет при живом юните ⇒ дефект | +| 10 | Банк-стоп | `runs.status = awaiting_bank`, exit **3**, `.bank.json` несёт непустой `proposed[]` | статус + exit + копия сайдкара | стоп не наступил ⇒ **исход «узел не наступил»**, а не провал (三 условия разом: `--verify-bank` И edit-волна И некогда не показанный кластер) | +| 11 | Проекция банка на платформе | `bank_terms` = **0** ПОКА прогон не закрыт и **> 0** ПОСЛЕ | `GET /v0/books/{id}/bank` + `select count(*)` | ноль ПОСЛЕ закрытия ⇒ дефект | +| 12 | Дверь правок | `POST …/bank/corrections` `preview:true` → `changed:true`; затем `preview:false` → `accepted[0].state = applied`; на диске появляется `.mined-delta.yaml` | тело ответа + файл | отказ двери ⇒ дефект | +| 13 | Резюм | `202`, вторая строка в `run_attempts` (attempt_no = 2), `exit_code = 0` | `run_attempts` | прогон не возобновился ⇒ дефект | +| 14 | Книга переведена | `units.state = translated` для всех **4** юнитов | `select state, count(*)` с контрольным счётом юнитов книги | любой юнит не `translated` ⇒ книга не доведена | +| 15 | Сборка | `tmctl build`: `complete: true`, `pending/withheld/incomplete/stale/ghost = 0`, `stale_unknown: false`, `config_drift: false` | stdout сборки (на API этих полей НЕТ) | любое не так ⇒ книга не целая | +| 16 | Выгрузка | `exports.state = ready`, `exports.complete = t` в Postgres, `size_bytes` = числу скачанных байт | строка `exports` + `%{size_download}` | `complete` не `t` ⇒ дверь отдала книгу с дырой | +| 17 | ФАЙЛ | скачаны байты, sha256 напечатан; EPUB читается структурно (zip + `container.xml` + `nav` + spine = 3) | sha256 + разбор zip | файл не открывается ⇒ провал | +| 18 | Человек прочитал | первый абзац перевода — связный русский текст | цитата в отчёте | не текст ⇒ провал | +| 19 | Деньги сошлись | Σ `request_log.cost_usd` ≈ `spend.committed` ≈ Σ `credit_ledger` settlement ≈ (грант − баланс); открытых резерваций **0** | четыре счёта в µUSD рядом | расхождение — **назвать**, не подгонять | +| 20 | Ничего не реплеено | на платной базе `count(*) where tm_hit=1` печатается С РАЗБИВКОЙ ПО ПОПЫТКАМ, `distinct model_actual` не содержит `local-*` | запросы к `project.db` | есть `local-*` ⇒ прогон измерил заглушку | +| 21 | Расхождение форм | механический счёт по ОТГРУЖЕННОМУ тексту: сколько канонических терминов отданы более чем одной передачей и в каких главах | `eval/door_to_file/spread.py` + `tmctl export --json --pairs` | — (это ЗАМЕР, у него нет «провала») | + +⚠ **Про «три следа»: их не три.** Postgres платформы МАТЕРИАЛИЗУЕТСЯ из `events.jsonl` движка +(`Sink materializes a stream into the reporting database`), а текст, манифест и файл платформа берёт +теми же вербами `tmctl`. ⇒ согласие Postgres, диска движка и отчёта сборки — это **одна база, согласная +сама с собой**, а не три свидетеля. По-настоящему независимы только **скачанные байты и их sha256**; +биллинга провайдера у нас нет. Это не отменяет сверку — это меняет то, что она доказывает. + +## 3. Стоп-правила (остановка и пинг оркестратору, не решение сессии) + +1. фактическая трата дошла до **$1.00** (1 000 000 µUSD); +2. `runs.paused_reason` НЕПУСТО — любое из четырёх (`run_limit_reached · credit_exhausted · daily_ceiling · ceiling_unknown`); +3. срез не совпал с §1 — стоп ДО оплаты; +4. `term_consistency_funded: false` — стоп ДО оплаты; +5. бинарь не из фриза — стоп до устранения. + +## 4. Инструменты (зафиксированы фризом вместе с пре-регом) + +* `eval/door_to_file/drive.sh` — драйвер фаз с гейтами; отказывает, а не предупреждает. +* `eval/door_to_file/collect.py` — сборщик следов: фриз-гейт, Postgres, диск движка, отчёт сборки, деньги. +* `eval/door_to_file/spread.py` — счёт расхождения форм по отгруженному тексту (строка бэклога 406). +* `eval/door_to_file/stub/` — $0-заглушка провайдера `local` (перенос штатной `fakeProvider` стенда). +* `eval/door_to_file/zeropipe/` — рендер $0-пайплайна (перенос штатного `zeroCostPipeline`). + +**Позитивный контроль каждого прибора исполнен ДО платной фазы** (§«Фаза 0» в результатах): подсаженный +ненулевой банк, удержанный юнит и открытая резервация красят сборщик; книга с дырой красит отчёт сборки; +чужой и беcштамповый бинарь красят гейт; самотест `spread.py` ловит подсаженную вторую форму. diff --git a/eval/door_to_file/README.md b/eval/door_to_file/README.md new file mode 100644 index 00000000..0ec1f209 --- /dev/null +++ b/eval/door_to_file/README.md @@ -0,0 +1,27 @@ +# `door_to_file` — харнесс холодного прогона «от двери до ФАЙЛА» + +Инструменты пака «ХОЛОДНЫЙ ПРОГОН A»: одна настоящая книга (蛊真人, главы 1–3, zh→ru) проводится от +интейка до скачанного файла, и у каждого узла печатается, ЧЕМ он предъявлен. Пре-регистрация, +таблица ожидаемых исходов и результаты — `docs/experiments/24-door-to-file.md`. + +| Файл | Что делает | Чем проверен | +|---|---|---| +| `drive.sh` | драйвер фаз: гейт · вход · грант · интейк · опции · старт · вахта · улика · подпись · резюм · выгрузка · сбор · формы. **Отказывает**, а не предупреждает | подсадка чужого бинаря, бинаря без штампа и неверного sha — каждый отказ назвал ИМЕННО подсаженное | +| `collect.py` | сборщик следов: фриз-гейт, Postgres платформы, диск движка, отчёт сборки, деньги в µUSD | подсадка ненулевого банка, удержанного юнита и открытой резервации — покраснел по всем трём осям | +| `spread.py` | сколько РАЗНЫХ передач получил один канонический термин в ОТГРУЖЕННОМ тексте (строка бэклога 406) | `--selftest`: подсаженная вторая форма ловится, склонение той же формы не двоится, неотрендеренный термин даёт 0 | +| `stub/` | $0-заглушка провайдера `local` на `127.0.0.1:11434` | перенос штатной `fakeProvider` стенда (`platform/internal/runner/translate_resnapshot_live_test.go`) | +| `zeropipe/` | рендер боевого пайплайна на $0-пару | перенос штатного `zeroCostPipeline` (`platform/internal/runner/bankapply_live_test.go`); печатает счёт достижимых ПЛАТНЫХ моделей в обоих файлах | + +## Три вещи, которые стоит знать, прежде чем переиспользовать это + +1. **`sed` по `model:` из рецепта стенда НЕДОСТАТОЧЕН.** Модели несут ещё и гейты (`model` и + `classify_model`), а на `pipeline-c1` терминолог с классификатором резолвятся в ПЛАТНОГО + провайдера, когда все стадии уже обнулены. Поэтому `zeropipe` — порт готового рендера, а не sed. +2. **Ноль достижимых платных моделей печатается рядом с их числом в боевом файле.** Иначе «конфиг + чист» и «прибор промолчал» неотличимы: замерено 8 достижимых платных моделей в `pipeline-c1.yaml` + и 0 в рендере. Счёт идёт по ЗНАЧЕНИЯМ разобранного YAML, а не по тексту: текстовый счёт считал бы + комментарии, а у рендера их нет — он выглядел бы чистым в тот момент, когда комментарии отпали. +3. **Книга, которую пайплайн переводит бесплатно, не может быть продана.** `ingest.Manifest.priced()` + отказывает при `step_max_usd <= 0`, а на $0-паре он именно ноль ⇒ `409 not_priced` и старта нет. + Дымовой стенд поэтому даёт локальной модели НОМИНАЛЬНУЮ цену — одной строкой в своём + `models-smoke.yaml`, — и это объявлено в отчёте, а не спрятано. diff --git a/eval/door_to_file/collect.py b/eval/door_to_file/collect.py new file mode 100644 index 00000000..99862ef5 --- /dev/null +++ b/eval/door_to_file/collect.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Trace collector for the cold run «door to file». + +It reads the three places a run leaves a trace and prints them as facts an outsider can check: +the platform's Postgres, the engine's own disk (project DB + sidecars + the frame stream), and the +HTTP surface plus the downloaded bytes. + +Two rules it follows everywhere, because both were paid for: + +* every zero is printed next to a CONTROL number, so «nothing happened» and «nothing was read» are + told apart on the page rather than in the reader's head; +* the three traces are NOT independent — the platform's rows are materialised from the engine's + event stream and its text comes from the same `tmctl` verbs — so the report says so rather than + presenting agreement between them as corroboration. + +Usage: collect.py --stand DIR --dsn DSN [--book ID] [--run ID] [--label NAME] [--json OUT] +""" + +import argparse +import json +import os +import re +import sqlite3 +import subprocess +import sys +from collections import Counter + +MICRO = 1_000_000 + + +def psql(dsn, sql): + """One query, rows as lists of strings. Separator is a tab; psql -A -t gives raw fields.""" + out = subprocess.run( + [os.path.expanduser("~/.local/pgsql/bin/psql"), dsn, "-At", "-F", "\t", "-c", sql], + capture_output=True, text=True) + if out.returncode != 0: + raise RuntimeError(f"psql failed: {out.stderr.strip()}\nSQL: {sql}") + return [line.split("\t") for line in out.stdout.strip().splitlines() if line != ""] + + +def one(dsn, sql, default="0"): + rows = psql(dsn, sql) + return rows[0][0] if rows and rows[0] and rows[0][0] != "" else default + + +def section(title): + print() + print("=" * 78) + print(title) + print("=" * 78) + + +def stamp_of(binary): + """The VCS stamp of a Go binary, as three answers: revision, modified, and how many vcs.* lines + were there at all. The third is the one that matters: a binary built in a linked worktree + carries NONE, and a gate that only refuses `vcs.modified=true` passes it every time.""" + out = subprocess.run(["go", "version", "-m", binary], capture_output=True, text=True).stdout + lines = [l.strip() for l in out.splitlines()] + vcs = [l for l in lines if "\tvcs." in l or l.startswith("build\tvcs.")] + rev = mod = "" + for l in vcs: + if "vcs.revision=" in l: + rev = l.split("vcs.revision=")[1].strip() + if "vcs.modified=" in l: + mod = l.split("vcs.modified=")[1].strip() + return {"binary": binary, "revision": rev, "modified": mod, + "vcs_lines": len(vcs), "build_lines": len(lines)} + + +def freeze_gate(stand, sha): + section("A. FREEZE GATE — every binary, three conditions, not one") + ok = True + for name in ("tmctl", "tmplatformd", "tmplatformctl"): + path = os.path.join(stand, "bin", name) + if not os.path.exists(path): + print(f" {name}: ABSENT at {path}") + ok = False + continue + s = stamp_of(path) + verdict = [] + if s["vcs_lines"] == 0: + verdict.append("NO vcs.* LINES AT ALL — the gate would be vacuous") + if s["revision"] != sha: + verdict.append(f"revision is not the frozen one ({sha[:12]})") + if s["modified"] != "false": + verdict.append(f"vcs.modified={s['modified']!r}") + print(f" {name}: revision={s['revision'][:12] or '(none)'} modified={s['modified'] or '(none)'} " + f"vcs_lines={s['vcs_lines']} (control: build lines {s['build_lines']}) " + f"→ {'ok' if not verdict else '⛔ ' + '; '.join(verdict)}") + ok = ok and not verdict + print(f" VERDICT: {'all three binaries are from the freeze' if ok else '⛔ NOT ALL BINARIES ARE FROM THE FREEZE'}") + return ok + + +def platform_trace(dsn, book, run): + section("B. PLATFORM (Postgres) — the rows, each zero beside its control") + tables = ["users", "books", "chapters", "units", "runs", "run_attempts", "reservations", + "credit_ledger", "book_events", "exports", "bank_terms"] + for t in tables: + n = one(dsn, f"select count(*) from {t}") + print(f" {t}: {n}") + if book: + print(f"\n -- this book ({book}) --") + print(" books row:", psql(dsn, f""" + select status, chapter_count, source_chars, expected_micro_usd, book_once_micro_usd, + step_max_micro_usd, ordered_at is not null + from books where id = '{book}'""")) + units = psql(dsn, f""" + select u.state, count(*) from units u + join chapters c on c.id = u.chapter_id + where c.book_id = '{book}' group by 1 order by 1""") + total = one(dsn, f"select count(*) from units u join chapters c on c.id=u.chapter_id where c.book_id='{book}'") + print(f" units by state: {units} (control: units of this book = {total})") + ev = psql(dsn, f"select event, count(*) from book_events where book_id='{book}' group by 1 order by 2 desc") + evtotal = one(dsn, f"select count(*) from book_events where book_id='{book}'") + print(f" book_events by event: {ev} (control: events of this book = {evtotal})") + bank_all = one(dsn, f"select count(*) from bank_terms where book_id='{book}'") + bank_ok = one(dsn, f"select count(*) from bank_terms where book_id='{book}' and status='approved'") + allbooks = one(dsn, "select count(*) from bank_terms") + print(f" bank_terms: {bank_all} rows, {bank_ok} approved (control: bank_terms of ALL books = {allbooks})") + ex = psql(dsn, f"""select id, format, state, complete, size_bytes, path + from exports where book_id='{book}' order by requested_at""") + print(f" exports: {ex or 'none'} (control: exports of ALL books = {one(dsn, 'select count(*) from exports')})") + if run: + print(f"\n -- this run ({run}) --") + print(" runs row:", psql(dsn, f""" + select status, verify_bank, bond_funded, coalesce(paused_reason,''), + coalesce(failure_reason,''), ceiling_chapters, finished_at is not null + from runs where id='{run}'""")) + att = psql(dsn, f""" + select attempt_no, exit_code, exit_result, last_seq, spend_micro_usd, + ceiling_arg_micro_usd, engine_binary + from run_attempts where run_id='{run}' order by attempt_no""") + print(f" run_attempts ({len(att)}): ") + for a in att: + print(" ", a) + print(" ⚠ attempts, not runs: a resume after a bank stop is a SECOND attempt under the same run id") + + +def money(dsn, book, run, projectdb): + section("C. MONEY — micro-USD, as a CHECK between paths and never as an identity") + led = psql(dsn, "select kind, count(*), coalesce(sum(amount_micro_usd),0) from credit_ledger group by 1 order by 1") + print(f" credit_ledger by kind: {led}") + grants = int(one(dsn, "select coalesce(sum(amount_micro_usd),0) from credit_ledger where kind='grant'")) + settle = int(one(dsn, "select coalesce(sum(amount_micro_usd),0) from credit_ledger where kind='settlement'")) + balance = int(one(dsn, "select coalesce(sum(amount_micro_usd),0) from credit_ledger")) + openres = one(dsn, "select count(*) from reservations where state='open'") + allres = one(dsn, "select count(*) from reservations") + print(f" grants: {grants:>12} µUSD (${grants/MICRO:.6f})") + print(f" settlements: {settle:>12} µUSD (${settle/MICRO:.6f})") + print(f" balance: {balance:>12} µUSD (${balance/MICRO:.6f}) = sum of ALL ledger rows") + print(f" open reservations: {openres} (control: reservations of every state = {allres})") + spend_att = one(dsn, f"select coalesce(sum(spend_micro_usd),0) from run_attempts where run_id='{run}'") if run else "0" + print(f" Σ run_attempts.spend_micro_usd for this run: {spend_att}") + if projectdb and os.path.exists(projectdb): + rows, total, hits, models, bad = request_log(projectdb) + print(f" engine request_log: {rows} rows, Σ cost_usd = ${total:.6f} → {round(total*MICRO)} µUSD") + print(f" tm_hit=1 (answer taken from a checkpoint, bought nothing): {hits} (control: rows = {rows})") + print(f" distinct model_actual: {models}") + print(f" rows with err or finish_reason != stop: {len(bad)} (control: rows = {rows})") + for b in bad: + print(" ", b) + print(f" ⚠ request_log is TELEMETRY, not the source of money (the engine's money lives in spend/checkpoints);") + print(f" a redrive deletes checkpoints, so a difference here is a named class, not an error to hide.") + sp = engine_spend(projectdb) + print(f" engine spend table: {sp}") + else: + print(f" engine request_log: NOT READ — no project db at {projectdb}") + + +def request_log(db): + c = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + rows = c.execute("select count(*) from request_log").fetchone()[0] + total = c.execute("select coalesce(sum(cost_usd),0) from request_log").fetchone()[0] + hits = c.execute("select count(*) from request_log where tm_hit=1").fetchone()[0] + models = [r[0] for r in c.execute("select distinct model_actual from request_log order by 1")] + bad = c.execute("""select id, stage, role, model_actual, finish_reason, err + from request_log + where coalesce(err,'') != '' or coalesce(finish_reason,'') not in ('stop','')""").fetchall() + return rows, total, hits, models, bad + + +def engine_spend(db): + c = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + return c.execute("select book_id, date, committed_usd, reserved_usd from spend").fetchall() + + +def engine_disk(bookdir): + section("D. ENGINE DISK — frames, sidecars, checkpoints") + if not os.path.isdir(bookdir): + print(f" no book directory at {bookdir}") + return + files = sorted(os.listdir(bookdir)) + print(f" book directory holds {len(files)} entries: {files}") + ev = os.path.join(bookdir, "events.jsonl") + if os.path.exists(ev): + kinds = Counter() + n = 0 + with open(ev, encoding="utf-8") as f: + for line in f: + n += 1 + try: + kinds[json.loads(line).get("type", "?")] += 1 + except json.JSONDecodeError: + kinds[""] += 1 + print(f" events.jsonl: {n} frames, by type: {dict(kinds)}") + else: + print(f" events.jsonl: ABSENT (control: {len(files)} files were listed in this directory)") + for name in sorted(f for f in files if ".bank" in f or "signature" in f or "mined" in f): + p = os.path.join(bookdir, name) + print(f" sidecar {name}: {os.path.getsize(p)} bytes sha256={sha256(p)}") + db = os.path.join(bookdir, "project.db") + if os.path.exists(db): + c = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + for t in ("checkpoints", "chunk_status", "glossary", "bank_stop_presented", "snapshots"): + try: + print(f" {t}: {c.execute(f'select count(*) from {t}').fetchone()[0]} rows") + except sqlite3.Error as e: + print(f" {t}: unreadable ({e})") + + +def sha256(path): + import hashlib + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def build_report(stand, bookdir, out): + section("E. BUILD REPORT — the seven counters the API does NOT carry") + cfg = os.path.join(bookdir, "book.yaml") + if not os.path.exists(cfg): + print(f" no book.yaml at {cfg}") + return + # --out names the FILE, not a directory, and an explicit path is never replaced — so each call + # writes its own copy rather than asking the engine to overwrite one. + os.makedirs(os.path.dirname(out), exist_ok=True) + # The verb first, its flags after, and no --json: the report is JSON regardless (the flag is + # refused by name). Order and flags are the platform's own BuildArgs (internal/runner/build.go). + cmd = [os.path.join(stand, "bin", "tmctl"), "build", "--config", cfg, + "--format", "txt", "--out", out, "--partial"] + res = subprocess.run(cmd, capture_output=True, text=True) + print(f" $ {' '.join(cmd)}") + print(f" exit {res.returncode}") + # The report is the WHOLE of stdout, pretty-printed over many lines: a per-line scan finds + # nothing and reports "no document", which looks exactly like a build that said nothing. + doc = None + try: + doc = json.loads(res.stdout) + except json.JSONDecodeError: + start = res.stdout.find("{") + if start >= 0: + try: + doc = json.loads(res.stdout[start:]) + except json.JSONDecodeError: + doc = None + if doc is None: + print(" stdout carried no report document:") + print(" ", res.stdout.strip()[:600]) + print(" stderr:", res.stderr.strip()[:600]) + return + keys = ["build_version", "total_units", "pending_units", "withheld_units", "incomplete_units", + "stale_units", "stale_unknown", "ghost_rows", "config_drift", "complete"] + for k in keys: + print(f" {k}: {doc.get(k)}") + print(f" removed_files: {doc.get('removed_files')} stale_copies: {doc.get('stale_copies')}") + clean = (doc.get("complete") is True and doc.get("stale_unknown") is False + and all(doc.get(k, 0) == 0 for k in + ("pending_units", "withheld_units", "incomplete_units", "stale_units", "ghost_rows"))) + print(f" VERDICT: {'a WHOLE book — all five counters zero, stale_unknown false, complete true' if clean else '⛔ NOT whole by this report'}") + print(" ⚠ stale_units == 0 proves nothing on its own: under config drift it is mechanically zero,") + print(" which is why stale_unknown is printed beside it rather than folded into the same count.") + return doc + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--stand", required=True) + ap.add_argument("--dsn", required=True) + ap.add_argument("--book", default="") + ap.add_argument("--run", default="") + ap.add_argument("--sha", default=os.environ.get("FREEZE_SHA", "")) + ap.add_argument("--label", default="collect") + ap.add_argument("--skip-build", action="store_true") + a = ap.parse_args() + + print(f"COLD RUN «DOOR TO FILE» — trace bundle «{a.label}»") + print(f"stand={a.stand} book={a.book or '(none)'} run={a.run or '(none)'}") + if a.sha: + freeze_gate(a.stand, a.sha) + bookdir = os.path.join(a.stand, "stand", "books", a.book) if a.book else "" + platform_trace(a.dsn, a.book, a.run) + money(a.dsn, a.book, a.run, os.path.join(bookdir, "project.db") if bookdir else "") + if bookdir: + engine_disk(bookdir) + if not a.skip_build: + import time as _t + build_report(a.stand, bookdir, os.path.join( + a.stand, "evidence", "builds", f"{a.label}-{_t.strftime('%Y%m%dT%H%M%S')}.txt")) + section("F. WHAT THIS BUNDLE DOES NOT PROVE") + print(" The platform's rows are MATERIALISED from the engine's event stream, and its text, its") + print(" manifest and its file come from the same `tmctl` verbs. So B, D and E agreeing is ONE") + print(" database agreeing with itself, not three witnesses. Independent of them are only the") + print(" bytes downloaded over HTTP and their sha256. There is no provider-side billing here.") + + +if __name__ == "__main__": + main() diff --git a/eval/door_to_file/drive.sh b/eval/door_to_file/drive.sh new file mode 100755 index 00000000..8dfac1ea --- /dev/null +++ b/eval/door_to_file/drive.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +# Driver of the cold run «door to file»: one real book, three chapters, zh→ru, from the intake door +# to a file a human opens. Every step is a subcommand so that a stop rule can stop the run between +# two of them — the five stops of this pack are a human's call and a ping, never a script's. +# +# It refuses rather than warns wherever the refusal is about money or about which code is running: +# * a binary that is not from the freeze (three conditions, not one: the frozen revision, a clean +# tree, and the stamp EXISTING — a binary built in a linked git worktree carries no vcs.* lines +# at all, and a gate that only forbids `vcs.modified=true` passes it every time); +# * an order whose size is not the pre-registered one; +# * an order whose bond is unfunded (`term_consistency_funded: false`), which buys a pipeline with +# its terminology consolidation silently switched off; +# * a run that has paused for any reason at all. +# +# Usage: drive.sh [args] (see `drive.sh help`) +set -u -o pipefail + +W=${W:-/home/ubuntu-26/tm-coldrun-a} +STATE=$W/stand/run-state.env +[ -f "$STATE" ] && . "$STATE" +FREEZE_SHA=${FREEZE_SHA:-b0f5d8915c8553993f42e6e3a9d6fbd18adfa68e} +ADDR=${ADDR:-127.0.0.1:8097} +DSN=${DSN:-postgres://postgres@/tm_coldrun_door?host=/tmp&port=55433&sslmode=disable} +PSQL=${PSQL:-$HOME/.local/pgsql/bin/psql} +JAR=${JAR:-$W/stand/cookies-paid.txt} +# The pack's money stop, in micro-USD. Reaching it is a STOP and a ping, not a decision of this script. +STOP_AT_MICRO=${STOP_AT_MICRO:-1000000} + +say() { printf '%s\n' "$*"; } +die() { printf '⛔ %s\n' "$*" >&2; exit 1; } +sql() { "$PSQL" "$DSN" -At -F $'\t' -c "$1"; } +api() { curl -s --noproxy '*' -b "$JAR" -H 'X-TM-Client: coldrun-driver' "$@"; } +remember() { printf '%s=%s\n' "$1" "$2" >> "$STATE"; say "remembered $1=$2"; } + +step_gate() { + local bad=0 + say "=== binaries: the frozen revision, a clean tree, and a stamp that EXISTS ===" + for b in tmctl tmplatformd tmplatformctl; do + local out rev mod n + out=$(go version -m "$W/bin/$b" 2>/dev/null) || die "$b is not readable" + n=$(printf '%s\n' "$out" | grep -c 'vcs\.') || true + rev=$(printf '%s\n' "$out" | sed -n 's/.*vcs\.revision=//p') + mod=$(printf '%s\n' "$out" | sed -n 's/.*vcs\.modified=//p') + printf ' %-15s revision=%.12s modified=%s vcs_lines=%s (control: build lines %s)\n' \ + "$b" "${rev:-none}" "${mod:-none}" "$n" "$(printf '%s\n' "$out" | wc -l)" + [ "$n" -gt 0 ] || { say " ⛔ no vcs.* lines: this gate would be vacuous"; bad=1; } + [ "$rev" = "$FREEZE_SHA" ] || { say " ⛔ not the frozen revision"; bad=1; } + [ "$mod" = "false" ] || { say " ⛔ built from a dirty tree"; bad=1; } + done + say "=== the stand is MINE: pid on each port, not a 200 from someone else's process ===" + for p in 11434 ${ADDR##*:}; do + local line; line=$(ss -ltnp "sport = :$p" 2>/dev/null | tail -n +2) + [ -n "$line" ] || { say " port $p: nothing is listening"; bad=1; continue; } + say " port $p: $(printf '%s' "$line" | sed 's/.*users://')" + done + say "=== what will be bought: the pipeline, its contrast artefact, the template ===" + # Printed AND checked against the pre-registration when it is given: printing alone leaves the + # comparison to a reader who has the other number somewhere else. + local f expect got + for f in "pipeline-c1.yaml:$W/mirror/cfg/pipeline-c1.yaml:${EXPECT_PIPELINE_SHA:-}" \ + "mining-contrast.zh.txt:$W/mirror/cfg/mining-contrast.zh.txt:${EXPECT_CONTRAST_SHA:-}" \ + "book-template-c1.yaml:$W/book-template-c1.yaml:${EXPECT_TEMPLATE_SHA:-}" \ + "source slice:${SOURCE_FILE:-/dev/null}:${EXPECT_SOURCE_SHA:-}"; do + local name path + name=${f%%:*}; path=$(printf '%s' "$f" | cut -d: -f2); expect=${f##*:} + [ -f "$path" ] || { say " $name: ABSENT at $path"; [ -n "$expect" ] && bad=1; continue; } + got=$(sha256sum "$path" | cut -d' ' -f1) + if [ -n "$expect" ] && [ "$expect" != "$got" ]; then + say " $name: $got ⛔ the pre-registration says $expect"; bad=1 + else + say " $name: $got${expect:+ (matches the pre-registration)}" + fi + done + cmp -s "$W/mirror/cfg/pipeline-c1.yaml" "$W/freeze/backend/configs/pipeline-c1.yaml" \ + && say " the pipeline is byte-identical to the freeze's pipeline-c1.yaml" \ + || { say " ⛔ the pipeline is NOT the freeze's c1"; bad=1; } + say "=== the freeze tree itself ===" + say " HEAD $(git -C "$W/freeze" rev-parse HEAD) dirty lines: $(git -C "$W/freeze" status --porcelain | wc -l)" + [ -e /tmp/.git ] && { say " ⛔ /tmp/.git exists: a build here would lose its stamp"; bad=1; } + [ "$bad" = 0 ] || die "the gate refused; nothing was bought" + say "GATE PASSED" +} + +step_login() { api -c "$JAR" -X POST "$ADDR/auth/dev-login" -o /dev/null -w 'dev-login HTTP %{http_code}\n'; } + +step_grant() { # $1 = user id, $2 = usd + "$W/bin/tmplatformctl" grant --user "$1" --usd "$2" --note "cold run door-to-file" + "$W/bin/tmplatformctl" balance --user "$1" +} + +step_intake() { # $1 = path to the source, $2 = title + local out id + out=$(api -F "title=$2" -F 'source_lang=zh' -F 'target_lang=ru' -F "file=@$1" "$ADDR/v0/books") + say "$out" + id=$(printf '%s' "$out" | python3 -c 'import sys,json;print(json.load(sys.stdin)["id"])') || die "no book id in the answer" + remember BOOK "$id" + say "source sha256: $(sha256sum "$1" | cut -d' ' -f1)" +} + +step_options() { # refuses unless the size and the bond are what the pre-registration says + local want_chapters=${1:-3} want_units=${2:-} + local out + for _ in $(seq 1 15); do + out=$(api "$ADDR/v0/books/$BOOK/run-options") + printf '%s' "$out" | grep -q '"order"' && break + sleep 3 + done + printf '%s' "$out" | python3 -m json.tool + local ch funded + ch=$(printf '%s' "$out" | python3 -c 'import sys,json;print(json.load(sys.stdin)["order"]["chapters_left"])') + funded=$(printf '%s' "$out" | python3 -c 'import sys,json;print(json.load(sys.stdin)["order"]["term_consistency_funded"])') + say "--- the book as the platform materialised it ---" + sql "select c.number, count(u.id) as units from chapters c left join units u on u.chapter_id=c.id + where c.book_id='$BOOK' group by 1 order by 1" | sed 's/^/ chapter /' + local units; units=$(sql "select count(*) from units u join chapters c on c.id=u.chapter_id where c.book_id='$BOOK'") + say " units_total = $units chapters = $ch" + [ "$ch" = "$want_chapters" ] || die "the slice is not the pre-registered one: $ch chapters, expected $want_chapters — STOP before paying" + [ -z "$want_units" ] || [ "$units" = "$want_units" ] || die "units_total is $units, the pre-registration says $want_units — STOP before paying" + [ "$funded" = "True" ] || die "term_consistency_funded is $funded: the hold does not carry the book bond, and the run would quietly lose its terminology consolidation — STOP before paying" + say "OPTIONS ACCEPTED: $ch chapters, $units units, the bond is funded" +} + +step_start() { + local out + out=$(api -H 'Content-Type: application/json' -d '{"stop_for_signing": true, "chapters": 3}' "$ADDR/v0/books/$BOOK/runs") + say "$out" + local id; id=$(printf '%s' "$out" | python3 -c 'import sys,json;print(json.load(sys.stdin)["id"])') || die "no run id" + remember RUN "$id" +} + +# watch prints the two numbers the pack asks for on two DIFFERENT axes — the engine's own request log +# and the platform's materialised journal — plus the money and the systemd unit. A growing log is not +# work, so neither number is read alone. +step_watch() { + local dir; dir=$(sql "select workdir from books where id='$BOOK'") + local rl=0 hits=0 cost=0 + if [ -f "$dir/project.db" ]; then + read -r rl hits cost <<<"$(python3 - "$dir/project.db" <<'PY' +import sqlite3,sys +c=sqlite3.connect(f"file:{sys.argv[1]}?mode=ro",uri=True) +print(c.execute("select count(*) from request_log").fetchone()[0], + c.execute("select count(*) from request_log where tm_hit=1").fetchone()[0], + round(c.execute("select coalesce(sum(cost_usd),0) from request_log").fetchone()[0]*1e6)) +PY +)" + fi + local seq ev st paused + seq=$(sql "select coalesce(max(last_seq),0) from run_attempts where run_id='$RUN'") + ev=$(sql "select count(*) from book_events where book_id='$BOOK'") + st=$(sql "select status from runs where id='$RUN'") + paused=$(sql "select coalesce(paused_reason,'') from runs where id='$RUN'") + local unit; unit=$(sql "select unit_name from run_attempts where run_id='$RUN' order by attempt_no desc limit 1") + local active; active=$(systemctl --user show "$unit.service" -p ActiveState --value 2>/dev/null || echo "unknown") + say "$(date +%H:%M:%S) alive: request_log=$rl (tm_hit=$hits) · run_attempts.last_seq=$seq · book_events=$ev · status=$st · unit=$active · spend≈${cost}µUSD" + if [ -n "$paused" ]; then die "paused_reason=$paused — STOP and ping the orchestrator"; fi + if [ "${cost:-0}" -ge "$STOP_AT_MICRO" ]; then die "spend reached ${cost}µUSD ≥ ${STOP_AT_MICRO}µUSD — STOP and ping the orchestrator"; fi +} + +# evidence takes the copy the resume would destroy: `exportBank` is called again at the start of the +# next attempt and overwrites `.bank.json` atomically, so the proposals a signing screen was opened +# on survive only as a copy taken here. `.bank-stop.txt` is written NON-atomically (truncate first), +# so it is only taken while the run stands still. +step_evidence() { # $1 = label + local dir; dir=$(sql "select workdir from books where id='$BOOK'") + local out=$W/evidence/${1:-evidence} + mkdir -p "$out" + local st; st=$(sql "select status from runs where id='$RUN'") + say "run status while the copy is taken: $st" + [ "$st" = "translating" ] && die "the run is still moving: .bank-stop.txt is truncated before it is rewritten and would be read half-written" + local n=0 + for f in project.db.bank.json project.db.mined-signature.yaml project.db.bank-stop.txt project.db.auto-bank.yaml project.db.manifest.json events.jsonl; do + if [ -f "$dir/$f" ]; then cp "$dir/$f" "$out/"; sha256sum "$dir/$f" | sed 's/^/ /'; n=$((n+1)); fi + done + say " copied $n artefacts (control: the book directory holds $(ls -1 "$dir" | wc -l) entries)" + api "$ADDR/v0/books/$BOOK/bank" | head -c 400; echo + sql "select count(*) all_rows, count(*) filter (where status='approved') approved from bank_terms where book_id='$BOOK'" | sed 's/^/ bank_terms: /' +} + +step_sign() { # $1 = json document of corrections + api -H 'Content-Type: application/json' --data-binary "@$1" "$ADDR/v0/books/$BOOK/bank/corrections" -w '\nHTTP %{http_code}\n' +} + +step_resume() { api -H 'Content-Type: application/json' -d '{}' "$ADDR/v0/runs/$RUN/resume" -w '\nHTTP %{http_code}\n'; } + +step_export() { # $1 = format + local out id + out=$(api -H 'Content-Type: application/json' -d "{\"format\":\"$1\"}" "$ADDR/v0/books/$BOOK/exports") + id=$(printf '%s' "$out" | python3 -c 'import sys,json;print(json.load(sys.stdin)["id"])') || die "no export id: $out" + for _ in $(seq 1 30); do + sleep 2 + out=$(api "$ADDR/v0/books/$BOOK/exports/$id") + printf '%s' "$out" | grep -q '"state":"ready"' && break + printf '%s' "$out" | grep -q '"state":"failed"' && die "the export failed: $out" + done + say "$out" + mkdir -p "$W/evidence/file" + api "$ADDR/v0/books/$BOOK/exports/$id/content" -o "$W/evidence/file/book.$1" -w "download HTTP %{http_code} bytes=%{size_download}\n" + say "sha256: $(sha256sum "$W/evidence/file/book.$1")" + sql "select id,format,state,complete,size_bytes from exports where id='$id'" | sed 's/^/ exports row: /' + say '⚠ complete is read from Postgres and never from the API: the wire deliberately does not carry it.' +} + +step_collect() { # $1 = label + python3 "$(dirname "$0")/collect.py" --stand "$W" --dsn "$DSN" --book "${BOOK:-}" --run "${RUN:-}" \ + --sha "$FREEZE_SHA" --label "${1:-collect}" +} + +step_spread() { # the shipped-text measure the door has never had (unified backlog row 406) + local dir; dir=$(sql "select workdir from books where id='$BOOK'") + "$W/bin/tmctl" export --config "$dir/book.yaml" --json --pairs > "$W/evidence/pairs.json" 2>/dev/null \ + || die "the export verb refused" + python3 "$(dirname "$0")/spread.py" --pairs "$W/evidence/pairs.json" \ + --bank "$dir/project.db.bank.json" --signature "$dir/project.db.mined-signature.yaml" \ + --json "$W/evidence/spread.json" +} + +case "${1:-help}" in + gate) step_gate ;; + login) step_login ;; + grant) shift; step_grant "$@" ;; + intake) shift; step_intake "$@" ;; + options) shift; step_options "$@" ;; + start) step_start ;; + watch) step_watch ;; + evidence) shift; step_evidence "$@" ;; + sign) shift; step_sign "$@" ;; + resume) step_resume ;; + export) shift; step_export "$@" ;; + collect) shift; step_collect "$@" ;; + spread) step_spread ;; + help|*) sed -n '1,20p' "$0" ;; +esac diff --git a/eval/door_to_file/go.mod b/eval/door_to_file/go.mod new file mode 100644 index 00000000..27a3654e --- /dev/null +++ b/eval/door_to_file/go.mod @@ -0,0 +1,5 @@ +module textmachine/eval/door_to_file + +go 1.26 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/eval/door_to_file/go.sum b/eval/door_to_file/go.sum new file mode 100644 index 00000000..a62c313c --- /dev/null +++ b/eval/door_to_file/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/eval/door_to_file/spread.py b/eval/door_to_file/spread.py new file mode 100644 index 00000000..bedce0c7 --- /dev/null +++ b/eval/door_to_file/spread.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""How many renderings did one canonical term get in the SHIPPED text — a mechanical count. + +Why it exists: the engine measures term spread on the DRAFT, at the bank-mining stop, before the +editor wave, and only the human signing table ever reads it. Over the text a reader is finally +handed, no such measure exists at all (unified backlog row 406), so «consistent terms across a whole +book» — the owner's first priority — is today unfalsifiable by construction. + +This is a COUNT, not a judgement. It says «term X left the door in three shapes, in chapters 1, 2 +and 3»; whether any of the three is good translation is not its question. + +THE RULE IT USES, stated so a reader can disagree with it rather than guess: + +* Input is `tmctl export --json --pairs` — the sanctioned extraction of shipped text (invariant 8, + D39.5) — plus the book's known renderings (bank sidecar `.bank.json`, signature map, auto-bank). +* For a source term T, the units considered are those whose SOURCE contains T. Only there can a + rendering of T be expected, and only there is its absence meaningful. +* A rendering found in a unit's target is any known rendering of T, or a NEAR-VARIANT of one: the + same first `stem` characters with a tail differing by at most `--tail` characters. Russian + inflects, so «Ли Чанфэна» and «Ли Чанфэн» are ONE rendering; «Ли Чанфын» is another. +* Spread(T) = the number of distinct renderings, after that folding, observed across those units. + +Usage: + spread.py --pairs pairs.json --bank project.db.bank.json [--signature map.yaml] [--tail 2] [--stem 4] + spread.py --selftest +""" + +import argparse +import json +import re +import sys +from collections import defaultdict + +CAP = re.compile(r"[А-ЯЁ][а-яёА-ЯЁ\-]+(?:\s+[А-ЯЁ][а-яёА-ЯЁ\-]+)*") + + +def lcp(a, b): + n = 0 + for x, y in zip(a, b): + if x != y: + break + n += 1 + return n + + +def same_rendering(a, b, stem, tail): + """One rendering inflected, or two different renderings? + + ⚠ THE RULE IS ABOUT WHERE THE DIFFERENCE IS, not how big it is, and the self-test is what forced + that: Russian inflection only ever changes the END of a word, while a different transliteration + changes something INSIDE it. So two strings are the same rendering when their common prefix + reaches all but `tail` characters of the shorter one — «Чжао Сяомань»/«Чжао Сяоманя» agree on + eleven of twelve — and are different when it does not, which is what separates «Ли Чанфэн» from + «Ли Чанфына» (they part at the seventh character of nine). The length allowance is separate and + generous, because an inflection may add two or three letters and a transliteration rarely does. + """ + a, b = a.strip(), b.strip() + if a.lower() == b.lower(): + return True + short = min(len(a), len(b)) + if short < stem: + return a.lower() == b.lower() + return lcp(a.lower(), b.lower()) >= short - tail and abs(len(a) - len(b)) <= 4 + + +def known_renderings(bank_path, signature_path): + """Every rendering this book has ever recorded for a source term, from the engine's own + sidecars. Both the signed bank and the proposals: a term signed late may have shipped under its + proposal earlier, which is exactly the case this instrument exists to find.""" + out = defaultdict(set) + if bank_path: + doc = json.load(open(bank_path, encoding="utf-8")) + for section in ("terms", "proposed"): + for t in doc.get(section) or []: + if t.get("src") and t.get("dst"): + out[t["src"]].add(t["dst"]) + if signature_path: + src = dst = None + for line in open(signature_path, encoding="utf-8"): + m = re.match(r"\s*-?\s*src:\s*(\S+)", line) + if m: + src = m.group(1).strip('"\'') + m = re.match(r"\s*dst:\s*(.+)", line) + if m and src: + dst = m.group(1).strip().strip('"\'') + if dst: + out[src].add(dst) + return out + + +def near(cap, known, stem): + """Is this capitalised token PLAUSIBLY a rendering of the same term — the same name spelled + another way, or the same spelling inflected? + + ⚠ Deliberately looser than `same_rendering`, and the two must not be the same test. The loose + one decides what to LOOK AT (a token that shares most of a known rendering's prefix); the strict + one decides what to COUNT AS ONE. Using the strict test for both made the instrument blind to + exactly what it exists to find: «Ли Чанфына» stopped being a variant of «Ли Чанфэн» and the + second shape vanished from the tally instead of being counted as a second shape.""" + short = min(len(cap), len(known)) + if short < stem: + return False + return lcp(cap.lower(), known.lower()) >= max(stem, (short * 3 + 4) // 5) and abs(len(cap) - len(known)) <= 5 + + +def variants_in(target, renderings, stem, tail): + """Which renderings of this term the target shows — known ones and near-variants of them.""" + found = set() + for r in renderings: + if r and r in target: + found.add(r) + for cap in CAP.findall(target): + for r in renderings: + if r and cap != r and near(cap, r, stem): + found.add(cap) + # fold the found set: inflections of one rendering are one rendering + folded = [] + for f in sorted(found, key=len): + if not any(same_rendering(f, g, stem, tail) for g in folded): + folded.append(f) + return folded + + +def run(pairs, renderings, stem, tail): + """pairs: [{chapter, unit, source, target}]. Returns rows sorted by spread.""" + rows = [] + for src, known in sorted(renderings.items()): + seen = {} + units = 0 + for p in pairs: + if src not in (p.get("source") or ""): + continue + units += 1 + for v in variants_in(p.get("target") or "", known, stem, tail): + # ⚠ FOLDED ACROSS UNITS, not only inside one. Folding per unit and keying the tally + # on the raw string counted «Чжао Сяомань» in chapter 1 and «Чжао Сяоманя» in + # chapter 2 as two shapes — the instrument's own self-test caught it. + key = next((k for k in seen if same_rendering(k, v, stem, tail)), v) + seen.setdefault(key, set()).add(p.get("chapter")) + if units == 0: + continue + rows.append({"src": src, "units_with_source": units, "renderings": {k: sorted(v) for k, v in seen.items()}, + "spread": len(seen)}) + rows.sort(key=lambda r: (-r["spread"], r["src"])) + return rows + + +def selftest(): + """A planted text: one term shipped in two shapes, one term shipped consistently (inflected), + one term never rendered at all. The instrument must say 2, 1 and 0 — the third is what tells a + real zero from an instrument that found nothing because it looked nowhere.""" + renderings = {"李长风": {"Ли Чанфэн"}, "赵小满": {"Чжао Сяомань"}, "青锋剑": {"Цинфэн"}} + pairs = [ + {"chapter": 1, "source": "李长风走在山路上。赵小满跟着。", "target": "Ли Чанфэн шёл по горной тропе. Чжао Сяомань шёл следом."}, + {"chapter": 2, "source": "李长风看着赵小满。", "target": "Ли Чанфына видел Чжао Сяоманя."}, + {"chapter": 3, "source": "青锋剑在手。", "target": "Меч был в руке."}, + # a third unit for 赵小满, inflected further: the length allowance must not split it off + {"chapter": 3, "source": "赵小满笑了。", "target": "С Чжао Сяоманем всё было ясно."}, + ] + rows = run(pairs, renderings, stem=4, tail=1) + got = {r["src"]: r["spread"] for r in rows} + want = {"李长风": 2, "赵小满": 1, "青锋剑": 0} + ok = got == want + print("self-test rows:") + for r in rows: + print(" ", r) + print(f"self-test: got {got} want {want} → {'PASS' if ok else 'FAIL'}") + print(" (control: the instrument was given 3 terms and 3 units; a silent instrument would print" + " spread 0 for all three, which is why one term is planted with two shapes and one with none)") + return 0 if ok else 1 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--pairs") + ap.add_argument("--bank") + ap.add_argument("--signature") + ap.add_argument("--stem", type=int, default=4) + ap.add_argument("--tail", type=int, default=1) + ap.add_argument("--json", dest="out") + ap.add_argument("--selftest", action="store_true") + a = ap.parse_args() + if a.selftest: + sys.exit(selftest()) + if not a.pairs: + ap.error("--pairs is required (tmctl export --json --pairs)") + doc = json.load(open(a.pairs, encoding="utf-8")) + pairs = normalise_pairs(doc) + renderings = known_renderings(a.bank, a.signature) + rows = run(pairs, renderings, a.stem, a.tail) + multi = [r for r in rows if r["spread"] >= 2] + withr = [r for r in rows if r["spread"] >= 1] + print(f"units read: {len(pairs)} canonical terms known: {len(renderings)}") + print(f"terms whose source appears in at least one unit: {len(rows)}") + print(f"terms rendered at all: {len(withr)} (control: terms considered {len(rows)})") + print(f"TERMS SHIPPED IN MORE THAN ONE SHAPE: {len(multi)} (control: terms rendered at all {len(withr)})") + for r in multi: + print(f" {r['src']}: {r['spread']} shapes over {r['units_with_source']} units") + for shape, chapters in r["renderings"].items(): + print(f" {shape!r} in chapters {chapters}") + if a.out: + json.dump(rows, open(a.out, "w", encoding="utf-8"), ensure_ascii=False, indent=1) + print(f"rows written to {a.out}") + + +def normalise_pairs(doc): + """`tmctl export --json --pairs` shape, kept tolerant: the fields this needs are the chapter + number and the two texts, wherever the envelope puts them.""" + # `tm-export-v1` puts them under `chunks`, with `final_text` for the shipped side and `source` + # for the other. Named first and explicitly, so a change of envelope is a loud KeyError rather + # than an empty list that reads as «this book has no terms out of line». + if isinstance(doc, dict): + for key in ("chunks", "pairs", "units", "items", "rows"): + if isinstance(doc.get(key), list): + doc = doc[key] + break + out = [] + for p in doc if isinstance(doc, list) else []: + out.append({ + "chapter": p.get("chapter", p.get("chapter_number", p.get("ch"))), + "unit": p.get("chunk_idx", p.get("unit", p.get("unit_id", p.get("id")))), + "source": p.get("source", p.get("src", "")) or "", + "target": p.get("final_text", p.get("target", p.get("translation", p.get("dst", "")))) or "", + }) + return out + + +if __name__ == "__main__": + main() diff --git a/eval/door_to_file/stub/main.go b/eval/door_to_file/stub/main.go new file mode 100644 index 00000000..826b8272 --- /dev/null +++ b/eval/door_to_file/stub/main.go @@ -0,0 +1,137 @@ +// Command stub serves the deployment's zero-cost pair over the local provider's address, so the +// whole door-to-file chain can be exercised before a cent is spent. It is the stand's own stub — +// the P9 live-probe's fake_provider.py as the platform's live tests carry it +// (platform/internal/runner/translate_resnapshot_live_test.go, fakeProvider/proseFor) — lifted out +// of the test binary so a driver can raise it, with one addition the tests do not need: every call +// is appended to a JSONL log, which is the only trace that says how many calls the chain really +// made and with which role. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + "regexp" + "strings" + "sync" + "time" +) + +var ( + addr = flag.String("addr", "127.0.0.1:11434", "address of the local provider") + logPath = flag.String("log", "", "append one JSON line per call here") +) + +var ( + mu sync.Mutex + calls int +) + +func main() { + flag.Parse() + ln, err := net.Listen("tcp", *addr) + if err != nil { + log.Fatalf("the local provider address is busy (another stand?): %v", err) + } + fmt.Printf("stub listening on %s pid=%d\n", *addr, os.Getpid()) + srv := &http.Server{Handler: http.HandlerFunc(serve)} + log.Fatal(srv.Serve(ln)) +} + +func serve(w http.ResponseWriter, r *http.Request) { + var req struct { + Model string `json:"model"` + Messages []struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"messages"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + var system, user strings.Builder + for _, m := range req.Messages { + switch m.Role { + case "system": + system.WriteString(m.Content) + case "user": + user.WriteString(m.Content) + } + } + content := proseFor(system.String(), user.String()) + mu.Lock() + calls++ + n := calls + mu.Unlock() + record(map[string]any{ + "at": time.Now().UTC().Format(time.RFC3339Nano), "seq": n, "path": r.URL.Path, + "model": req.Model, "role": roleOf(system.String()), + "system_bytes": system.Len(), "user_bytes": user.Len(), "out_bytes": len(content), + }) + resp := map[string]any{ + "id": "stub-" + fmt.Sprint(n), "object": "chat.completion", "model": req.Model, + "choices": []map[string]any{{ + "index": 0, + "message": map[string]any{"role": "assistant", "content": content}, + "finish_reason": "stop", + }}, + "usage": map[string]any{"prompt_tokens": 100, "completion_tokens": max(1, len(content)/3), + "total_tokens": 100 + max(1, len(content)/3)}, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) +} + +func record(row map[string]any) { + if *logPath == "" { + return + } + f, err := os.OpenFile(*logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return + } + defer f.Close() + line, _ := json.Marshal(row) + _, _ = f.Write(append(line, '\n')) +} + +// roleOf names which of the chain's roles asked, by the same marks proseFor answers to. +func roleOf(system string) string { + switch { + case strings.Contains(system, "терминолог"): + return "terminologist" + case strings.Contains(system, "классиф") || strings.Contains(system, "класс (name | place | title | term)"): + return "classifier" + default: + return "prose" + } +} + +var termKeys = regexp.MustCompile(`(?m)^key: (.+)$`) + +func proseFor(system, user string) string { + if strings.Contains(system, "терминолог") { + var b strings.Builder + for i, m := range termKeys.FindAllStringSubmatch(user, -1) { + fmt.Fprintf(&b, "%s\tЗаглушка-%d\t55\n", m[1], i+1) + } + return b.String() + } + if strings.Contains(system, "классиф") || strings.Contains(system, "класс (name | place | title | term)") { + var b strings.Builder + for _, m := range termKeys.FindAllStringSubmatch(user, -1) { + fmt.Fprintf(&b, "%s\tname\n", m[1]) + } + return b.String() + } + const filler = "Фан Юань неторопливо шёл по горной тропе, и ветер приносил запах трав. " + + "Старейшина посмотрел на него и тяжело вздохнул, вспоминая давние годы. " + + "В долине клубился туман, и где-то вдалеке кричала ночная птица. " + out := filler + for len(out) < len(user) { + out += filler + } + return out[:max(len(user), len(filler))] +} diff --git a/eval/door_to_file/zeropipe/main.go b/eval/door_to_file/zeropipe/main.go new file mode 100644 index 00000000..7c5ed8e9 --- /dev/null +++ b/eval/door_to_file/zeropipe/main.go @@ -0,0 +1,221 @@ +// Command zeropipe renders a deployment's shipping pipeline onto the deployment's zero-cost pair, +// so the chain can be smoked without buying anything. It is a port of the platform's own renderer +// (platform/internal/runner/bankapply_live_test.go, zeroCostPipeline) rather than the sed over +// `model:` the stand recipe gives: the gates carry models too, and on a pipeline-c1 template the +// terminologist and the classifier resolve to a paid provider with every stage already zeroed. +// +// It also answers the question a rendered file cannot answer about itself: -check counts, in both +// the shipping file and the rendered one, the models this deployment would have to pay for. A +// silent renderer and a clean pipeline look the same until the paid count beside the zero is +// printed too. +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +type modelsFile struct { + Providers map[string]struct { + Kind string `yaml:"kind"` + } `yaml:"providers"` + Models map[string]struct { + Provider string `yaml:"provider"` + } `yaml:"models"` +} + +func main() { + pipelinePath := flag.String("pipeline", "", "the deployment's shipping pipeline") + modelsPath := flag.String("models", "", "the models registry the book points at") + out := flag.String("out", "", "where to write the rendered zero-cost pipeline") + flag.Parse() + if *pipelinePath == "" || *modelsPath == "" || *out == "" { + die("usage: zeropipe -pipeline -models -out ") + } + + models := readModels(*modelsPath) + free, paid := partition(models) + if free == "" { + die("this deployment's %s declares no provider of kind `local`: the probe would have to buy its calls", *modelsPath) + } + + raw, err := os.ReadFile(*pipelinePath) + if err != nil { + die("the pipeline cannot be read (%s): %v", *pipelinePath, err) + } + var pipe map[string]any + if err := yaml.Unmarshal(raw, &pipe); err != nil { + die("%v", err) + } + + stages, ok := pipe["stages"].([]any) + if !ok || len(stages) == 0 { + die("the pipeline declares no stages: %s", *pipelinePath) + } + for _, s := range stages { + stage, ok := s.(map[string]any) + if !ok { + die("a stage of %s is not a mapping", *pipelinePath) + } + stage["model"] = free + delete(stage, "escalate_to") + delete(stage, "label_models") + } + // The gates carry models of their own, and both of them: `classify_model` falls back to `model` + // only when it is empty, so a pipeline that sets it would resolve a paid model here with `model` + // already zeroed. + if gates, ok := pipe["gates"].(map[string]any); ok { + for _, g := range gates { + gate, ok := g.(map[string]any) + if !ok { + continue + } + if _, has := gate["model"]; has { + gate["model"] = free + } + if _, has := gate["classify_model"]; has { + gate["classify_model"] = free + } + delete(gate, "escalate_to") + } + } + if esc, ok := pipe["escalation"].(map[string]any); ok { + esc["budget_usd"] = 0 + delete(esc, "chains") + } + // The bank contour's one deployment-provided input is resolved against the directory of the + // pipeline file, and this render is about to move: absolutise it, and refuse rather than let the + // engine die at its write-path guard with a message that reads like somebody else's defect. + if mining, ok := pipe["mining"].(map[string]any); ok { + if rel, ok := mining["contrast_path"].(string); ok && rel != "" { + abs := rel + if !filepath.IsAbs(abs) { + abs = filepath.Join(filepath.Dir(*pipelinePath), abs) + } + if _, err := os.Stat(abs); err != nil { + die("the pipeline enables the bank contour and names %s, which is not on this host", abs) + } + mining["contrast_path"] = abs + } + } + + rendered, err := yaml.Marshal(pipe) + if err != nil { + die("%v", err) + } + if err := os.WriteFile(*out, rendered, 0o644); err != nil { + die("%v", err) + } + + var shipping map[string]any + if err := yaml.Unmarshal(raw, &shipping); err != nil { + die("%v", err) + } + var check map[string]any + if err := yaml.Unmarshal(rendered, &check); err != nil { + die("the rendered pipeline does not parse back: %v", err) + } + shippingHits := hits(shipping, paid) + renderedHits := hits(check, paid) + fmt.Printf("free model: %s\n", free) + fmt.Printf("paid models this deployment declares: %d (%s)\n", len(paid), strings.Join(paid, " ")) + fmt.Printf("paid models REACHABLE in %s: %d\n", filepath.Base(*pipelinePath), total(shippingHits)) + for _, name := range sorted(shippingHits) { + fmt.Printf(" %s: %d\n", name, shippingHits[name]) + } + fmt.Printf("paid models REACHABLE in %s: %d\n", filepath.Base(*out), total(renderedHits)) + for _, name := range sorted(renderedHits) { + fmt.Printf(" %s: %d\n", name, renderedHits[name]) + } + if total(renderedHits) != 0 { + die("the rendered pipeline still names a paid model: the probe would buy its calls") + } + if total(shippingHits) == 0 { + die("the shipping pipeline names no paid model either: this check proves nothing, and the "+ + "zero in the render is the instrument staying silent rather than the render being clean (%s)", *pipelinePath) + } +} + +func readModels(path string) modelsFile { + raw, err := os.ReadFile(path) + if err != nil { + die("the models file cannot be read (%s): %v", path, err) + } + var m modelsFile + if err := yaml.Unmarshal(raw, &m); err != nil { + die("%v", err) + } + return m +} + +// partition names the cheapest truth about this registry: which model costs nothing because its +// provider runs on this host, and which ones a call would be billed for. +func partition(m modelsFile) (free string, paid []string) { + for name, model := range m.Models { + if m.Providers[model.Provider].Kind == "local" { + if free == "" || name < free { + free = name + } + continue + } + paid = append(paid, name) + } + sort.Strings(paid) + return free, paid +} + +// hits counts paid models where they are REACHABLE — as values in the parsed configuration, not as +// text. Counting text would count the comments, and a rendered file has none: the render would look +// clean the moment the comments were dropped, whether or not a paid model still stood in a gate. +func hits(node any, names []string) map[string]int { + out := map[string]int{} + var walk func(any) + walk = func(n any) { + switch v := n.(type) { + case map[string]any: + for _, child := range v { + walk(child) + } + case []any: + for _, child := range v { + walk(child) + } + case string: + for _, name := range names { + if v == name { + out[name]++ + } + } + } + } + walk(node) + return out +} + +func total(m map[string]int) int { + sum := 0 + for _, v := range m { + sum += v + } + return sum +} + +func sorted(m map[string]int) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func die(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +}