From aa00339abba4615556c880b955a250e08b60959f Mon Sep 17 00:00:00 2001 From: "Claude (backend session)" Date: Sun, 5 Jul 2026 08:13:04 +0300 Subject: [PATCH] Land memory bank v2 (glossary hot path, injection surface, F1 materialization, decl-aware post-check) with 8 self-review fixes --- backend/cmd/tmctl/main.go | 34 + backend/internal/config/book.go | 14 +- backend/internal/config/pipeline.go | 13 + backend/internal/pipeline/data/trad2simp.txt | 563 +++++++++++++++++ backend/internal/pipeline/disposition.go | 9 + backend/internal/pipeline/memnorm.go | 182 ++++++ backend/internal/pipeline/memnorm_test.go | 116 ++++ backend/internal/pipeline/memory.go | 592 ++++++++++++++++++ backend/internal/pipeline/memory_e1_test.go | 177 ++++++ backend/internal/pipeline/memory_f1_test.go | 99 +++ backend/internal/pipeline/memory_test.go | 372 +++++++++++ backend/internal/pipeline/mempostcheck.go | 109 ++++ backend/internal/pipeline/memseed.go | 253 ++++++++ backend/internal/pipeline/memseed_test.go | 162 +++++ backend/internal/pipeline/render.go | 36 +- .../internal/pipeline/render_memory_test.go | 93 +++ backend/internal/pipeline/runner.go | 252 +++++++- .../internal/pipeline/runner_memory_test.go | 245 ++++++++ backend/internal/pipeline/runner_test.go | 17 +- backend/internal/store/glossary.go | 322 ++++++++++ backend/internal/store/glossary_test.go | 153 +++++ backend/internal/store/migrate.go | 92 +++ docs/PROGRESS.md | 24 + 23 files changed, 3886 insertions(+), 43 deletions(-) create mode 100644 backend/internal/pipeline/data/trad2simp.txt create mode 100644 backend/internal/pipeline/memnorm.go create mode 100644 backend/internal/pipeline/memnorm_test.go create mode 100644 backend/internal/pipeline/memory.go create mode 100644 backend/internal/pipeline/memory_e1_test.go create mode 100644 backend/internal/pipeline/memory_f1_test.go create mode 100644 backend/internal/pipeline/memory_test.go create mode 100644 backend/internal/pipeline/mempostcheck.go create mode 100644 backend/internal/pipeline/memseed.go create mode 100644 backend/internal/pipeline/memseed_test.go create mode 100644 backend/internal/pipeline/render_memory_test.go create mode 100644 backend/internal/pipeline/runner_memory_test.go create mode 100644 backend/internal/store/glossary.go create mode 100644 backend/internal/store/glossary_test.go diff --git a/backend/cmd/tmctl/main.go b/backend/cmd/tmctl/main.go index bd5406a..6977b60 100644 --- a/backend/cmd/tmctl/main.go +++ b/backend/cmd/tmctl/main.go @@ -180,6 +180,40 @@ func report(cfgPath string) error { f.Chapter, f.ChunkIdx, f.Stage, f.Disposition, f.FlagReason, f.Attempts, f.CostUSD, f.Detail) } + // Memory section (шаг 4): the per-chunk retrieval-state, surfaced so silent glossary + // degradation is LOUD. The aggregate line always prints when a glossary is in use; + // each chunk with a post-check miss is listed (the flagger-mode signal — the model + // ignored an approved term, or we injected the wrong dst) with the offending src→dst. + states, err := r.Store.RetrievalStatesForBook(r.Book.BookID) + if err != nil { + return err + } + if len(states) > 0 { + var injected, sticky, ambiguous, spoiler, evicted, misses int + for _, rs := range states { + injected += rs.NExactHits + sticky += rs.NSticky + ambiguous += rs.NAmbiguousFlagged + spoiler += rs.NSpoilerBlocked + evicted += rs.NEvicted + misses += rs.NPostcheckMiss + } + fmt.Printf("\n=== ПАМЯТЬ (retrieval-state) ===\n") + fmt.Printf("инъекций(точных)=%d sticky=%d ambiguous=%d спойлер-блок=%d вытеснено=%d post-check-промахов=%d\n", + injected, sticky, ambiguous, spoiler, evicted, misses) + printed := false + for _, rs := range states { + if rs.NPostcheckMiss == 0 { + continue + } + if !printed { + fmt.Printf("%-4s %-6s %6s %s\n", "ch", "chunk", "misses", "detail (src→dst не найден в выводе)") + printed = true + } + fmt.Printf("%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NPostcheckMiss, rs.PostcheckDetail) + } + } + committed, reserved, err := r.Store.SpentUSD(r.Book.BookID) if err != nil { return err diff --git a/backend/internal/config/book.go b/backend/internal/config/book.go index 0e1b62e..00f879e 100644 --- a/backend/internal/config/book.go +++ b/backend/internal/config/book.go @@ -40,7 +40,13 @@ type Book struct { ModelsFile string `yaml:"models"` SourceFile string `yaml:"source_file"` // Фаза 0: один файл = один чанк ProjectDB string `yaml:"project_db"` // default: .db рядом с book.yaml - Ceilings BookCap `yaml:"ceilings"` + // GlossarySeed is the optional path to the manual glossary seed YAML (memory v2, + // шаг 4). Its curated approved/draft terms + the classified ruby readings are the + // deterministic inputs the glossary is REPLACED from each run; editing it changes the + // materialized approved set → a loud --resnapshot (F1). Empty = ruby-seed only (or an + // empty glossary → the injection is inert, a safe no-op). + GlossarySeed string `yaml:"glossary_seed"` + Ceilings BookCap `yaml:"ceilings"` } // BookCap are the ledger admission limits (Р7: потолок $ на книгу/день). @@ -70,6 +76,7 @@ func LoadBook(path string) (*Book, error) { b.Pipeline = resolve(b.Pipeline) b.ModelsFile = resolve(b.ModelsFile) b.SourceFile = resolve(b.SourceFile) + b.GlossarySeed = resolve(b.GlossarySeed) if b.ProjectDB == "" { b.ProjectDB = filepath.Join(dir, b.BookID+".db") } else { @@ -98,6 +105,11 @@ func LoadBook(path string) (*Book, error) { } else if _, err := os.Stat(b.SourceFile); err != nil { bad("source_file %s is not readable: %v", b.SourceFile, err) } + if b.GlossarySeed != "" { + if _, err := os.Stat(b.GlossarySeed); err != nil { + bad("glossary_seed %s is not readable: %v", b.GlossarySeed, err) + } + } if b.Ceilings.BookUSD <= 0 && b.Ceilings.DayUSD <= 0 { bad("ceilings: at least one of book_usd/day_usd must be set (ledger без потолка запрещён Р7)") } diff --git a/backend/internal/config/pipeline.go b/backend/internal/config/pipeline.go index 667a8a7..4531d55 100644 --- a/backend/internal/config/pipeline.go +++ b/backend/internal/config/pipeline.go @@ -81,6 +81,19 @@ type Stage struct { // Gates is the QA-gate config skeleton (пороги полигона; исполнение — Фаза 1). type Gates struct { Coverage CoverageGate `yaml:"coverage"` + Glossary GlossaryGate `yaml:"glossary"` +} + +// GlossaryGate controls the memory-bank post-check (E1). The post-check ALWAYS runs +// (observability — it records misses into the retrieval-state), converting silent +// glossary drift into a loud, visible signal. PostcheckGate promotes a miss from a +// mere record to a DISPOSITION flag (the chunk is flagged, downstream stages skip): +// opt-in (default false = flagger), flipped on only AFTER the owner validates the +// false-flag rate on real chapters (E1) — exactly the coverage gate's opt-in discipline +// (D12 Q4). A naive/under-filled decl false-flags (research/14 §2), so a premature hard +// gate would flag-storm and train editors to ignore it. +type GlossaryGate struct { + PostcheckGate bool `yaml:"postcheck_gate"` } // CoverageGate v1 (Р7): метрика — символы без пробелов; нижняя граница — diff --git a/backend/internal/pipeline/data/trad2simp.txt b/backend/internal/pipeline/data/trad2simp.txt new file mode 100644 index 0000000..8e8cf20 --- /dev/null +++ b/backend/internal/pipeline/data/trad2simp.txt @@ -0,0 +1,563 @@ +# Traditional → Simplified Chinese character map (A4 normalization, memory bank v2). +# +# Format: one mapping per line, " " (whitespace-separated, single rune +# each). '#' comments and blank lines are ignored. Loaded once via go:embed and +# applied character-by-character in memnorm.go, SYMMETRICALLY to glossary keys and +# to chunk text — so a key written in Simplified still matches a Traditional source +# surface (and vice versa), closing the "тихо пусто" orthography hole (registry A4). +# +# PROVENANCE / COMPLETENESS (read before trusting): this is a curated HIGH-FREQUENCY +# set, NOT the full OpenCC TSCharacters table. It was authored offline (no OpenCC/ +# Unihan data available on the stand) with accuracy prioritized over size — a WRONG +# trad→simp mapping silently corrupts matching, so only high-confidence single-char +# mappings are included; phrase-level OpenCC (one-to-many, e.g. 後→后 vs 后→后) is +# deliberately out of scope for v1. The table's content is versioned by +# memoryNormVersion (memnorm.go), folded into the snapshot via memoryVersion() — so +# COMPLETING it with the full OpenCC data is a data-file swap that fires a loud +# --resnapshot, never a silent behaviour change. The acceptance book is ja→ru, where +# trad→simp is not load-bearing (NFKC + kana-fold + ruby carry ja); this table matters +# for zh sources and is the one tracked data-completion item before a zh acceptance run. +# Known live-bug cases from research/14 (爺→爷 etc.) and every retrieval_bench trap +# character are covered and asserted in memnorm_test.go. + +# --- referenced by the eval specs / research/14 live bugs (must stay covered) --- +魯 鲁 +鎮 镇 +趙 赵 +蕭 萧 +煉 炼 +門 门 +闆 板 +莊 庄 +錢 钱 +陳 陈 +萬 万 +舊 旧 +臉 脸 +爺 爷 +羅 罗 +莊 庄 + +# --- 訁 (speech) radical --- +說 说 +話 话 +語 语 +讀 读 +誰 谁 +課 课 +談 谈 +請 请 +謝 谢 +詩 诗 +詞 词 +試 试 +記 记 +訪 访 +許 许 +論 论 +訴 诉 +譯 译 +議 议 +護 护 +譽 誉 +讚 赞 +變 变 +讓 让 +認 认 +識 识 +訊 讯 +訂 订 +計 计 +訓 训 +設 设 +詳 详 +誠 诚 +語 语 +誤 误 +說 说 +誦 诵 +調 调 +諒 谅 +談 谈 +謎 谜 +講 讲 +謙 谦 +謠 谣 +證 证 +評 评 + +# --- 貝 (shell/money) radical --- +貝 贝 +財 财 +貨 货 +貧 贫 +貪 贪 +責 责 +貫 贯 +貴 贵 +買 买 +費 费 +貼 贴 +賀 贺 +貿 贸 +資 资 +賊 贼 +賓 宾 +賦 赋 +賠 赔 +賣 卖 +賤 贱 +賬 账 +賭 赌 +賴 赖 +購 购 +賽 赛 +贈 赠 +贏 赢 +賺 赚 +賞 赏 +賢 贤 +質 质 +贖 赎 +貸 贷 +賃 赁 +賄 贿 +賂 赂 + +# --- 車 (vehicle) radical --- +車 车 +輪 轮 +軍 军 +較 较 +輕 轻 +轉 转 +輸 输 +輩 辈 +輝 辉 +軟 软 +軌 轨 +轟 轰 +輔 辅 +輛 辆 +載 载 +轎 轿 + +# --- 門 (gate) radical --- +開 开 +閉 闭 +問 问 +間 间 +閃 闪 +閒 闲 +閑 闲 +閣 阁 +閩 闽 +閱 阅 +闊 阔 +關 关 +闖 闯 +闕 阙 +悶 闷 +聞 闻 + +# --- 食 (food) radical --- +飛 飞 +風 风 +飯 饭 +飲 饮 +餓 饿 +館 馆 +養 养 +餘 余 +餅 饼 +饑 饥 +饞 馋 +飄 飘 + +# --- 金 (metal) radical --- +鐘 钟 +鐵 铁 +銀 银 +銅 铜 +鋼 钢 +錯 错 +鎖 锁 +鏡 镜 +鑰 钥 +鑽 钻 +鍋 锅 +鍵 键 +錄 录 +針 针 +釘 钉 +鈴 铃 +鉛 铅 +銷 销 +鋒 锋 +鏈 链 +鑼 锣 +鈔 钞 +鑲 镶 + +# --- 頁 (head/page) radical --- +頁 页 +頂 顶 +項 项 +順 顺 +須 须 +預 预 +頑 顽 +頓 顿 +頗 颇 +領 领 +頸 颈 +頻 频 +顆 颗 +題 题 +額 额 +顏 颜 +願 愿 +類 类 +顧 顾 +顯 显 +顫 颤 + +# --- 馬 (horse) radical --- +馬 马 +駕 驾 +駛 驶 +駐 驻 +駝 驼 +騎 骑 +騙 骗 +驕 骄 +驗 验 +驚 惊 +驢 驴 +駭 骇 +駱 骆 +騰 腾 + +# --- 鳥 (bird) radical --- +鳥 鸟 +鴨 鸭 +鵝 鹅 +鴻 鸿 +鵬 鹏 +鶴 鹤 +鷹 鹰 +鷗 鸥 +鴉 鸦 +鳴 鸣 +鴿 鸽 + +# --- 魚 (fish) radical --- +魚 鱼 +鮮 鲜 +鯨 鲸 +鯉 鲤 +鱗 鳞 + +# --- 糸 (silk/thread) radical --- +紀 纪 +約 约 +紅 红 +紙 纸 +級 级 +納 纳 +紛 纷 +純 纯 +紗 纱 +綱 纲 +網 网 +綿 绵 +線 线 +練 练 +組 组 +細 细 +終 终 +結 结 +絕 绝 +給 给 +絡 络 +統 统 +綠 绿 +維 维 +綜 综 +緊 紧 +緒 绪 +編 编 +緩 缓 +締 缔 +緣 缘 +縣 县 +縫 缝 +縮 缩 +績 绩 +繩 绳 +繫 系 +繪 绘 +繼 继 +續 续 +纏 缠 +紋 纹 +綁 绑 +繳 缴 +經 经 +絲 丝 +繡 绣 + +# --- water / fire / misc very-high-frequency --- +漢 汉 +灣 湾 +濟 济 +潔 洁 +澤 泽 +濕 湿 +灑 洒 +滅 灭 +溫 温 +準 准 +淚 泪 +減 减 +滾 滚 +滿 满 +漁 渔 +漸 渐 +潑 泼 +濁 浊 +瀉 泻 +決 决 +沒 没 +沖 冲 +況 况 +淨 净 +涼 凉 +凍 冻 +熱 热 +煩 烦 +燒 烧 +燈 灯 +燦 灿 +爛 烂 +營 营 +爐 炉 +煙 烟 +熾 炽 +燭 烛 +氣 气 +氫 氢 + +# --- 犭 (animal) radical --- +獨 独 +獲 获 +獸 兽 +獻 献 +獄 狱 +猶 犹 +狹 狭 +獅 狮 +獵 猎 +豬 猪 + +# --- 阝 (mound/city) radical --- +陰 阴 +陽 阳 +階 阶 +隊 队 +隨 随 +險 险 +隱 隐 +際 际 +陸 陆 +陝 陕 +陣 阵 +鄉 乡 +鄰 邻 +鄭 郑 +鄧 邓 + +# --- 木 (wood) radical --- +條 条 +極 极 +樓 楼 +標 标 +樣 样 +樹 树 +橋 桥 +機 机 +檢 检 +櫃 柜 +欄 栏 +權 权 +檔 档 +樸 朴 +橫 横 +檻 槛 +櫥 橱 +楊 杨 +榮 荣 +構 构 +槍 枪 +樑 梁 +櫻 樱 +樂 乐 +業 业 +東 东 + +# --- structural / very common standalone --- +國 国 +學 学 +會 会 +對 对 +時 时 +見 见 +這 这 +們 们 +個 个 +來 来 +為 为 +麼 么 +頭 头 +過 过 +還 还 +進 进 +種 种 +應 应 +關 关 +點 点 +動 动 +於 于 +實 实 +發 发 +現 现 +長 长 +兒 儿 +東 东 +與 与 +舉 举 +興 兴 +覺 觉 +觀 观 +歡 欢 +勸 劝 +農 农 +辦 办 +務 务 +勞 劳 +勢 势 +勝 胜 +勵 励 +單 单 +嚴 严 +嘗 尝 +嘆 叹 +嚇 吓 +嚮 向 +圖 图 +團 团 +圓 圆 +園 园 +圍 围 +囑 嘱 +專 专 +將 将 +尋 寻 +導 导 +屬 属 +層 层 +屢 屡 +廣 广 +廠 厂 +廢 废 +廟 庙 +廚 厨 +廈 厦 +廳 厅 +華 华 +葉 叶 +蓋 盖 +蒼 苍 +薦 荐 +薩 萨 +藍 蓝 +藝 艺 +藥 药 +蘇 苏 +蘭 兰 +蘋 苹 +蟲 虫 +傷 伤 +傳 传 +債 债 +傾 倾 +僅 仅 +僕 仆 +僑 侨 +價 价 +儀 仪 +億 亿 +儉 俭 +儲 储 +償 偿 +優 优 +側 侧 +偉 伟 +偵 侦 +師 师 +帥 帅 +幣 币 +帳 帐 +幫 帮 +歲 岁 +歸 归 +歷 历 +曆 历 +歐 欧 +殺 杀 +毀 毁 +愛 爱 +節 节 +範 范 +築 筑 +簡 简 +籃 篮 +籠 笼 +籤 签 +篤 笃 +簾 帘 +籌 筹 +醫 医 +產 产 +麗 丽 +麵 面 +麥 麦 +黃 黄 +黨 党 +龍 龙 +鳳 凤 +龜 龟 +齒 齿 +齊 齐 +書 书 +畫 画 +邊 边 +飛 飞 +決 决 +擊 击 +擔 担 +據 据 +擁 拥 +撲 扑 +擴 扩 +攔 拦 +掃 扫 +撿 捡 +擋 挡 +掛 挂 +換 换 +損 损 +搶 抢 +攜 携 +擾 扰 +撐 撑 diff --git a/backend/internal/pipeline/disposition.go b/backend/internal/pipeline/disposition.go index cae7dc3..d80c169 100644 --- a/backend/internal/pipeline/disposition.go +++ b/backend/internal/pipeline/disposition.go @@ -72,6 +72,15 @@ const ( FlagExcisionSuspect FlagReason = "excision_suspect" FlagHardBlock FlagReason = "hard_block" FlagUpstreamNotOK FlagReason = "upstream_not_ok" + + // FlagGlossaryMiss is the memory-bank post-check verdict (E1, шаг 4): an approved + // term's src fired in the chunk but no accepted dst form appears in the output — the + // model ignored the glossary, or we injected the wrong dst and it obeyed. Emitted + // ONLY when the opt-in glossary post-check gate is enabled (config); in the default + // flagger mode a miss is recorded in the retrieval-state, not a disposition. NOT + // retryable (a same-model retry re-produces the same rendering) and NOT auto- + // escalatable in v1 (the L3 targeted re-ask is a Phase-2 remedy, research/14 §9). + FlagGlossaryMiss FlagReason = "glossary_miss" ) // classifierVersion versions the INTRINSIC classify() verdict logic — the refusal diff --git a/backend/internal/pipeline/memnorm.go b/backend/internal/pipeline/memnorm.go new file mode 100644 index 0000000..f72b834 --- /dev/null +++ b/backend/internal/pipeline/memnorm.go @@ -0,0 +1,182 @@ +package pipeline + +import ( + "crypto/sha256" + _ "embed" + "encoding/hex" + "fmt" + "strings" + "unicode" + + "golang.org/x/text/unicode/norm" +) + +// memnorm.go: the memory-bank NORMALIZATION artifact (registry A4 — the most likely +// "тихо пусто" bug: a glossary key is present in the chunk but in a different +// orthographic form, so the record silently fails to match). Two pure, deterministic, +// VERSIONED normalizers, applied SYMMETRICALLY to both sides of every comparison: +// +// normalizeSourceKey — zh/ja source surfaces (glossary keys/aliases AND chunk text): +// NFKC (fold half/full-width, e.g. ﾃ→テ, Ｑ→Q) → trad→simp (per-char, the +// embedded table) → katakana→hiragana fold → Unicode lower. So a key stored in +// Simplified still matches a Traditional source surface (魯鎮 ↔ 鲁镇), a full-width +// Ｑ matches a half-width Q, and スズキ matches すずき. +// normalizeTargetForm — Russian dst forms (stored decl forms AND model output) for +// the post-check substring test: NFC → Unicode lower → ё→е fold. +// +// The whole artifact is versioned by memoryNormVersion, which is FOLDED INTO the +// snapshot via memoryVersion() (runner.go): a change to the algorithm OR to the +// embedded trad→simp table is a loud --resnapshot, never a silent match-behaviour +// change on already-checkpointed chunks. Determinism is load-bearing — the injected +// glossary block is a pure function of (frozen glossary, normalized chunk), and it +// enters request_hash via the rendered messages. + +//go:embed data/trad2simp.txt +var trad2simpRaw string + +// memoryNormAlgoVersion tags the normalization ALGORITHM (the pipeline of steps and +// their order). Bump it on an algorithm change. The trad→simp TABLE content is +// versioned SEPARATELY by hashing trad2simpRaw into memoryNormVersion below, so +// editing/completing the data file also invalidates — drift-proof (D8): you cannot +// forget to bump a version when you change the table, because the table's bytes ARE +// the version. +const memoryNormAlgoVersion = "memnorm-v1-nfkc+trad+kana+lower/nfc+lower+yofold" + +var ( + // trad2simp is the parsed Traditional→Simplified single-char map (data/trad2simp.txt). + trad2simp map[rune]rune + // memoryNormVersion is the snapshot-load-bearing version of the entire + // normalization artifact: the algorithm tag + a content hash of the embedded + // trad→simp table. memoryVersion() (runner.go) folds it into the job snapshot. + memoryNormVersion string +) + +func init() { + trad2simp = parseTradTable(trad2simpRaw) + h := sha256.Sum256([]byte(memoryNormAlgoVersion + "\x00" + trad2simpRaw)) + memoryNormVersion = memoryNormAlgoVersion + "-" + hex.EncodeToString(h[:])[:12] +} + +// parseTradTable parses the embedded " " table. Comment ('#') and blank +// lines are ignored; a non-empty content line that is NOT exactly two +// single-rune whitespace-separated fields is a corrupt table → panic at init (a +// loud build/test failure, caught the moment any test in the package runs, rather +// than a silently truncated map that would re-open the A4 hole). The table is +// authored in-repo and asserted by memnorm_test.go, so a panic here means the data +// file was corrupted, not bad user input. +func parseTradTable(raw string) map[rune]rune { + m := make(map[rune]rune, 1024) + for i, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + panic(fmt.Sprintf("pipeline: trad2simp table line %d: want 2 fields, got %d (%q)", i+1, len(fields), line)) + } + tr := []rune(fields[0]) + si := []rune(fields[1]) + if len(tr) != 1 || len(si) != 1 { + panic(fmt.Sprintf("pipeline: trad2simp table line %d: each side must be a single rune (%q → %q)", i+1, fields[0], fields[1])) + } + // A later duplicate wins deterministically (the file may list a char under + // two families); assert the mapping is consistent to catch a typo where the + // same trad char is mapped to two different simp chars. + if prev, ok := m[tr[0]]; ok && prev != si[0] { + panic(fmt.Sprintf("pipeline: trad2simp table line %d: %q maps to both %q and %q", i+1, fields[0], string(prev), fields[1])) + } + m[tr[0]] = si[0] + } + return m +} + +// normalizeSourceKey normalizes a zh/ja SOURCE surface (a glossary key/alias OR the +// chunk text) for exact matching. Pure and deterministic; the pipeline is +// NFKC → trad→simp (per rune) → katakana→hiragana → Unicode lower. Applied to BOTH +// keys and text so orthographic variants match (A4). Kana folding mirrors the eval +// spec (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to +// hiragana; the prolonged-sound mark ー (U+30FC) and half-width forms are already +// handled (the latter by NFKC). +func normalizeSourceKey(s string) string { + s = norm.NFKC.String(s) + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + if m, ok := trad2simp[r]; ok { + r = m + } + if r >= 0x30A1 && r <= 0x30F6 { // katakana → hiragana + r -= 0x60 + } + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} + +// normalizeTargetForm normalizes a Russian TARGET form (a stored decl form OR the +// model output) for the post-check whole-word test. Applied SYMMETRICALLY to both +// sides so a correctly-rendered term is NEVER a false MISS (over-flagging is the +// failure mode research/14 §2 warns about). Steps, each closing a self-review finding: +// - NFC; +// - dash/hyphen variants (en-dash, NB-hyphen, minus, soft hyphen, …) → ASCII '-', so +// «А–кью»/«А‑кью» match a stored «А-кью» (self-review #8); +// - runs of ANY Unicode whitespace (newline, NBSP U+00A0, ideographic space, …) → +// one ASCII space, then trim, so a multi-word name wrapped across a line break +// («Бородатого\nВана») matches a stored «Бородатого Вана» (self-review #2); +// - Unicode lower + ё→е fold (the ёфикатор's е/ё variation). +// Pure and deterministic. +func normalizeTargetForm(s string) string { + s = norm.NFC.String(s) + var b strings.Builder + b.Grow(len(s)) + prevSpace := false + for _, r := range s { + if isDashVariant(r) { + r = '-' + } + if unicode.IsSpace(r) { + if !prevSpace { + b.WriteByte(' ') + prevSpace = true + } + continue + } + prevSpace = false + r = unicode.ToLower(r) + if r == 'ё' { + r = 'е' + } + b.WriteRune(r) + } + return strings.TrimSpace(b.String()) +} + +// isDashVariant reports whether r is a hyphen/dash/minus variant that should fold to +// ASCII '-' for the post-check (Unicode dash punctuation + the math minus + soft hyphen). +func isDashVariant(r rune) bool { + switch r { + case '‐', '‑', '‒', '–', '—', '―', '−', '­': + return true + } + return false +} + +// significantLen counts the "significant" characters of a normalized key — CJK +// ideographs/kana plus letters (Latin/Cyrillic) — for the single-key ban (registry +// A3): a key whose significant length is below minKeyLen may not fire on its own +// (a lone Han/kana/letter like 炎/气/钱 matches unrelated text on the wrong sense). +// Digits and punctuation do not count (so "阿Q" = 2, but a bare "Q" = 1). Mirrors the +// eval spec's cjk_len. +func significantLen(normalized string) int { + n := 0 + for _, r := range normalized { + switch { + case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul): + n++ + case unicode.IsLetter(r): + n++ + } + } + return n +} diff --git a/backend/internal/pipeline/memnorm_test.go b/backend/internal/pipeline/memnorm_test.go new file mode 100644 index 0000000..777634a --- /dev/null +++ b/backend/internal/pipeline/memnorm_test.go @@ -0,0 +1,116 @@ +package pipeline + +import ( + "strings" + "testing" +) + +// TestTradTableCoversKnownCases asserts the embedded trad→simp table covers every +// character referenced by the eval specs / research/14 live bugs. A mutation +// (deleting a mapping line) fails exactly here — the guard against silently +// regressing the A4 hole (爺→爷 was research/14's live "тихо пусто" bug). +func TestTradTableCoversKnownCases(t *testing.T) { + want := map[rune]rune{ + '魯': '鲁', '鎮': '镇', '趙': '赵', '蕭': '萧', '煉': '炼', + '門': '门', '闆': '板', '莊': '庄', '錢': '钱', '陳': '陈', + '萬': '万', '舊': '旧', '臉': '脸', '爺': '爷', '羅': '罗', + } + for tr, si := range want { + if got, ok := trad2simp[tr]; !ok || got != si { + t.Errorf("trad2simp[%q] = %q, ok=%v; want %q", string(tr), string(got), ok, string(si)) + } + } + if len(trad2simp) < 200 { + t.Errorf("trad2simp table has only %d entries — expected the curated high-frequency set (≥200)", len(trad2simp)) + } + if memoryNormVersion == "" || !strings.HasPrefix(memoryNormVersion, memoryNormAlgoVersion) { + t.Errorf("memoryNormVersion=%q must start with the algo tag %q", memoryNormVersion, memoryNormAlgoVersion) + } +} + +func TestNormalizeSourceKey(t *testing.T) { + cases := []struct{ name, in, want string }{ + {"trad→simp two-char", "魯鎮", "鲁镇"}, + {"trad→simp name", "趙太爺", "赵太爷"}, + {"fullwidth latin lowered", "阿Ｑ", "阿q"}, + {"halfwidth latin lowered", "阿Q", "阿q"}, + {"katakana→hiragana", "スズキ", "すずき"}, + {"halfwidth katakana via NFKC", "ｽｽｷ", "すすき"}, // NFKC folds half→full katakana, then kana-fold to hiragana + {"plain han unchanged", "送灶", "送灶"}, + {"mixed", "萧炎", "萧炎"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := normalizeSourceKey(c.in); got != c.want { + t.Errorf("normalizeSourceKey(%q) = %q; want %q", c.in, got, c.want) + } + }) + } +} + +// TestNormalizeSymmetry is the A4 invariant: a key written in one orthography +// matches a chunk written in another, because normalization is applied to BOTH. +func TestNormalizeSymmetry(t *testing.T) { + // glossary key in Simplified, chunk text in Traditional + key := normalizeSourceKey("鲁镇") // simplified + text := normalizeSourceKey("魯鎮的冬天很冷。") // traditional + if !strings.Contains(text, key) { + t.Errorf("normalized simplified key %q not found in normalized traditional text %q", key, text) + } + // katakana name key vs hiragana surface + k2 := normalizeSourceKey("スズキ") + t2 := normalizeSourceKey("すずきさんが来た。") + if !strings.Contains(t2, k2) { + t.Errorf("normalized katakana key %q not found in normalized hiragana text %q", k2, t2) + } +} + +func TestNormalizeTargetForm(t *testing.T) { + cases := []struct{ in, want string }{ + {"А-кью", "а-кью"}, + {"Вэйчжуан", "вэйчжуан"}, + {"Тёмный страж", "темный страж"}, // NFC → lower → ё→е fold + } + for _, c := range cases { + if got := normalizeTargetForm(c.in); got != c.want { + t.Errorf("normalizeTargetForm(%q) = %q; want %q", c.in, got, c.want) + } + } + // ё and е must fold to the SAME normalized form (symmetric — no false miss) + if normalizeTargetForm("сёстры") != normalizeTargetForm("сестры") { + t.Error("ё→е fold is not symmetric: сёстры vs сестры differ after normalization") + } + // self-review #2: internal whitespace runs (incl. newline/NBSP) collapse to one space. + for _, v := range []string{"Бородатого\nВана", "Бородатого Вана", "Бородатого Вана"} { + if got := normalizeTargetForm(v); got != "бородатого вана" { + t.Errorf("whitespace not canonicalized: %q → %q", v, got) + } + } + // self-review #8: dash/hyphen variants fold to ASCII '-'. + for _, v := range []string{"А–кью", "А‑кью", "А−кью"} { + if got := normalizeTargetForm(v); got != "а-кью" { + t.Errorf("dash variant not unified: %q → %q", v, got) + } + } +} + +func TestSignificantLen(t *testing.T) { + cases := []struct { + in string + want int + }{ + {"阿q", 2}, // han + latin letter + {"炎", 1}, // single han — banned as a standalone key + {"气", 1}, // single han + {"钱", 1}, // single han (research/14 homograph) + {"送灶", 2}, // bigram — allowed + {"すずき", 3}, // three kana + {"q", 1}, // bare latin letter + {"а-кью", 4}, // cyrillic letters, hyphen not counted: а к ь ю = 4 + } + for _, c := range cases { + if got := significantLen(c.in); got != c.want { + t.Errorf("significantLen(%q) = %d; want %d", c.in, got, c.want) + } + } +} diff --git a/backend/internal/pipeline/memory.go b/backend/internal/pipeline/memory.go new file mode 100644 index 0000000..669e029 --- /dev/null +++ b/backend/internal/pipeline/memory.go @@ -0,0 +1,592 @@ +package pipeline + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" + "strconv" + "strings" + "unicode" + + "textmachine/backend/internal/store" +) + +// memory.go: the DETERMINISTIC hot path of the memory bank v2 (registry §Контракт +// горячего пути / research/13 Q3). $0, no LLM, no embeddings, no FTS5 — a multi-pattern +// exact match (Aho-Corasick) of glossary keys/aliases over the NORMALIZED chunk, with +// the single-key ban (A3), longest-match whole-entity replacement, the spoiler +// hard-reject window (C1), sticky scene-inertia (A5), a priority token budget with a +// logged eviction (F2), and the three-way injection disposition (A2). Pure and +// deterministic (no map-order output, no time/rand): the injected block is a pure +// function of (frozen bank, normalized chunk, chapter, sticky, budget), and it enters +// request_hash via the rendered messages — so a resumed chunk reproduces it for free. +// +// The registry's principle drives every choice here: "лучше ничего, чем мусор" and +// PRECISION over recall. A wrong injection (a homograph / short alias firing on the +// wrong sense) is the MAIN source of silent degradation (2510.00829) — worse than an +// empty match, which is a safe fallback to the model's base behaviour. So the matcher +// errs toward NOT firing, and the post-check (memcheck) makes what did fire observable. + +// memoryMatchVersion versions the matcher + injection-selection ALGORITHM (min-key +// ban, longest-match containment, spoiler gate, sticky, budget priority order). Folded +// into memoryVersion() → a change is a loud --resnapshot (it shifts the injected bytes +// → the wire → a resumed checkpoint's content). Sibling of memoryNormVersion. +const memoryMatchVersion = "memmatch-v1-longest+minkey2+spoiler+sticky+budget+postcheck-declaware" + +// defaultMinKeyLen is the minimum significant length (significantLen) a key must have +// to fire on its own (A3 single-key ban): a lone Han/kana/letter (炎/气/钱) matches +// unrelated text on the wrong sense — empirically confirmed (research/14). A per-entry +// allow_short overrides it (rare, guarded). Per-language tuning (registry min_key_len +// per-язык) is a later knob; v1 uses the eval spec's global MIN_KEY=2. +const defaultMinKeyLen = 2 + +// stickyDepth is the scene-inertia window (A5): entries exact-matched in the previous +// N chunks of the SAME chapter are carried into a pronominal chunk that names nobody. +// A code rule (Р2), reset at each chapter boundary (a new chapter is a scene change). +const stickyDepth = 2 + +// injectionDisposition is the three-way per-record decision (registry A2) — the core +// mechanism that converts silent degradation into loud. Distinct from the chunk×stage +// Disposition (disposition.go): that is ok|flagged|skipped for a completion; this is +// how much to TRUST an injected glossary record. +type injectionDisposition string + +const ( + memConfirmed injectionDisposition = "confirmed" // exact key/alias, approved, spoiler-valid → authoritative + memAmbiguous injectionDisposition = "ambiguous" // auto/draft term → inject "unverified" + forced post-check + memReject injectionDisposition = "reject" // spoiler-window violation → dropped and logged (C1 safety) +) + +// declInfo is the parsed decl column: the dst declension forms the post-check accepts. +type declInfo struct { + Invariant bool `json:"invariant"` + Forms []string `json:"forms"` +} + +// memoryEntry is one frozen, materialized glossary entry with its matchable keys +// pre-normalized. Immutable for a job. +type memoryEntry struct { + id string // stable unique id: src\x1f sense\x1f since\x1f until (= the UNIQUE key) + src string // raw source key + dst string // raw approved translation (may be "" for a ruby candidate) + status string // auto|draft|approved + sense string + sinceCh int + untilCh int + // normKeys are the normalized source surfaces (src + aliases) ELIGIBLE to fire + // (significantLen ≥ minKeyLen OR allow_short). A key too short is dropped here, so + // it is never in the automaton — the single-key ban is structural, not a runtime skip. + normKeys []string + // declForms are the normalized-target dst forms the post-check accepts. Empty → + // the post-check falls back to the single normalized dst (the naive base form that + // research/14 shows false-flags on inflection — the reason full decl matters). + declForms []string + declInvariant bool +} + +// MemoryBank is a book's frozen glossary materialized for one job: the entries, the +// Aho-Corasick automaton over all eligible keys, and the F1 memoryVersion. Built once +// (materializeMemory) before the chunk loop and never mutated. +type MemoryBank struct { + entries []memoryEntry + ac *ahoCorasick + keyOwners map[string][]int // normalized key → indices of entries that contributed it + version string // memoryVersion(): hash of frozen APPROVED rows + algo versions (F1/D8) +} + +// pickedEntry is one selected record for a chunk with its firing key and disposition. +type pickedEntry struct { + entry *memoryEntry + via string // the matching key, or "sticky" + disp injectionDisposition +} + +// memorySelection is the hot path's output for one chunk. +type memorySelection struct { + injected []pickedEntry // in budget priority order (what the model sees) + rejected []pickedEntry // spoiler-window rejects (C1, logged) + evicted []pickedEntry // dropped by the token budget (F2, logged) + activeIDs map[string]bool // exact-matched ids (NOT sticky) → the next chunk's sticky_prev +} + +func (b *MemoryBank) Version() string { return b.version } + +// materializeMemory builds a MemoryBank from the book's stored glossary rows (ORDER +// BY-stable — GlossaryForBook). Pure and deterministic. It computes memoryVersion as a +// content hash of the frozen APPROVED rows (D8: content-hash, not a version-counter — +// drift-proof) plus the normalization + matcher algorithm versions, so ANY change to +// the approved glossary OR to the deterministic machinery is a loud --resnapshot (F1). +func materializeMemory(rows []store.GlossaryEntry, gateOn bool) *MemoryBank { + b := &MemoryBank{keyOwners: map[string][]int{}} + var allKeys []string + seenKey := map[string]bool{} + + for _, row := range rows { + e := memoryEntry{ + id: row.Src + "\x1f" + row.Sense + "\x1f" + strconv.Itoa(row.SinceCh) + "\x1f" + strconv.Itoa(row.UntilCh), + src: row.Src, + dst: row.Dst, + status: row.Status, + sense: row.Sense, + sinceCh: row.SinceCh, + untilCh: row.UntilCh, + } + // Eligible source surfaces (src + aliases) under the single-key ban. A record + // with NO dst yet (a ruby auto-candidate) is NOT matchable: it renders nothing + // (renderGlossaryBlock/postcheck both skip empty dst), so admitting its keys would + // let it consume the token budget, evict a renderable line, and inflate the + // retrieval-state exact-hit count for a record the model never sees (self-review + // #1). It stays in the store for future promotion, just inert on the hot path. + var surfaces []string + if strings.TrimSpace(row.Dst) != "" { + surfaces = append(surfaces, row.Src) + for _, a := range row.Aliases { + surfaces = append(surfaces, a.Alias) + } + } + for _, s := range surfaces { + nk := normalizeSourceKey(s) + if nk == "" { + continue + } + if significantLen(nk) < defaultMinKeyLen && !row.AllowShort { + continue // A3: a lone Han/kana/letter never fires on its own + } + e.normKeys = append(e.normKeys, nk) + } + // Parse decl forms for the post-check (normalized target side). + if row.Decl != "" { + var d declInfo + if err := json.Unmarshal([]byte(row.Decl), &d); err == nil { + e.declInvariant = d.Invariant + for _, f := range d.Forms { + if nf := normalizeTargetForm(f); nf != "" { + e.declForms = append(e.declForms, nf) + } + } + } + } + idx := len(b.entries) + b.entries = append(b.entries, e) + for _, nk := range e.normKeys { + b.keyOwners[nk] = append(b.keyOwners[nk], idx) + if !seenKey[nk] { + seenKey[nk] = true + allKeys = append(allKeys, nk) + } + } + } + sort.Strings(allKeys) // deterministic automaton construction + b.ac = buildAC(allKeys) + b.version = computeMemoryVersion(rows, gateOn) + return b +} + +// computeMemoryVersion is the F1 content-hash: the frozen rows (ORDER BY-stable) plus the +// normalization and matcher algorithm versions. Approved-only per D8/§8 when the post-check +// hard gate is OFF (auto/draft injected-content changes are caught at the per-chunk +// content_hash level; their decl changes affect only the recomputed retrieval-state). +// When the gate is ON, the resolved chunk disposition depends on the decl forms of ALL +// injected records (approved AND auto/draft), and those are in NEITHER content_hash (decl +// is not injected) NOR the approved-only hash — so a decl edit would silently flip a +// resumed chunk's disposition (self-review #4). So gateOn folds every row's content +// (incl. decl + status) into the hash, making any such edit a loud --resnapshot. The +// glossary.id autoincrement is deliberately EXCLUDED (fresh each replace); only content +// columns are hashed. +func computeMemoryVersion(rows []store.GlossaryEntry, gateOn bool) string { + h := sha256.New() + h.Write([]byte("tm-memory-v2\x00")) + h.Write([]byte(memoryNormVersion + "\x00" + memoryMatchVersion + "\x00")) + h.Write([]byte("gate:" + strconv.FormatBool(gateOn) + "\x00")) + for _, r := range rows { + if r.Status != "approved" && !gateOn { + continue + } + // A fixed, length-prefixed field layout so no content can forge a boundary. + writeField(h, r.Status) + writeField(h, r.Src) + writeField(h, r.Dst) + writeField(h, r.Sense) + writeField(h, r.Type) + writeField(h, r.Gender) + writeField(h, r.Decl) + writeField(h, strconv.Itoa(r.SinceCh)) + writeField(h, strconv.Itoa(r.UntilCh)) + writeField(h, strconv.FormatBool(r.AllowShort)) + writeField(h, strconv.Itoa(len(r.Aliases))) + for _, a := range r.Aliases { // GlossaryForBook returns aliases ORDER BY alias + writeField(h, a.Alias) + writeField(h, a.AliasType) + } + } + return hex.EncodeToString(h.Sum(nil)) +} + +func writeField(h interface{ Write([]byte) (int, error) }, s string) { + var lb [8]byte + n := uint64(len(s)) + for i := 0; i < 8; i++ { + lb[i] = byte(n >> (8 * i)) + } + h.Write(lb[:]) + h.Write([]byte(s)) +} + +// glossaryLineTokens is the injected token cost of one record's "src → dst" line — the +// unit the token budget (config glossary_token_budget) spends. It is FLOOR-FREE (the raw +// cjk + other/3 estimate, NOT EstimateTokens' 16-token minimum): the floor is for sizing +// a whole request's max_tokens, but applying it PER glossary line ~1.85×-overcounts a +// block of short name lines and needlessly evicts records that fit (self-review #7). The +// header's cost is small and constant; omitting it keeps the unit purely per-record. +// Shared by Select's budget and the eviction test so the two never diverge. +func glossaryLineTokens(e *memoryEntry) int { + cjk, other := 0, 0 + for _, r := range e.src + " → " + e.dst { + switch { + case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul): + cjk++ + case unicode.IsSpace(r): + default: + other++ + } + } + return cjk + other/3 +} + +// Select is the hot path for ONE chunk: match → spoiler-reject → disposition → sticky +// → priority token budget. Pure and deterministic. stickyPrev is the set of ids +// exact-matched in the prior chunk(s) of the same chapter (A5). budgetTokens ≤ 0 means +// unbounded; otherwise records are kept in priority order while the cumulative injected +// token cost stays within budget — the rest are EVICTED (dropped but logged, F2; the +// budget may be underfilled, "лучше ничего, чем мусор"). +func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]bool, budgetTokens int) memorySelection { + ntext := []rune(normalizeSourceKey(chunk)) + occ := b.ac.matches(ntext) + // Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when + // the longer key belongs to a spoiler-VALID entry at this chapter (self-review #6). A + // spoiler-blocked longer key must not suppress a valid shorter name nested inside it. + occ = b.suppressContained(occ, chapter) + + // Map surviving key occurrences → entries, recording the LONGEST firing key per entry. + matchedVia := map[int]string{} + for _, m := range occ { + k := b.ac.keys[m.keyIdx] + for _, ei := range b.keyOwners[k] { + if cur, ok := matchedVia[ei]; !ok || len([]rune(k)) > len([]rune(cur)) { + matchedVia[ei] = k + } + } + } + + sel := memorySelection{activeIDs: map[string]bool{}} + hits := map[string]pickedEntry{} // id → picked (exact) + + // Iterate entries in their frozen (ORDER BY-stable) order — never map order. + for ei := range b.entries { + via, ok := matchedVia[ei] + if !ok { + continue + } + e := &b.entries[ei] + if blocked := spoilerBlocked(e, chapter); blocked { + sel.rejected = append(sel.rejected, pickedEntry{e, via, memReject}) + continue + } + hits[e.id] = pickedEntry{e, via, dispositionFor(e)} + sel.activeIDs[e.id] = true // exact-matched ids feed the next chunk's sticky (NOT sticky-carried) + } + + // Sticky scene-inertia: carry prior-chunk exact matches not re-matched here, unless + // the spoiler window blocks them (spoiler beats sticky). + for ei := range b.entries { + e := &b.entries[ei] + if !stickyPrev[e.id] { + continue + } + if _, already := hits[e.id]; already { + continue + } + if spoilerBlocked(e, chapter) { + continue + } + hits[e.id] = pickedEntry{e, "sticky", dispositionFor(e)} + } + + // Deterministic priority order for the budget: confirmed>ambiguous, exact>sticky, + // approved>auto. Collect in frozen-entry order (NOT map order) then stable-sort, so + // ties keep the deterministic base order. + order := make([]pickedEntry, 0, len(hits)) + for ei := range b.entries { + if p, ok := hits[b.entries[ei].id]; ok { + order = append(order, p) + } + } + sort.SliceStable(order, func(i, j int) bool { + return priorityRank(order[i]) < priorityRank(order[j]) + }) + + if budgetTokens > 0 { + used, cut := 0, len(order) + for i := range order { + used += glossaryLineTokens(order[i].entry) + if used > budgetTokens { + cut = i // this record and everything after it overflow the budget + break + } + } + sel.injected = order[:cut] + sel.evicted = order[cut:] // F2: dropped, but LOGGED via the retrieval-state + } else { + sel.injected = order + } + return sel +} + +// spoilerBlocked reports whether the entry's since_ch/until_ch window excludes this +// chapter (C1: a hard reject gate — a fact the current chapter must not know yet). +func spoilerBlocked(e *memoryEntry, chapter int) bool { + if e.sinceCh > 0 && chapter < e.sinceCh { + return true + } + if e.untilCh > 0 && chapter > e.untilCh { + return true + } + return false +} + +// dispositionFor maps a term's status to its injection disposition (A2): approved → +// CONFIRMED (authoritative); auto/draft → AMBIGUOUS (inject "unverified" + forced +// post-check). +func dispositionFor(e *memoryEntry) injectionDisposition { + if e.status == "approved" { + return memConfirmed + } + return memAmbiguous +} + +// priorityRank orders the budget: confirmed before ambiguous, exact before sticky, +// approved before auto/draft (F2 eviction keeps the most trustworthy records). +func priorityRank(p pickedEntry) int { + rank := 0 + if p.disp != memConfirmed { + rank |= 1 << 2 + } + if p.via == "sticky" { + rank |= 1 << 1 + } + if p.entry.status != "approved" { + rank |= 1 << 0 + } + return rank +} + +// glossaryBlockHeader introduces the injected glossary block. The layout mirrors the +// SakuraLLM/GalTransl convention "src → dst", with unverified (ambiguous) records +// tagged so the model — and a human reviewer — see they are not authoritative (A2). +const glossaryBlockHeader = "ГЛОССАРИЙ (используй эти утверждённые переводы имён и терминов последовательно; строки с пометкой ⟨проверить⟩ — неподтверждённые кандидаты):" + +// renderGlossaryBlock serializes the selected records into the injection message +// (§C). Records with no dst yet (ruby candidates) are skipped — a "src → " line +// carries nothing. Returns "" when nothing renders, so an empty selection injects NO +// message at all ("лучше ничего, чем мусор"). Deterministic: the injected order is the +// budget priority order fixed by Select. +func renderGlossaryBlock(injected []pickedEntry) string { + var lines []string + for _, p := range injected { + if strings.TrimSpace(p.entry.dst) == "" { + continue + } + line := p.entry.src + " → " + p.entry.dst + if p.disp != memConfirmed { + line += " ⟨проверить⟩" + } + lines = append(lines, line) + } + if len(lines) == 0 { + return "" + } + return glossaryBlockHeader + "\n" + strings.Join(lines, "\n") +} + +// --- Aho-Corasick multi-pattern automaton over runes ---------------------------- + +type acMatch struct { + keyIdx int + start, end int // rune indices [start, end) +} + +type acNode struct { + next map[rune]int + fail int + out []int // key indices whose pattern ends at this node (incl. via fail links) +} + +type ahoCorasick struct { + nodes []acNode + keys []string + klen []int // rune length of each key +} + +// buildAC constructs the automaton from the (already sorted, unique) normalized keys. +// Deterministic in RESULT regardless of map iteration order: fail links and outputs are +// a function of the key set, not the BFS visit order. +func buildAC(keys []string) *ahoCorasick { + ac := &ahoCorasick{keys: keys, klen: make([]int, len(keys))} + ac.nodes = []acNode{{next: map[rune]int{}}} // root = node 0 + for i, k := range keys { + kr := []rune(k) + ac.klen[i] = len(kr) + cur := 0 + for _, r := range kr { + nxt, ok := ac.nodes[cur].next[r] + if !ok { + nxt = len(ac.nodes) + ac.nodes = append(ac.nodes, acNode{next: map[rune]int{}}) + ac.nodes[cur].next[r] = nxt + } + cur = nxt + } + ac.nodes[cur].out = append(ac.nodes[cur].out, i) + } + // BFS to compute fail links; propagate outputs down fail chains one level (each + // node inherits its fail node's already-complete output set). + var queue []int + for _, c := range ac.nodes[0].next { + ac.nodes[c].fail = 0 + queue = append(queue, c) + } + for len(queue) > 0 { + cur := queue[0] + queue = queue[1:] + for r, nxt := range ac.nodes[cur].next { + queue = append(queue, nxt) + f := ac.nodes[cur].fail + for f != 0 { + if _, ok := ac.nodes[f].next[r]; ok { + break + } + f = ac.nodes[f].fail + } + if fn, ok := ac.nodes[f].next[r]; ok && fn != nxt { + ac.nodes[nxt].fail = fn + } else { + ac.nodes[nxt].fail = 0 + } + ac.nodes[nxt].out = append(ac.nodes[nxt].out, ac.nodes[ac.nodes[nxt].fail].out...) + } + } + return ac +} + +// matches returns every key occurrence in text (rune indices), sorted deterministically. +func (ac *ahoCorasick) matches(text []rune) []acMatch { + var out []acMatch + cur := 0 + for i, r := range text { + for cur != 0 { + if _, ok := ac.nodes[cur].next[r]; ok { + break + } + cur = ac.nodes[cur].fail + } + if nxt, ok := ac.nodes[cur].next[r]; ok { + cur = nxt + } else { + cur = 0 + } + for _, ki := range ac.nodes[cur].out { + out = append(out, acMatch{keyIdx: ki, start: i - ac.klen[ki] + 1, end: i + 1}) + } + } + sort.Slice(out, func(a, b int) bool { + if out[a].start != out[b].start { + return out[a].start < out[b].start + } + if out[a].end != out[b].end { + return out[a].end < out[b].end + } + return out[a].keyIdx < out[b].keyIdx + }) + return out +} + +// suppressContained drops an occurrence fully contained inside a STRICTLY longer one +// (longest-match, whole-entity replacement — A3), where the longer one belongs to a +// spoiler-VALID entry at this chapter. Gating the SUPPRESSOR on spoiler validity closes +// self-review #6: a spoiler-blocked longer key (e.g. 林动的父亲, until_ch=3, read at ch10) +// must not silently drop a valid shorter name nested inside it (林动) — the shorter name +// survives and injects, the longer one is separately spoiler-rejected+logged. Equal-length +// overlaps are both kept (genuine surface ambiguity, surfaced by the injectivity check). +// O(n²) over the few matches in a chunk. +func (b *MemoryBank) suppressContained(ms []acMatch, chapter int) []acMatch { + validSuppressor := func(m acMatch) bool { + for _, ei := range b.keyOwners[b.ac.keys[m.keyIdx]] { + if !spoilerBlocked(&b.entries[ei], chapter) { + return true + } + } + return false + } + var kept []acMatch + for i, m := range ms { + contained := false + for j, o := range ms { + if i == j { + continue + } + if o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start) && validSuppressor(o) { + contained = true + break + } + } + if !contained { + kept = append(kept, m) + } + } + return kept +} + +// injectivityCollisions reports approved dst collisions (B2): two distinct source +// terms mapped to the SAME dst (one Russian surface for two entities → the reader +// cannot tell them apart), returned as human-readable strings for a load-time warning. +// A pure diagnostic; it does not reject (some collisions are legitimate, e.g. a title +// shared by rank tiers), so the caller logs it, not aborts. +func injectivityCollisions(rows []store.GlossaryEntry) []string { + bySurface := map[string][]string{} + for _, r := range rows { + if r.Status != "approved" || strings.TrimSpace(r.Dst) == "" { + continue + } + key := normalizeTargetForm(r.Dst) + bySurface[key] = append(bySurface[key], r.Src) + } + var out []string + // Deterministic order: iterate a sorted key list, not the map. + keys := make([]string, 0, len(bySurface)) + for k := range bySurface { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + srcs := bySurface[k] + if len(distinct(srcs)) > 1 { + sort.Strings(srcs) + out = append(out, "dst "+strconv.Quote(k)+" ← "+strings.Join(distinct(srcs), ", ")) + } + } + return out +} + +func distinct(ss []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range ss { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.Strings(out) + return out +} diff --git a/backend/internal/pipeline/memory_e1_test.go b/backend/internal/pipeline/memory_e1_test.go new file mode 100644 index 0000000..a1484c2 --- /dev/null +++ b/backend/internal/pipeline/memory_e1_test.go @@ -0,0 +1,177 @@ +package pipeline + +import ( + "testing" + + "textmachine/backend/internal/store" +) + +// memory_e1_test.go: the E1 MEASUREMENT the onboarding contract requires BEFORE the +// post-check may be trusted as a hard gate (research/13 §2 "post-check в русском — +// несущий И невалидированный"; research/14 §2 quantified it at 18–36% false flags for a +// naive matcher). It measures the decl-aware post-check's false-flag rate on REAL +// inflected Russian (the adaptive_levers/adaptive_reask canon: 阿Q ch.5–9, Rogov), +// empirically — not "примерно" (the agent's explicit ask). +// +// It answers two things the agent flagged: +// 1. Stored-decl WORKS when the forms are complete → 0 false flags on a correctly +// rendered, inflected chunk (the arm that lets the post-check be a real signal). +// 2. Stored-decl is NOT free-reliable: with only the base form (an under-filled decl, +// the OOV-translit risk — one cheap LLM call may miss cases) the SAME chunk yields +// false flags. This is why v1 keeps the post-check a FLAGGER, not a hard gate: if +// the seed under-fills decl, it must degrade to observability, not block chunks. +// +// The inflection cases are the exact ones research/14 measured a naive matcher losing: +// «Вэйчжуане» (loc, a SUFFIX inflection substring-matching handles), «Бородатого Вана» +// (a modifier that inflects INTERNALLY — substring of the base fails), «сюцая» +// (stem-final й→я — the base is not a substring), «Маленького Дэна» (internal). + +// e1Entity is one canon entity with its full decl set. +type e1Entity struct { + src string + baseDst string + fullDecl []string +} + +var e1Canon = []e1Entity{ + {"阿Q", "А-кью", []string{"А-кью"}}, // hyphenated translit, invariant + {"未庄", "Вэйчжуан", []string{"Вэйчжуан", "Вэйчжуане", "Вэйчжуана", "Вэйчжуану", "Вэйчжуаном"}}, + {"王胡", "Бородатый Ван", []string{"Бородатый Ван", "Бородатого Вана", "Бородатому Вану", "Бородатым Ваном", "Бородатом Ване"}}, + {"秀才", "сюцай", []string{"сюцай", "сюцая", "сюцаю", "сюцае", "сюцаем"}}, + {"小D", "Маленький Дэн", []string{"Маленький Дэн", "Маленького Дэна", "Маленькому Дэну", "Маленьким Дэном", "Маленьком Дэне"}}, + {"邹七嫂", "тётушка Цзоу Седьмая", []string{"тётушка Цзоу Седьмая", "тётушки Цзоу Седьмой", "тётушку Цзоу Седьмую"}}, +} + +// e1Source contains every src key so all six entities are injected. +const e1Source = "阿Q在未庄遇见王胡和秀才，又碰到小D，邹七嫂在一旁看着。" + +// e1CorrectOutput is a publisher-quality rendering: every entity present but INFLECTED +// (the case a boundary regexp / a base-only matcher false-flags). +const e1CorrectOutput = "В Вэйчжуане Бородатого Вана и сюцая знали все. " + + "А-кью повстречал Маленького Дэна у реки; тётушка Цзоу Седьмая молча прошла мимо." + +func e1Glossary(useFullDecl bool) []store.GlossaryEntry { + var out []store.GlossaryEntry + for _, e := range e1Canon { + g := gl(e.src, e.baseDst, "", "approved") + if useFullDecl { + g.Decl = declJSON(false, e.fullDecl...) + } else { + g.Decl = declJSON(true, e.baseDst) // naive: only the base form + } + out = append(out, g) + } + return out +} + +func TestE1PostcheckFalseFlagRate(t *testing.T) { + // Arm A — full decl: the post-check must NOT false-flag a correct inflected output. + bFull := bankFrom(e1Glossary(true)) + selFull := bFull.Select(e1Source, 1, nil, 0) + if len(selFull.injected) != len(e1Canon) { + t.Fatalf("expected all %d canon entities injected, got %d", len(e1Canon), len(selFull.injected)) + } + fullMiss := bFull.postcheck(selFull.injected, e1CorrectOutput) + if len(fullMiss) != 0 { + t.Errorf("STORED-DECL FALSE FLAGS on correct output: %d/%d — %v", len(fullMiss), len(selFull.injected), fullMiss) + } + + // Arm B — base only (under-filled decl): the SAME correct output now false-flags. + bBase := bankFrom(e1Glossary(false)) + selBase := bBase.Select(e1Source, 1, nil, 0) + baseMiss := bBase.postcheck(selBase.injected, e1CorrectOutput) + + fullRate := float64(len(fullMiss)) / float64(len(selFull.injected)) + baseRate := float64(len(baseMiss)) / float64(len(selBase.injected)) + t.Logf("E1 false-flag rate on real inflected Russian: full-decl %.0f%% (%d/%d), base-only %.0f%% (%d/%d)", + fullRate*100, len(fullMiss), len(selFull.injected), + baseRate*100, len(baseMiss), len(selBase.injected)) + + // The measurement's verdict: decl-awareness is essential (full ≪ base), and the + // stored-decl matcher is trustworthy ONLY when complete. Base-only reproduces the + // research/14 failure — the reason the post-check stays a flagger until validated. + if len(baseMiss) == 0 { + t.Error("base-only decl produced 0 false flags — the E1 measurement lost its hard inflection cases; check the fixture") + } + if len(fullMiss) >= len(baseMiss) { + t.Errorf("full decl (%d) did not reduce false flags vs base (%d)", len(fullMiss), len(baseMiss)) + } + // The specific research/14 internal-inflection cases MUST be the base-only misses. + for _, src := range []string{"王胡", "秀才", "小D"} { + if !hasMiss(baseMiss, src) { + t.Errorf("base-only decl did not false-flag %s (expected an internal-inflection false flag)", src) + } + } +} + +// TestPostcheckWhitespaceAndDashRobust covers self-review #2 (multi-word name wrapped +// across a newline / doubled space / NBSP) and #8 (hyphen/dash variants): a correctly +// rendered term in ANY of these forms must NOT be false-flagged. +func TestPostcheckWhitespaceAndDashRobust(t *testing.T) { + wang := gl("王胡", "Бородатый Ван", "", "approved") + wang.Decl = declJSON(false, "Бородатый Ван", "Бородатого Вана") + ahq := gl("阿Q", "А-кью", "", "approved") + ahq.Decl = declJSON(true, "А-кью") + b := bankFrom([]store.GlossaryEntry{wang, ahq}) + sel := b.Select("王胡和阿Q在场。", 1, nil, 0) + + robust := []string{ + "Бородатого Вана и А-кью видели.", // baseline + "Бородатого\nВана и А-кью видели.", // #2 newline between the name's words + "Бородатого Вана и А-кью видели.", // #2 doubled space + "Бородатого Вана и А-кью видели.", // #2 NBSP + "Бородатого Вана и А–кью видели.", // #8 en-dash in А-кью + "Бородатого Вана и А‑кью видели.", // #8 non-breaking hyphen + } + for _, out := range robust { + if m := b.postcheck(sel.injected, out); len(m) != 0 { + t.Errorf("false flag on a correctly-rendered variant %q: %v", out, m) + } + } +} + +// TestPostcheckWholeWordNoMasking covers self-review #3: a DROPPED short translit name +// must not be masked by an unrelated common word that merely contains its letters. +func TestPostcheckWholeWordNoMasking(t *testing.T) { + van := gl("王", "Ван", "", "approved") + van.AllowShort = true // single-char src, allowed here to exercise the target-side check + van.Decl = declJSON(false, "Ван", "Вана", "Вану", "Ваном", "Ване") + b := bankFrom([]store.GlossaryEntry{van}) + sel := b.Select("王走了。", 1, nil, 0) + if len(sel.injected) != 1 { + t.Fatalf("expected 王 injected, got %d", len(sel.injected)) + } + // 王 is ABSENT from each output, yet a naive substring would find «ван» inside these. + for _, masked := range []string{"Мимо прошёл караван верблюдов.", "Он сел на диван и молчал.", "Иван Петрович кивнул."} { + if m := b.postcheck(sel.injected, masked); !hasMiss(m, "王") { + t.Errorf("post-check masked a dropped name by an unrelated word: %q → %v", masked, m) + } + } + // A genuine rendering (inflected) is found as a whole word. + if m := b.postcheck(sel.injected, "К нему подошёл Ван."); len(m) != 0 { + t.Errorf("false flag on a correct whole-word rendering: %v", m) + } + if m := b.postcheck(sel.injected, "Он говорил с Ваном о делах."); len(m) != 0 { + t.Errorf("false flag on a correct inflected rendering: %v", m) + } +} + +// TestE1PostcheckCatchesPoisoning: with complete decl, a WRONG rendering (poisoning / +// the model ignoring the glossary) is still a true-positive flag — the post-check's +// actual job. Precision does not come for free: the decl forms are inflections of the +// FULL dst phrase, so a foreign name that merely shares the head noun does NOT slip past. +func TestE1PostcheckCatchesPoisoning(t *testing.T) { + b := bankFrom(e1Glossary(true)) + sel := b.Select("王胡走过来。", 1, nil, 0) // only 王胡 fires + if len(sel.injected) != 1 { + t.Fatalf("expected 1 injected, got %d", len(sel.injected)) + } + // Correct inflected rendering → no flag. + if m := b.postcheck(sel.injected, "К нему подошёл Бородатый Ван."); len(m) != 0 { + t.Errorf("false flag on correct rendering: %v", m) + } + // Poisoned rendering (a different name) → flagged (true positive). + if m := b.postcheck(sel.injected, "К нему подошёл Ван Ху."); !hasMiss(m, "王胡") { + t.Errorf("post-check missed the poisoning (Ван Ху ≠ Бородатый Ван): %v", m) + } +} diff --git a/backend/internal/pipeline/memory_f1_test.go b/backend/internal/pipeline/memory_f1_test.go new file mode 100644 index 0000000..88a06cb --- /dev/null +++ b/backend/internal/pipeline/memory_f1_test.go @@ -0,0 +1,99 @@ +package pipeline + +import ( + "testing" + + "textmachine/backend/internal/store" +) + +// memory_f1_test.go covers the F1 content-hash (D8): memoryVersion is a pure function of +// the frozen APPROVED rows + the algorithm versions. A change to the approved glossary +// re-pins loudly; auto/draft churn does not; the row id never leaks in. + +func TestMemoryVersionStableAndSensitive(t *testing.T) { + base := []store.GlossaryEntry{ + gl("阿Q", "А-кью", "name", "approved"), + gl("鲁镇", "Лучжэнь", "", "approved"), + } + v0 := computeMemoryVersion(base, false) + + // Identical rows → identical version (idempotent; drift-proof content hash). + if computeMemoryVersion(base, false) != v0 { + t.Fatal("memoryVersion is not stable for identical rows") + } + + // Changing an APPROVED dst → different version (a loud --resnapshot). + changed := []store.GlossaryEntry{ + gl("阿Q", "А-Кью (изменено)", "name", "approved"), + gl("鲁镇", "Лучжэнь", "", "approved"), + } + if computeMemoryVersion(changed, false) == v0 { + t.Error("changing an approved dst did not change memoryVersion — F1 would silently regress") + } + + // Adding an alias to an approved row → different version (aliases affect matching). + withAlias := []store.GlossaryEntry{ + func() store.GlossaryEntry { e := gl("阿Q", "А-кью", "name", "approved"); e.Aliases = []store.GlossaryAlias{alias("阿桂")}; return e }(), + gl("鲁镇", "Лучжэнь", "", "approved"), + } + if computeMemoryVersion(withAlias, false) == v0 { + t.Error("adding an alias to an approved row did not change memoryVersion") + } + + // Changing an AUTO row → SAME version (approved-only, D8; auto changes are caught by + // the per-chunk content_hash instead). + withAuto := append(append([]store.GlossaryEntry{}, base...), gl("小D", "Малыш Дэ", "", "auto")) + if computeMemoryVersion(withAuto, false) != v0 { + t.Error("an auto row changed memoryVersion — it must hash only approved rows (D8)") + } + withAuto2 := append(append([]store.GlossaryEntry{}, base...), gl("小D", "Другой Дэ", "", "auto")) + if computeMemoryVersion(withAuto, false) != computeMemoryVersion(withAuto2, false) { + t.Error("editing an auto row changed memoryVersion — auto must not enter the F1 hash") + } +} + +// TestMemoryVersionIgnoresID: the autoincrement glossary.id is fresh on every replace; +// it must NOT feed the hash (else a re-seed of identical content would false-resnapshot). +func TestMemoryVersionIgnoresID(t *testing.T) { + a := []store.GlossaryEntry{{BookID: "b", ID: 1, Src: "甲", Dst: "А", Status: "approved"}} + b := []store.GlossaryEntry{{BookID: "b", ID: 999, Src: "甲", Dst: "А", Status: "approved"}} + if computeMemoryVersion(a, false) != computeMemoryVersion(b, false) { + t.Error("memoryVersion depends on glossary.id — a re-seed would false-resnapshot") + } +} + +// TestMaterializeMemoryVersionMatches: the bank's Version() equals computeMemoryVersion +// of its input rows (the runner hashes the same thing it matches on). +func TestMaterializeMemoryVersionMatches(t *testing.T) { + rows := specGlossary() + b := materializeMemory(rows, false) + if b.Version() != computeMemoryVersion(rows, false) { + t.Error("bank Version() diverges from computeMemoryVersion of its rows") + } +} + +// TestMemoryVersionGateFoldsUnapprovedDecl covers self-review #4: when the post-check +// hard gate is ON, editing a DRAFT/auto row's decl (which changes the gate's resolved +// disposition but is in NEITHER content_hash NOR the approved-only hash) MUST change +// memoryVersion — else a $0 resume silently flips a chunk. Gate OFF: it must NOT change +// it (D8 approved-only holds; decl affects only the recomputed retrieval-state). +func TestMemoryVersionGateFoldsUnapprovedDecl(t *testing.T) { + mk := func(declForm string) []store.GlossaryEntry { + e := gl("鈴木", "Судзуки", "", "draft") + e.Decl = declJSON(false, declForm) + return []store.GlossaryEntry{gl("阿Q", "А-кью", "", "approved"), e} + } + a := mk("Судзуки") + b := mk("Судзуку") // only the DRAFT decl form changed (dst unchanged, status draft) + + if computeMemoryVersion(a, false) != computeMemoryVersion(b, false) { + t.Error("gate OFF: a draft decl change must NOT move memoryVersion (D8 approved-only)") + } + if computeMemoryVersion(a, true) == computeMemoryVersion(b, true) { + t.Error("gate ON: a draft decl change MUST move memoryVersion (else a resume silently flips the chunk)") + } + // Toggling the gate itself also changes the version (a mode change is a loud re-pin). + if computeMemoryVersion(a, false) == computeMemoryVersion(a, true) { + t.Error("toggling the post-check gate must change memoryVersion") + } +} diff --git a/backend/internal/pipeline/memory_test.go b/backend/internal/pipeline/memory_test.go new file mode 100644 index 0000000..a746d90 --- /dev/null +++ b/backend/internal/pipeline/memory_test.go @@ -0,0 +1,372 @@ +package pipeline + +import ( + "encoding/json" + "strings" + "testing" + + "textmachine/backend/internal/store" +) + +// memory_test.go ports the executable spec eval/memory_hotpath.py (T1–T10) to Go +// unit tests, per the onboarding contract — but NOT the accept-REGEXPS verbatim +// (research/14: a \b-regexp is the 18–36% false-flag source). The selection tests +// (T1–T8) are the same cases; the post-check tests (T9/T10) use the decl-aware +// matcher (memory_e1_test.go measures its false-flag rate on real Russian). + +func gl(src, dst, sense, status string) store.GlossaryEntry { + return store.GlossaryEntry{BookID: "b", Src: src, Dst: dst, Sense: sense, Status: status, Source: "seed"} +} + +func declJSON(invariant bool, forms ...string) string { + b, _ := json.Marshal(declInfo{Invariant: invariant, Forms: forms}) + return string(b) +} + +func alias(a string) store.GlossaryAlias { return store.GlossaryAlias{Alias: a, AliasType: "прозвище"} } + +// specGlossary mirrors eval/memory_hotpath.py's G (with decl forms replacing the +// toy accept-regexps). +func specGlossary() []store.GlossaryEntry { + ahq := gl("阿Q", "А-кью", "name", "approved") + ahq.Decl = declJSON(true, "А-кью", "А кью") + + sishu := gl("四叔", "Четвёртый дядюшка", "char", "approved") + sishu.Aliases = []store.GlossaryAlias{alias("鲁四老爷")} + sishu.Decl = declJSON(false, "дядюшка", "дядюшки", "дядюшке", "дядюшку", "дядюшкой") + + luzhen := gl("鲁镇", "Лучжэнь", "", "approved") + luzhen.Decl = declJSON(false, "Лучжэнь", "Лучжэня", "Лучжэне") + + songzao := gl("送灶", "проводы бога очага", "festival", "approved") + songzao.Decl = declJSON(true, "бога очага") + + xiaoyan := gl("萧炎", "Сяо Янь", "name", "approved") + xiaoyan.Aliases = []store.GlossaryAlias{alias("炎哥")} + xiaoyan.Decl = declJSON(true, "Сяо Янь") + + xiuwei := gl("修为", "уровень совершенствования", "cultivation", "approved") + xiuwei.Decl = declJSON(false, "совершенствования", "совершенствование") + + shadow := gl("影卫", "Тёмный страж", "spoiler", "approved") + shadow.SinceCh = 200 + shadow.Decl = declJSON(false, "Тёмный страж", "Тёмного стража") + + newname := gl("小D", "Малыш Дэ", "", "auto") + newname.Decl = declJSON(false, "Малыш Дэ", "Малыша Дэ") + + return []store.GlossaryEntry{ahq, sishu, luzhen, songzao, xiaoyan, xiuwei, shadow, newname} +} + +func bankFrom(entries []store.GlossaryEntry) *MemoryBank { return materializeMemory(entries, false) } + +// injMap maps injected src → disposition for assertions. +func injMap(sel memorySelection) map[string]injectionDisposition { + m := map[string]injectionDisposition{} + for _, p := range sel.injected { + m[p.entry.src] = p.disp + } + return m +} + +func TestHotPathSpecT1toT10(t *testing.T) { + b := bankFrom(specGlossary()) + const budget = 0 + + // T1: exact + alias select; alias 鲁四老爷 pulls 四叔. + sel := b.Select("阿Q走进鲁镇，遇见鲁四老爷。", 1, nil, budget) + m := injMap(sel) + if m["阿Q"] != memConfirmed || m["鲁镇"] != memConfirmed || m["四叔"] != memConfirmed { + t.Errorf("T1 exact+alias: %v", m) + } + + // T2: 送到灶 (literal) — 送灶 must NOT inject (bigram absent, 灶 single-key banned). + sel = b.Select("厨房里，老妈子把饭菜送到灶间去热一热。", 1, nil, budget) + if _, ok := injMap(sel)["送灶"]; ok { + t.Error("T2 wrong-sense 送灶 injected (A3 homograph)") + } + + // T3: 炎势 (flame) — 萧炎 must NOT inject. + sel = b.Select("山火蔓延，炎势冲天，村民连夜逃走。", 1, nil, budget) + if _, ok := injMap(sel)["萧炎"]; ok { + t.Error("T3 wrong-sense 炎 injected") + } + + // T4: 修好 (repair) — 修为 must NOT inject. + sel = b.Select("工匠把石桥修好了。", 1, nil, budget) + if _, ok := injMap(sel)["修为"]; ok { + t.Error("T4 wrong-sense 修 injected") + } + + // T5: trad→simp normalization — 魯鎮 matches key 鲁镇 (A4). + sel = b.Select("魯鎮的冬天很冷。", 1, nil, budget) + if _, ok := injMap(sel)["鲁镇"]; !ok { + t.Error("T5 trad→simp: 魯鎮 did not match 鲁镇") + } + + // T6: pronominal chunk — sticky carries 四叔 from the previous chunk. + sishuID := "四叔\x1fchar\x1f0\x1f0" + sel = b.Select("他慢慢站起来，叹了口气。", 2, map[string]bool{sishuID: true}, budget) + var stickySishu *pickedEntry + for i := range sel.injected { + if sel.injected[i].entry.src == "四叔" { + stickySishu = &sel.injected[i] + } + } + if stickySishu == nil || stickySishu.via != "sticky" { + t.Errorf("T6 sticky scene-inertia failed: %+v", injMap(sel)) + } + + // T7: spoiler — 影卫 (since_ch=200) in chapter 5 → hard reject. + sel = b.Select("影卫出现了。", 5, nil, budget) + if _, ok := injMap(sel)["影卫"]; ok { + t.Error("T7 spoiler 影卫 injected in ch5") + } + if len(sel.rejected) != 1 || sel.rejected[0].entry.src != "影卫" { + t.Errorf("T7 spoiler not rejected+logged: %+v", sel.rejected) + } + // T7b: same entry passes after the reveal chapter. + sel = b.Select("影卫出现了。", 250, nil, budget) + if _, ok := injMap(sel)["影卫"]; !ok { + t.Error("T7b spoiler entry did not pass at ch250") + } + + // T8: auto term 小D → ambiguous disposition. + sel = b.Select("小D也来了。", 1, nil, budget) + if injMap(sel)["小D"] != memAmbiguous { + t.Errorf("T8 auto-term not ambiguous: %v", injMap(sel)) + } + + // T9: post-check catches a missing dst (leak) and passes a present one. + sel = b.Select("阿Q走进鲁镇。", 1, nil, budget) + bad := b.postcheck(sel.injected, "Некто вошёл в Лучжэнь.") // А-кью dropped + good := b.postcheck(sel.injected, "А-кью вошёл в Лучжэнь.") + if !hasMiss(bad, "阿Q") { + t.Errorf("T9 post-check missed the leak: %+v", bad) + } + if len(good) != 0 { + t.Errorf("T9 post-check false-flagged a correct output: %+v", good) + } + + // T10: post-check catches poisoning — injected dst absent, output used a foreign name. + sel = b.Select("阿Q走进鲁镇。", 1, nil, budget) + poison := b.postcheck(sel.injected, "Линь Чун вошёл в деревню Вэйчжуан.") + if !hasMiss(poison, "阿Q") { + t.Errorf("T10 post-check missed the poisoning: %+v", poison) + } +} + +func hasMiss(misses []postcheckMiss, src string) bool { + for _, m := range misses { + if m.Src == src { + return true + } + } + return false +} + +// TestTrapCorpus ports the retrieval_bench wrong-sense traps: exact multi-char match +// must fire 0/N on homograph chunks that contain the KEY CHARACTER but not the term. +func TestTrapCorpus(t *testing.T) { + entries := []store.GlossaryEntry{ + gl("送灶", "проводы бога очага", "festival", "approved"), + gl("修为", "уровень совершенствования", "cultivation", "approved"), + gl("萧炎", "Сяо Янь", "name", "approved"), + gl("斗气", "боевая ци", "term", "approved"), + gl("钱老爷", "господин Цянь", "name", "approved"), // multi-char; the bare 钱 homograph must not fire + } + b := bankFrom(entries) + traps := []struct{ name, text string }{ + {"送到灶 literal", "厨房里冷了，老妈子把剩下的饭菜送到灶间去重新热一热。"}, + {"修好 repair", "工匠们花了三天，终于把那条被山洪冲垮的石桥修好了。"}, + {"炎势 flame", "山火蔓延，炎势冲天，浓烟滚滚，村民连夜收拾细软。"}, + {"雾气 mist", "清晨的山谷里雾气很重，空气格外清新，露水打湿了草叶。"}, + {"钱 money homograph", "他摸出几个铜钱，买了一碗热汤。"}, // 钱 present but 钱老爷 absent + {"Chekhov negative", "Дмитрий Гуров сидел в ялтинском павильоне и лениво пил чай."}, + } + for _, tr := range traps { + t.Run(tr.name, func(t *testing.T) { + sel := b.Select(tr.text, 1, nil, 0) + if len(sel.injected) != 0 { + t.Errorf("trap fired %d injection(s), want 0: %v", len(sel.injected), injMap(sel)) + } + }) + } +} + +// TestLongestMatchContainment: a shorter key fully inside a longer key's span is +// suppressed (longest-match / whole-entity replacement, A3), but the shorter entity +// still fires when it stands alone. +func TestLongestMatchContainment(t *testing.T) { + entries := []store.GlossaryEntry{ + gl("阿Q", "А-кью", "name", "approved"), + gl("阿Q正传", "«Подлинная история А-кью»", "title", "approved"), + } + b := bankFrom(entries) + // Inside the title: only the title fires. + sel := b.Select("我读了《阿Q正传》。", 1, nil, 0) + m := injMap(sel) + if _, ok := m["阿Q正传"]; !ok { + t.Errorf("title did not fire: %v", m) + } + if _, ok := m["阿Q"]; ok { + t.Errorf("short key 阿Q fired inside the longer title span: %v", m) + } + // Standalone: the character name fires. + sel = b.Select("阿Q走进村子。", 1, nil, 0) + if _, ok := injMap(sel)["阿Q"]; !ok { + t.Error("阿Q did not fire standalone") + } +} + +func TestSingleKeyBanAndAllowShort(t *testing.T) { + banned := gl("炎", "пламя-как-имя", "", "approved") // single Han → banned by default + b := bankFrom([]store.GlossaryEntry{banned}) + if sel := b.Select("炎势冲天。", 1, nil, 0); len(sel.injected) != 0 { + t.Error("single-char key fired despite the A3 ban") + } + // allow_short escape hatch lets it fire. + banned.AllowShort = true + b = bankFrom([]store.GlossaryEntry{banned}) + if sel := b.Select("炎势冲天。", 1, nil, 0); len(sel.injected) != 1 { + t.Error("allow_short did not re-enable the single-char key") + } +} + +func TestBudgetEviction(t *testing.T) { + entries := []store.GlossaryEntry{ + gl("甲甲", "Альфа", "", "approved"), + gl("乙乙", "Бета", "", "auto"), // ambiguous → lower priority + gl("丙丙", "Гамма", "", "approved"), + } + b := bankFrom(entries) + chunk := "甲甲、乙乙、丙丙都在。" + // Unbounded: all three fire, in priority order (the two approved before the auto). + all := b.Select(chunk, 1, nil, 0) + if len(all.injected) != 3 { + t.Fatalf("unbounded: want 3 injected, got %d", len(all.injected)) + } + // A token budget that fits exactly the top TWO priority records (computed with the + // same cost helper Select uses, so the test never drifts from the implementation). + budget := glossaryLineTokens(all.injected[0].entry) + glossaryLineTokens(all.injected[1].entry) + sel := b.Select(chunk, 1, nil, budget) + if len(sel.injected) != 2 || len(sel.evicted) != 1 { + t.Fatalf("budget: injected=%d evicted=%d (budget=%d)", len(sel.injected), len(sel.evicted), budget) + } + // The auto (ambiguous) term is the one evicted (confirmed>ambiguous priority, F2). + if sel.evicted[0].entry.src != "乙乙" { + t.Errorf("eviction priority wrong: evicted %s (want the auto term 乙乙)", sel.evicted[0].entry.src) + } +} + +// TestSelectDeterminism runs Select many times over a multi-hit chunk and asserts the +// injected order is byte-identical — guards against a map-iteration leak (the +// determinism invariant: the injected bytes enter request_hash). +func TestSelectDeterminism(t *testing.T) { + b := bankFrom(specGlossary()) + chunk := "阿Q走进鲁镇，遇见鲁四老爷，谈起修为。" + first := selKey(b.Select(chunk, 1, nil, 0)) + for i := 0; i < 200; i++ { + if got := selKey(b.Select(chunk, 1, nil, 0)); got != first { + t.Fatalf("non-deterministic Select: run %d gave %q, first was %q", i, got, first) + } + } +} + +func selKey(sel memorySelection) string { + s := "" + for _, p := range sel.injected { + s += p.entry.src + ":" + string(p.disp) + "|" + } + return s +} + +// TestEmptyDstNotMatchable covers self-review #1: a ruby candidate (dst="") must not be +// matchable — it renders nothing, so admitting it would waste budget, evict a real line, +// and inflate the exact-hit count. +func TestEmptyDstNotMatchable(t *testing.T) { + phantom := gl("甲甲", "", "", "auto") // ruby-style candidate, no dst + real := gl("乙乙", "Настоящий", "", "auto") + b := bankFrom([]store.GlossaryEntry{phantom, real}) + // Tiny budget that would fit exactly one line: the phantom must not steal it. + budget := glossaryLineTokens(&memoryEntry{src: "乙乙", dst: "Настоящий"}) + sel := b.Select("甲甲和乙乙都在场。", 1, nil, budget) + if len(sel.injected) != 1 || sel.injected[0].entry.src != "乙乙" { + t.Fatalf("empty-dst phantom polluted selection: injected=%v", injMap(sel)) + } + if renderGlossaryBlock(sel.injected) == "" { + t.Error("a renderable line was evicted by an empty-dst phantom") + } + // Unbounded: the phantom never appears as an exact hit. + sel = b.Select("甲甲和乙乙都在场。", 1, nil, 0) + if _, ok := injMap(sel)["甲甲"]; ok { + t.Error("empty-dst phantom counted as an exact hit") + } +} + +// TestSpoilerBeforeSuppression covers self-review #6: a spoiler-blocked LONGER key must +// not suppress a valid SHORTER name nested inside it. +func TestSpoilerBeforeSuppression(t *testing.T) { + short := gl("林动", "Линь Дун", "", "approved") + long := gl("林动的父亲", "отец Линь Дуна", "", "approved") + long.UntilCh = 3 // a spoiler: valid only through chapter 3 + b := bankFrom([]store.GlossaryEntry{short, long}) + + // Chapter 10: the longer key is spoiler-blocked; the shorter valid name must survive. + sel := b.Select("林动的父亲缓缓转身。", 10, nil, 0) + if _, ok := injMap(sel)["林动"]; !ok { + t.Errorf("valid shorter name dropped by a spoiler-blocked longer key: %v", injMap(sel)) + } + if _, ok := injMap(sel)["林动的父亲"]; ok { + t.Error("spoiler-blocked longer key was injected") + } + if len(sel.rejected) != 1 || sel.rejected[0].entry.src != "林动的父亲" { + t.Errorf("spoiler-blocked longer key not logged as rejected: %+v", sel.rejected) + } + // Chapter 2 (longer key in window): longest-match still prefers the longer entity. + sel = b.Select("林动的父亲缓缓转身。", 2, nil, 0) + if _, ok := injMap(sel)["林动"]; ok { + t.Error("longest-match failed when the longer key is spoiler-valid") + } + if _, ok := injMap(sel)["林动的父亲"]; !ok { + t.Error("spoiler-valid longer key was not injected at ch2") + } +} + +// TestBudgetFloorFree covers self-review #7: the per-line budget cost is floor-free, so a +// block of short name lines is not ~1.85×-overcounted and needlessly evicted. +func TestBudgetFloorFree(t *testing.T) { + var entries []store.GlossaryEntry + srcs := []string{"甲甲", "乙乙", "丙丙", "丁丁", "戊戊", "己己", "庚庚", "辛辛"} + dsts := []string{"Ко", "Оцу", "Хэй", "Тэй", "Бо", "Ки", "Ко2", "Син"} + for i := range srcs { + entries = append(entries, gl(srcs[i], dsts[i], "", "approved")) + } + b := bankFrom(entries) + chunk := strings.Join(srcs, "、") + "都在场。" + all := b.Select(chunk, 1, nil, 0) + if len(all.injected) != 8 { + t.Fatalf("want 8 injected, got %d", len(all.injected)) + } + // A budget equal to the EXACT sum of the (floor-free) line costs fits all 8. + budget := 0 + for _, p := range all.injected { + budget += glossaryLineTokens(p.entry) + } + sel := b.Select(chunk, 1, nil, budget) + if len(sel.injected) != 8 || len(sel.evicted) != 0 { + t.Errorf("floor-free budget=%d should fit all 8: injected=%d evicted=%d", budget, len(sel.injected), len(sel.evicted)) + } +} + +func TestInjectivityCollision(t *testing.T) { + entries := []store.GlossaryEntry{ + {BookID: "b", Src: "甲", Dst: "Мастер", Status: "approved"}, + {BookID: "b", Src: "乙", Dst: "Мастер", Status: "approved"}, // same dst → collision (B2) + {BookID: "b", Src: "丙", Dst: "Ученик", Status: "approved"}, + } + cols := injectivityCollisions(entries) + if len(cols) != 1 { + t.Fatalf("want 1 dst collision, got %d: %v", len(cols), cols) + } +} diff --git a/backend/internal/pipeline/mempostcheck.go b/backend/internal/pipeline/mempostcheck.go new file mode 100644 index 0000000..e8e0e9c --- /dev/null +++ b/backend/internal/pipeline/mempostcheck.go @@ -0,0 +1,109 @@ +package pipeline + +import ( + "strings" + "unicode" +) + +// mempostcheck.go: the post-check (registry E1/E2 / research/13 Q5) — the main, almost +// free detector of silent degradation. For every injected record whose SRC actually +// fired in THIS chunk, it asks: did an accepted DST form appear in the output? A miss +// catches BOTH failure modes — "the model ignored the glossary" (soft-following loses +// 17–36% of terms, 2310.05824) AND "we injected the wrong dst and the model obeyed it" +// (the owner's fear, 2510.00829). It is a post-CHECK, never a blind post-REPLACE (E2): +// forcing a dictionary form into an oblique Russian case breaks agreement (46% of +// constrained-model errors in en-cs are agreement, 2106.12398), so v1 only FLAGS. +// +// Decl-awareness is load-bearing (research/14 §2, quantified): a naive \b-regexp on the +// base form gives 18–36% FALSE flags on a hard chunk — inflected forms rendered +// CORRECTLY ("Вэйчжуане", "Бородатого Вана", "Дэна", "сюцая") that the boundary regexp +// misses. So the check matches against the STORED decl forms (filled at term-commit), +// not the bare lemma. BUT stored-decl reliability is itself a function of decl +// COMPLETENESS — an OOV translit name whose forms were under-filled re-introduces the +// same false flags. That is why v1 keeps the post-check a FLAGGER (observable, in the +// retrieval-state), NOT a hard disposition gate, until the false-flag rate is MEASURED +// on real inflected Russian (E1 — memory_e1_test.go). A hard gate flips on only after +// the owner validates precision (mirrors the coverage gate's opt-in, D12 Q4). +// +// Sticky records are DELIBERATELY excluded: a sticky entry's src is NOT in this chunk +// (that is why scene-inertia carries it), so its dst is legitimately absent from the +// output — post-checking it would false-flag every pronominal chunk. Only records whose +// key exactly fired here are expected in the output. + +// postcheckMiss is one flagged term: an injected record whose src fired but whose dst +// (any accepted form) is absent from the output. +type postcheckMiss struct { + Src string `json:"src"` + Dst string `json:"dst"` + Disp string `json:"disp"` // confirmed | ambiguous (A2: ambiguous injections force a post-check) +} + +// postcheck runs the decl-aware check over the injected records against the model +// output. Pure and deterministic. Returns the misses (empty = all fired terms present). +func (b *MemoryBank) postcheck(injected []pickedEntry, output string) []postcheckMiss { + nout := normalizeTargetForm(output) + var misses []postcheckMiss + noutRunes := []rune(nout) + for _, p := range injected { + if p.via == "sticky" { // sticky context is not expected in the output + continue + } + if strings.TrimSpace(p.entry.dst) == "" { // a ruby candidate with no dst yet — nothing to check + continue + } + if !dstFormPresent(p.entry, noutRunes) { + misses = append(misses, postcheckMiss{Src: p.entry.src, Dst: p.entry.dst, Disp: string(p.disp)}) + } + } + return misses +} + +// dstFormPresent reports whether any accepted dst form of the entry appears in the +// normalized output as a WHOLE WORD (bounded by non-letters or string edges), not a bare +// substring. The whole-word rule closes self-review #3: a DROPPED short translit name +// («Ван») must not be masked by an unrelated common word that merely contains its letters +// («караВАН», «диВАН», «ИВАН» — a different person). Decl-aware (the research/14 +// requirement): the stored decl forms carry the inflections, so precision and recall are +// both satisfied when decl is complete; the base-dst fallback (empty decl) is the naive +// case the E1 measurement quantifies. noutRunes is the normalized output pre-decomposed. +func dstFormPresent(e *memoryEntry, noutRunes []rune) bool { + forms := e.declForms + if len(forms) == 0 { + forms = []string{normalizeTargetForm(e.dst)} + } + for _, f := range forms { + if f != "" && containsWholeWord(noutRunes, []rune(f)) { + return true + } + } + return false +} + +// containsWholeWord reports whether form occurs in text with a letter-boundary on both +// ends (the char before the match is not a letter or is the start; likewise after). Runs +// on runes so Cyrillic boundaries are correct. O(len(text)·len(form)) — fine, the +// post-check runs once per chunk over a handful of short forms. +func containsWholeWord(text, form []rune) bool { + n := len(form) + if n == 0 || n > len(text) { + return false + } + for i := 0; i+n <= len(text); i++ { + if !runesEqual(text[i:i+n], form) { + continue + } + if (i == 0 || !unicode.IsLetter(text[i-1])) && (i+n == len(text) || !unicode.IsLetter(text[i+n])) { + return true + } + } + return false +} + +func runesEqual(a, b []rune) bool { + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/backend/internal/pipeline/memseed.go b/backend/internal/pipeline/memseed.go new file mode 100644 index 0000000..84fe3d3 --- /dev/null +++ b/backend/internal/pipeline/memseed.go @@ -0,0 +1,253 @@ +package pipeline + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "unicode" + + "gopkg.in/yaml.v3" + + "textmachine/backend/internal/store" +) + +// memseed.go: the DETERMINISTIC seeding of the glossary from its two milestone inputs — +// a manual seed file (curated approved/draft terms) and the captured ruby readings +// (classified into auto candidates). No LLM (autopopulation/adjudication are a later +// milestone). The output is a []store.GlossaryEntry that seedGlossary REPLACES into the +// book (idempotent full-replace: same inputs → same rows, an edit converges every column). + +// --- manual seed file (YAML) ---------------------------------------------------- + +type seedFile struct { + Terms []seedTerm `yaml:"terms"` +} + +type seedTerm struct { + Src string `yaml:"src"` + Dst string `yaml:"dst"` + Type string `yaml:"type"` + Sense string `yaml:"sense"` + Gender string `yaml:"gender"` + Speech string `yaml:"speech"` + Decl *seedDecl `yaml:"decl"` + TranslitPolicy string `yaml:"translit_policy"` + FirstPerson string `yaml:"first_person"` + NicknameTranslation string `yaml:"nickname_translation"` + SinceCh int `yaml:"since_ch"` + UntilCh int `yaml:"until_ch"` + Status string `yaml:"status"` + AllowShort bool `yaml:"allow_short"` + Note string `yaml:"note"` + Aliases []seedAlias `yaml:"aliases"` +} + +type seedDecl struct { + Invariant bool `yaml:"invariant"` + Forms []string `yaml:"forms"` +} + +type seedAlias struct { + Alias string `yaml:"alias"` + Type string `yaml:"type"` +} + +// loadGlossarySeed parses a glossary seed YAML into store entries. A manual seed is +// curated, so an empty status defaults to "approved" (the author downgrades explicitly +// with status: draft|auto). Validation is fail-loud: a term without src, or with an +// unknown status, is a config error (a silently-dropped term is exactly the A-class +// hole this bank exists to close). +func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("pipeline: read glossary seed %s: %w", path, err) + } + var sf seedFile + if err := yaml.Unmarshal(raw, &sf); err != nil { + return nil, fmt.Errorf("pipeline: parse glossary seed %s: %w", path, err) + } + var out []store.GlossaryEntry + var problems []string + termKeys := map[[4]string]bool{} // (src,sense,since,until) → detect a duplicate term key + for i, t := range sf.Terms { + if strings.TrimSpace(t.Src) == "" { + problems = append(problems, fmt.Sprintf("term %d: src is required", i)) + continue + } + // A duplicate (src, sense, window) would crash ReplaceGlossary on the store's + // UNIQUE constraint mid-run (aborting the whole book) — catch it here as a clear + // config error instead (self-review #5, sibling of the alias case below). + tk := [4]string{t.Src, t.Sense, fmt.Sprint(t.SinceCh), fmt.Sprint(t.UntilCh)} + if termKeys[tk] { + problems = append(problems, fmt.Sprintf("term %q: duplicate (src, sense=%q, since_ch=%d, until_ch=%d) — a term may appear once per spoiler window", t.Src, t.Sense, t.SinceCh, t.UntilCh)) + continue + } + termKeys[tk] = true + status := t.Status + if status == "" { + status = "approved" + } + switch status { + case "auto", "draft", "approved": + default: + problems = append(problems, fmt.Sprintf("term %q: status must be auto|draft|approved, got %q", t.Src, status)) + continue + } + decl := "" + if t.Decl != nil { + b, mErr := json.Marshal(declInfo{Invariant: t.Decl.Invariant, Forms: t.Decl.Forms}) + if mErr != nil { + return nil, mErr + } + decl = string(b) + } + e := store.GlossaryEntry{ + BookID: "", Src: t.Src, Dst: t.Dst, Type: t.Type, Sense: t.Sense, + Gender: t.Gender, Speech: t.Speech, Decl: decl, + TranslitPolicy: t.TranslitPolicy, FirstPerson: t.FirstPerson, + NicknameTranslation: t.NicknameTranslation, + SinceCh: t.SinceCh, UntilCh: t.UntilCh, Status: status, + AllowShort: t.AllowShort, Source: "seed", Note: t.Note, + } + // Dedup aliases within a term by surface (keep the first type). A repeated alias + // surface — a copy-paste, or the same name annotated with two types — would + // otherwise crash ReplaceGlossary on UNIQUE(book_id,term_id,alias) and abort the + // whole book run (self-review #5). Deduping is the forgiving fix: the surface is + // what matters for matching, and a redundant alias is a harmless authoring slip. + seenAlias := map[string]bool{} + for _, a := range t.Aliases { + if strings.TrimSpace(a.Alias) == "" { + problems = append(problems, fmt.Sprintf("term %q: an alias is empty", t.Src)) + continue + } + if seenAlias[a.Alias] { + continue + } + seenAlias[a.Alias] = true + e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: a.Alias, AliasType: a.Type}) + } + out = append(out, e) + } + if len(problems) > 0 { + return nil, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - ")) + } + return out, nil +} + +// --- ruby → auto candidates ----------------------------------------------------- + +// rubyToCandidates classifies the captured ruby readings into auto glossary candidates +// (registry §Ruby-seed / §8). One candidate per BASE (the dominant reading by occurrence, +// ties broken lexically) so variant readings never collide on UNIQUE(src,sense,window) +// — the (base,reading) source of truth stays in ruby_readings. A base already covered by +// the manual seed is skipped (the curated entry wins). Every candidate is status=auto +// with NO dst: the reading is the Polivanov bridge (B6, Phase 2), not a dst — so nothing +// is auto-injected, and a human/batch-judge promotes it (never a blind name-lock, which +// would be a mistranslation). The ruby_class is a deterministic HINT for that promotion. +func rubyToCandidates(readings []store.RubyReading, manualSrcs map[string]bool) []store.GlossaryEntry { + // Group by base → the dominant (base,reading). + type best struct { + reading string + occ int + firstCh int + } + byBase := map[string]best{} + order := []string{} + for _, rr := range readings { + if manualSrcs[rr.Base] { + continue // the curated seed already owns this src + } + cur, ok := byBase[rr.Base] + if !ok { + byBase[rr.Base] = best{rr.Reading, rr.Occurrences, rr.FirstChapter} + order = append(order, rr.Base) + continue + } + // Dominant reading: higher occurrences wins; tie → lexically smaller reading. + if rr.Occurrences > cur.occ || (rr.Occurrences == cur.occ && rr.Reading < cur.reading) { + cur.reading, cur.occ = rr.Reading, rr.Occurrences + } + if rr.FirstChapter < cur.firstCh { // earliest appearance across readings + cur.firstCh = rr.FirstChapter + } + byBase[rr.Base] = cur + } + sort.Strings(order) // deterministic output regardless of input order + var out []store.GlossaryEntry + for _, base := range order { + b := byBase[base] + class := classifyRubyReading(base, b.reading) + typ := "" + if class == rubyClassName { + typ = "name" + } + out = append(out, store.GlossaryEntry{ + Src: base, Dst: "", Type: typ, Status: "auto", Source: "ruby", + RubyReading: b.reading, RubyClass: class, SinceCh: b.firstCh, + Confidence: b.occ, + Note: "ruby auto-candidate (" + class + "); reading is the Polivanov bridge, not a dst — promote before use", + }) + } + return out +} + +// ruby classifier classes (deterministic HINTS for human promotion). +const ( + rubyClassName = "name_candidate" // all-Han base + all-kana reading: the furigana-name SHAPE + rubyClassGloss = "gloss_candidate" // reading carries Han: likely an author double-reading → footnote, NOT a lock + rubyClassAmbiguous = "ambiguous" // anything else → flag to a human +) + +// classifyRubyReading is the deterministic clear-case rule (§8). It NEVER auto-locks: +// the all-Han+all-kana shape is a name-lock CANDIDATE (it could still be a double-reading +// like 強敵→とも "friend", indistinguishable without a yomi dictionary — Phase 2), so it is +// flagged for human confirmation, not locked. A reading that carries kanji is a clear +// author's double-reading/gloss → a footnote candidate. Over-flagging is safe; a blind +// lock is a mistranslation. Pure and deterministic. +func classifyRubyReading(base, reading string) string { + if runesAnyHan(reading) { + return rubyClassGloss // the "reading" is itself a word (semantic gloss / double-reading) + } + if runesAllHan(base) && runesAllKana(reading) { + return rubyClassName + } + return rubyClassAmbiguous +} + +func runesAllHan(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if !unicode.Is(unicode.Han, r) { + return false + } + } + return true +} + +func runesAnyHan(s string) bool { + for _, r := range s { + if unicode.Is(unicode.Han, r) { + return true + } + } + return false +} + +func runesAllKana(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r == 'ー' || r == '・' { // prolonged-sound mark, middle dot — allowed within kana readings + continue + } + if !unicode.Is(unicode.Hiragana, r) && !unicode.Is(unicode.Katakana, r) { + return false + } + } + return true +} diff --git a/backend/internal/pipeline/memseed_test.go b/backend/internal/pipeline/memseed_test.go new file mode 100644 index 0000000..6fa6f30 --- /dev/null +++ b/backend/internal/pipeline/memseed_test.go @@ -0,0 +1,162 @@ +package pipeline + +import ( + "os" + "path/filepath" + "testing" + + "textmachine/backend/internal/store" +) + +func writeSeed(t *testing.T, content string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "glossary.yaml") + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func TestLoadGlossarySeed(t *testing.T) { + seed := writeSeed(t, ` +terms: + - src: 阿Q + dst: А-кью + type: name + decl: { invariant: true, forms: ["А-кью"] } + aliases: + - { alias: 阿桂, type: прозвище } + - src: 王胡 + dst: Бородатый Ван + status: draft + decl: { forms: ["Бородатый Ван", "Бородатого Вана"] } +`) + entries, err := loadGlossarySeed(seed) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 { + t.Fatalf("want 2 entries, got %d", len(entries)) + } + // Empty status defaults to approved (a manual seed is curated). + if entries[0].Status != "approved" { + t.Errorf("default status = %q, want approved", entries[0].Status) + } + if entries[0].Source != "seed" { + t.Errorf("source = %q, want seed", entries[0].Source) + } + if len(entries[0].Aliases) != 1 || entries[0].Aliases[0].Alias != "阿桂" { + t.Errorf("alias not parsed: %+v", entries[0].Aliases) + } + if entries[0].Decl == "" { + t.Error("decl JSON not marshaled") + } + if entries[1].Status != "draft" { + t.Errorf("explicit status not honored: %q", entries[1].Status) + } +} + +func TestLoadGlossarySeedRejectsBad(t *testing.T) { + // Missing src → fail loud (a silently-dropped term is the A-class hole). + if _, err := loadGlossarySeed(writeSeed(t, "terms:\n - dst: без ключа\n")); err == nil { + t.Error("expected error for a term without src") + } + // Unknown status → fail loud. + if _, err := loadGlossarySeed(writeSeed(t, "terms:\n - src: 甲\n status: bogus\n")); err == nil { + t.Error("expected error for an unknown status") + } +} + +// TestSeedAliasDedupAndDuplicateTerm covers self-review #5: a duplicate alias within a +// term must be deduped (not crash the run on UNIQUE), and a duplicate (src,sense,window) +// term must fail loud (not crash mid-run). +func TestSeedAliasDedupAndDuplicateTerm(t *testing.T) { + // Duplicate alias surface (annotated with two types) → deduped, no error. + entries, err := loadGlossarySeed(writeSeed(t, ` +terms: + - src: 四叔 + dst: Четвёртый дядюшка + aliases: + - { alias: 鲁四老爷, type: имя } + - { alias: 鲁四老爷, type: титул } +`)) + if err != nil { + t.Fatalf("duplicate alias should be deduped, not error: %v", err) + } + if len(entries) != 1 || len(entries[0].Aliases) != 1 { + t.Fatalf("alias not deduped: %+v", entries) + } + // Duplicate (src, sense, window) → fail loud. + if _, err := loadGlossarySeed(writeSeed(t, ` +terms: + - { src: 道, dst: путь } + - { src: 道, dst: дао } +`)); err == nil { + t.Error("duplicate (src,sense,window) term must fail loud") + } + // Same src, different SENSE → allowed (polysemy). + if _, err := loadGlossarySeed(writeSeed(t, ` +terms: + - { src: 道, dst: путь, sense: way } + - { src: 道, dst: дао, sense: dao } +`)); err != nil { + t.Errorf("distinct senses should be allowed: %v", err) + } +} + +func TestClassifyRubyReading(t *testing.T) { + cases := []struct { + base, reading, want string + }{ + {"鈴木", "すずき", rubyClassName}, // all-Han base + all-kana reading → name candidate + {"強敵", "とも", rubyClassName}, // SAME shape (a double-reading!) — deliberately NOT distinguishable offline → still a CANDIDATE, never auto-locked + {"強敵", "きょうてき", rubyClassName}, // real reading, same class + {"泣蟲", "泣き虫", rubyClassGloss}, // reading carries kanji → a semantic gloss/double-reading → footnote, not a lock + {"ゲーム", "gēmu", rubyClassAmbiguous}, // base is kana → not a plain furigana-name + } + for _, c := range cases { + if got := classifyRubyReading(c.base, c.reading); got != c.want { + t.Errorf("classifyRubyReading(%q,%q) = %q; want %q", c.base, c.reading, got, c.want) + } + } +} + +func TestRubyToCandidates(t *testing.T) { + readings := []store.RubyReading{ + {BookID: "b", Base: "鈴木", Reading: "すずき", FirstChapter: 2, Occurrences: 9}, + {BookID: "b", Base: "鈴木", Reading: "スズキ", FirstChapter: 5, Occurrences: 3}, // variant → deduped by base + {BookID: "b", Base: "田中", Reading: "たなか", FirstChapter: 1, Occurrences: 4}, + {BookID: "b", Base: "図書館", Reading: "としょかん", FirstChapter: 1, Occurrences: 20}, + } + // 図書館 is owned by the manual seed → its ruby candidate is skipped. + manual := map[string]bool{"図書館": true} + cands := rubyToCandidates(readings, manual) + + if len(cands) != 2 { + t.Fatalf("want 2 candidates (鈴木, 田中; 図書館 skipped), got %d: %+v", len(cands), cands) + } + byBase := map[string]store.GlossaryEntry{} + for _, c := range cands { + byBase[c.Src] = c + if c.Status != "auto" || c.Source != "ruby" || c.Dst != "" { + t.Errorf("ruby candidate must be auto/ruby with no dst: %+v", c) + } + } + // 鈴木 deduped to the DOMINANT reading (すずき, 9 occ > スズキ, 3) with the earliest chapter. + suzuki := byBase["鈴木"] + if suzuki.RubyReading != "すずき" { + t.Errorf("dominant reading = %q, want すずき", suzuki.RubyReading) + } + if suzuki.SinceCh != 2 { + t.Errorf("since_ch = %d, want the earliest (2)", suzuki.SinceCh) + } + if suzuki.Confidence != 9 { + t.Errorf("confidence = %d, want 9 (occurrences of the dominant reading)", suzuki.Confidence) + } + if suzuki.RubyClass != rubyClassName { + t.Errorf("ruby_class = %q, want %q", suzuki.RubyClass, rubyClassName) + } + if _, skipped := byBase["図書館"]; skipped { + t.Error("図書館 should have been skipped (owned by the manual seed)") + } +} diff --git a/backend/internal/pipeline/render.go b/backend/internal/pipeline/render.go index 5d9ca40..c9cb432 100644 --- a/backend/internal/pipeline/render.go +++ b/backend/internal/pipeline/render.go @@ -150,11 +150,28 @@ func snippetStr(s string) string { return s } -// Messages builds the wire-neutral message list for a stage. Layout по Р5: -// system (стабильный префикс, кэш-граница на последнем стабильном блоке) → -// user (волатильный хвост: чанк/черновик). Инъекция глоссария/STM встанет -// между ними в Фазе 1, не меняя схемы. +// Messages builds the wire-neutral message list for a stage with no memory injection +// (the Веха-0/2 behaviour). Layout по Р5: system (стабильный префикс, кэш-граница) → +// user (волатильный хвост). Kept as the zero-injection convenience over +// MessagesWithInjection so existing callers/tests are untouched. func Messages(tpl *PromptTemplate, v RenderVars) ([]llm.Message, error) { + return MessagesWithInjection(tpl, v, "") +} + +// MessagesWithInjection builds the stage messages with the memory bank v2 injection +// surface (шаг 4 §C, orchestrator variant "а" — a code-assembled message, NOT a new +// RenderVars placeholder, so the fixed template stays stable). Layout по Р5/A6: +// +// system(стабильный префикс, CacheBoundary) → [injection: глоссарий/STM] → user(волатильный хвост) +// +// The injection is its OWN message placed AFTER the cache boundary (which stays on the +// stable system prefix — the DeepSeek byte-prefix cache still hits; Anthropic's removed, +// no cache_control concern). It is NOT part of ch.Text, so it never leaks into the +// coverage len_ratio or the {{text}} placeholder (the 3a ch.Text contract). Because the +// injection is a message, it is folded into request_hash/content_hash automatically — a +// changed injection is a changed request (a resumed chunk reproduces it deterministically +// from the frozen bank). An empty injection yields the exact 2-message list of Messages. +func MessagesWithInjection(tpl *PromptTemplate, v RenderVars, injection string) ([]llm.Message, error) { sys, err := Render(tpl.System, v) if err != nil { return nil, err @@ -163,10 +180,13 @@ func Messages(tpl *PromptTemplate, v RenderVars) ([]llm.Message, error) { if err != nil { return nil, err } - return []llm.Message{ - {Role: "system", Content: sys, CacheBoundary: true}, - {Role: "user", Content: user}, - }, nil + msgs := make([]llm.Message, 0, 3) + msgs = append(msgs, llm.Message{Role: "system", Content: sys, CacheBoundary: true}) + if strings.TrimSpace(injection) != "" { + msgs = append(msgs, llm.Message{Role: "system", Content: injection, CacheBoundary: false}) + } + msgs = append(msgs, llm.Message{Role: "user", Content: user}) + return msgs, nil } // RequestHash is the checkpoint/resume key (§3.1): a stable hash of everything diff --git a/backend/internal/pipeline/render_memory_test.go b/backend/internal/pipeline/render_memory_test.go new file mode 100644 index 0000000..1ee2dd9 --- /dev/null +++ b/backend/internal/pipeline/render_memory_test.go @@ -0,0 +1,93 @@ +package pipeline + +import ( + "strings" + "testing" +) + +func TestMessagesInjectionLayout(t *testing.T) { + tpl := writeTemplate(t, "Стабильный системный префикс.\n---USER---\nПереведи: {{text}}") + v := RenderVars{Book: testBook(), Text: "исходный чанк"} + injection := "ГЛОССАРИЙ:\n阿Q → А-кью" + + msgs, err := MessagesWithInjection(tpl, v, injection) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 3 { + t.Fatalf("want 3 messages (system, injection, user), got %d", len(msgs)) + } + // [0] stable system prefix carries the cache boundary. + if msgs[0].Role != "system" || !msgs[0].CacheBoundary { + t.Errorf("msg[0] must be the system prefix with CacheBoundary=true: %+v", msgs[0]) + } + // [1] injection sits AFTER the boundary (volatile → CacheBoundary=false). + if msgs[1].Role != "system" || msgs[1].CacheBoundary || msgs[1].Content != injection { + t.Errorf("msg[1] must be the injection with CacheBoundary=false: %+v", msgs[1]) + } + // [2] the volatile user chunk. It must contain ONLY the rendered chunk, never the + // glossary (ch.Text stays clean — the 3a contract; the coverage gate reads ch.Text). + if msgs[2].Role != "user" { + t.Errorf("msg[2] must be user: %+v", msgs[2]) + } + if strings.Contains(msgs[2].Content, "ГЛОССАРИЙ") || strings.Contains(msgs[2].Content, "阿Q") { + t.Errorf("glossary leaked into the user (ch.Text) message: %q", msgs[2].Content) + } + if !strings.Contains(msgs[2].Content, "исходный чанк") { + t.Errorf("user message lost the chunk: %q", msgs[2].Content) + } +} + +func TestMessagesEmptyInjectionIdentical(t *testing.T) { + tpl := writeTemplate(t, "Системный префикс.\n---USER---\nТекст: {{text}}") + v := RenderVars{Book: testBook(), Text: "чанк"} + plain, _ := Messages(tpl, v) + withEmpty, _ := MessagesWithInjection(tpl, v, " ") // whitespace-only → no injection + if len(plain) != 2 || len(withEmpty) != 2 { + t.Fatalf("empty injection must yield 2 messages: %d vs %d", len(plain), len(withEmpty)) + } + for i := range plain { + if plain[i] != withEmpty[i] { + t.Errorf("empty-injection message %d differs from plain: %+v vs %+v", i, withEmpty[i], plain[i]) + } + } +} + +// TestInjectionChangesRequestHash: the injection is folded into request_hash via the +// messages, so a different glossary is a different request (a changed approved term +// re-derives the chunk, never serves a stale checkpoint). +func TestInjectionChangesRequestHash(t *testing.T) { + tpl := writeTemplate(t, "Префикс.\n---USER---\n{{text}}") + v := RenderVars{Book: testBook(), Text: "чанк"} + none, _ := Messages(tpl, v) + inj, _ := MessagesWithInjection(tpl, v, "ГЛОССАРИЙ:\n阿Q → А-кью") + hNone := RequestHash("b", 1, 0, 0, "draft", "translator", "m", 0.3, "off", false, 1024, "snap", none) + hInj := RequestHash("b", 1, 0, 0, "draft", "translator", "m", 0.3, "off", false, 1024, "snap", inj) + if hNone == hInj { + t.Error("injection did not change request_hash — a glossary change would silently hit a stale checkpoint") + } +} + +func TestRenderGlossaryBlock(t *testing.T) { + confirmed := pickedEntry{entry: &memoryEntry{src: "阿Q", dst: "А-кью", status: "approved"}, disp: memConfirmed} + ambiguous := pickedEntry{entry: &memoryEntry{src: "小D", dst: "Малыш Дэ", status: "auto"}, disp: memAmbiguous} + emptyDst := pickedEntry{entry: &memoryEntry{src: "鈴木", dst: "", status: "auto"}, disp: memAmbiguous} + + block := renderGlossaryBlock([]pickedEntry{confirmed, ambiguous, emptyDst}) + if !strings.Contains(block, "阿Q → А-кью") { + t.Errorf("confirmed line missing: %q", block) + } + if !strings.Contains(block, "小D → Малыш Дэ ⟨проверить⟩") { + t.Errorf("ambiguous line missing its unverified tag: %q", block) + } + if strings.Contains(block, "鈴木") { + t.Errorf("empty-dst candidate should be skipped (nothing to inject): %q", block) + } + // Nothing to inject → empty block (лучше ничего). + if renderGlossaryBlock(nil) != "" { + t.Error("empty selection must render an empty block") + } + if renderGlossaryBlock([]pickedEntry{emptyDst}) != "" { + t.Error("only-empty-dst selection must render an empty block") + } +} diff --git a/backend/internal/pipeline/runner.go b/backend/internal/pipeline/runner.go index 2dc0b47..d4a2ac5 100644 --- a/backend/internal/pipeline/runner.go +++ b/backend/internal/pipeline/runner.go @@ -56,6 +56,13 @@ type Runner struct { clients map[string]llm.LLMClient templates map[string]*PromptTemplate + + // memory is the book's glossary FROZEN for this job (materialized once in + // TranslateBook, after seeding, before snapshotID). Its Version() is the F1 + // content-hash folded into the snapshot; its Select drives per-chunk injection. + // nil until materialized (report path / no glossary) → memoryVersion() falls back + // to the empty-materialization hash, a stable constant. + memory *MemoryBank } // NewRunner loads all three configs rooted at book.yaml and opens the project @@ -220,19 +227,21 @@ func (r *Runner) escalationBudgetRemains() (bool, error) { return spent < budget, nil } -// memoryVersion is the content-hash of the deterministically materialized -// injected memory (approved glossary + summaries + series-bible), the memory -// component of the snapshot (D5.2/D8). Until the memory bank v2 migration lands -// the materialization is empty, so the hash is a stable constant; once memory -// enters msgs this changes and the resnapshot-gate fires loudly instead of a -// stale checkpoint re-paying a diverged translation. STM is excluded (rebuilt -// from checkpoints, §3.2). +// memoryVersion is the content-hash of the deterministically materialized injected +// memory (the frozen APPROVED glossary rows + the normalization/matcher algorithm +// versions), the memory component of the snapshot (D5.2/D8, F1 CLOSED). It is the +// Version() of the bank materialized once before the loop, so a change to the approved +// glossary — or to the deterministic machinery (trad→simp table, matcher) — fires the +// resnapshot gate LOUDLY instead of a stale checkpoint re-paying a diverged translation. +// nil bank (report path / a book with no glossary) → the empty-materialization hash, a +// stable constant. STM is excluded (rebuilt from checkpoints, §3.2). Auto/draft rows are +// excluded per D8 (their injected-content changes are caught at the per-chunk +// content_hash level; a future autopopulation milestone revisits this). func (r *Runner) memoryVersion() string { - h := sha256.New() - // Domain tag + empty materialization; the store query lands with the v2 - // memory schema and feeds ORDER BY-stable rows here. - h.Write([]byte("tm-memory-v1\x00")) - return hex.EncodeToString(h.Sum(nil)) + if r.memory != nil { + return r.memory.Version() + } + return computeMemoryVersion(nil, r.Pipeline.Gates.Glossary.PostcheckGate) } // snapshotID materializes the job-context snapshot (§3.2): brief_hash, chunker @@ -324,6 +333,12 @@ func (r *Runner) snapshotID() (id, payload string, err error) { // хэш стабилен; когда память войдёт в msgs, поле сменится и resnapshot- // гейт сработает громко. STM сюда НЕ входит (пересобирается из чекпоинтов). MemoryVersion string `json:"memory_version"` + // PostcheckGate — the memory-bank post-check gate (E1). When true a post-check + // miss flips the chunk to flagged; folding its on/off state makes toggling it a + // loud --resnapshot (it changes a checkpoint's RESOLVED disposition), not a silent + // re-verdict on resume (same class as coverage/classifier). The post-check + // ALGORITHM version is already inside MemoryVersion (memoryMatchVersion). + PostcheckGate bool `json:"postcheck_gate"` // Coverage — excision QA-gate config (D12 Q3). It does not touch the wire, // but it determines a checkpoint's RESOLVED verdict, so a gate change must be // a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot). @@ -346,6 +361,7 @@ func (r *Runner) snapshotID() (id, payload string, err error) { CacheTTL: r.Pipeline.Context.CacheTTL, }, MemoryVersion: r.memoryVersion(), + PostcheckGate: r.Pipeline.Gates.Glossary.PostcheckGate, Coverage: r.coverageSnapshot(), } for _, st := range r.Pipeline.Stages { @@ -480,8 +496,23 @@ func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) { return nil, err } + // Persist captured ruby readings (idempotent; consumed by seedGlossary below into + // auto glossary candidates — шаг 4). Never injected into a prompt here (§7d); a + // resume re-persists the same rows at $0. + if err := r.persistRuby(doc.Ruby); err != nil { + return nil, fmt.Errorf("pipeline: persist ruby readings: %w", err) + } + + // Seed + materialize the glossary BEFORE the snapshot: the frozen approved rows feed + // memoryVersion() → the snapshot (F1), so the snapshot must be computed with the bank + // already materialized. Deterministic and $0 (seed file + classified ruby, no LLM). + if err := r.seedGlossary(ctx); err != nil { + return nil, err + } + // Snapshot is book-level (brief + stage plan + memory), computed once and - // upserted before the loop; every job pins to it. + // upserted before the loop; every job pins to it. memoryVersion() now reflects the + // materialized bank, so a changed approved glossary re-pins loudly (F1). snapID, snapPayload, err := r.snapshotID() if err != nil { return nil, err @@ -495,17 +526,19 @@ func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) { return nil, fmt.Errorf("pipeline: source file %s produced no chunks after normalization", r.Book.SourceFile) } - // Persist captured ruby readings (idempotent; consumed by memory v2 for a - // glossary name-lock — шаг 4). Never injected into a prompt here (§7d), so it - // touches neither the snapshot nor request_hash; a resume re-persists the same - // rows at $0. - if err := r.persistRuby(doc.Ruby); err != nil { - return nil, fmt.Errorf("pipeline: persist ruby readings: %w", err) - } - res := &BookResult{BookID: r.Book.BookID} + // Sticky scene-inertia state (A5): the exact-matched ids of the recent chunks in + // the CURRENT chapter, reset at a chapter boundary (a new chapter is a scene change). + // Rebuilt deterministically each run because translateChunk recomputes the ($0) + // selection for EVERY chunk, resumed ones included — so sticky is resume-stable. + var stickyWin []map[string]bool + prevChapter := 0 for _, ch := range chunks { - outcome, err := r.translateChunk(ctx, snapID, ch) + if ch.Chapter != prevChapter { + stickyWin = nil + prevChapter = ch.Chapter + } + outcome, activeIDs, err := r.translateChunk(ctx, snapID, ch, unionSticky(stickyWin)) if err != nil { // Infra failure: abort. Chunks already committed are checkpointed; // resume continues from here at $0 for the done work. @@ -516,10 +549,73 @@ func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) { if outcome.Disposition == DispFlagged { res.Flagged++ } + // Advance the sticky window (keep the last stickyDepth chunks' exact matches). + stickyWin = append(stickyWin, activeIDs) + if len(stickyWin) > stickyDepth { + stickyWin = stickyWin[len(stickyWin)-stickyDepth:] + } } return res, nil } +// unionSticky merges the recent chunks' exact-matched id sets into one sticky_prev. +func unionSticky(win []map[string]bool) map[string]bool { + if len(win) == 0 { + return nil + } + out := map[string]bool{} + for _, s := range win { + for id := range s { + out[id] = true + } + } + return out +} + +// seedGlossary REPLACES the book's glossary from its deterministic inputs — the manual +// seed file (curated approved/draft terms) and the captured ruby readings (classified +// into auto candidates) — then MATERIALIZES the frozen bank for this job (r.memory). Run +// once before snapshotID: the frozen APPROVED rows are hashed into memoryVersion (F1), so +// editing the seed is a loud --resnapshot. Idempotent (full replace), $0, no LLM. B2 +// approved dst-collisions are logged (not fatal — some collisions are legitimate). +func (r *Runner) seedGlossary(ctx context.Context) error { + var entries []store.GlossaryEntry + if r.Book.GlossarySeed != "" { + seed, err := loadGlossarySeed(r.Book.GlossarySeed) + if err != nil { + return err + } + entries = append(entries, seed...) + } + manualSrcs := map[string]bool{} + for _, e := range entries { + manualSrcs[e.Src] = true + } + ruby, err := r.Store.RubyReadingsForBook(r.Book.BookID) + if err != nil { + return err + } + entries = append(entries, rubyToCandidates(ruby, manualSrcs)...) + for i := range entries { + entries[i].BookID = r.Book.BookID + } + if err := r.Store.ReplaceGlossary(r.Book.BookID, entries); err != nil { + return err + } + rows, err := r.Store.GlossaryForBook(r.Book.BookID) + if err != nil { + return err + } + r.memory = materializeMemory(rows, r.Pipeline.Gates.Glossary.PostcheckGate) + if cols := injectivityCollisions(rows); len(cols) > 0 { + r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)", + "collisions", strings.Join(cols, "; ")) + } + r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID, + "entries", len(rows), "memory_version", r.memory.Version()[:12]) + return nil +} + // persistRuby aggregates the ingested ruby occurrences into one row per // (base, reading) — first_chapter = MIN, occurrences = full-book count — and // REPLACES the book's whole ruby set (store.ReplaceRubyReadings). Idempotent: the @@ -569,14 +665,30 @@ func (r *Runner) persistRuby(readings []RubyReading) error { // translateChunk drives ONE chunk through the stage list. Stages run in order, // each feeding the next; the FIRST flagged stage stops the chunk — later stages -// are recorded `skipped` (no paid edit over a garbage draft, D2). Returns an -// error only on an infra failure. -func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (*ChunkOutcome, error) { +// are recorded `skipped` (no paid edit over a garbage draft, D2). It also runs the +// memory bank v2 hot path: it Selects the glossary injection once ($0, deterministic), +// injects it into the TRANSLATOR stage, post-checks the translator output (E1), and +// persists the per-chunk retrieval-state (observability). Returns the chunk outcome, the +// exact-matched entity ids (the next chunk's sticky_prev, A5), and an error only on an +// infra failure. stickyPrev is the prior chunks' exact matches in this chapter. +func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, stickyPrev map[string]bool) (*ChunkOutcome, map[string]bool, error) { out := &ChunkOutcome{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: DispOK} prev := "" flagged := false var flagReason FlagReason + // Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and + // serialize the translator injection block. Recomputed on every run (resumed chunks + // included) so the sticky window and the injected bytes are resume-stable. + var memSel memorySelection + var translatorInjection string + activeIDs := map[string]bool{} + if r.memory != nil { + memSel = r.memory.Select(ch.Text, ch.Chapter, stickyPrev, r.Pipeline.Context.GlossaryTokenBudget) + translatorInjection = renderGlossaryBlock(memSel.injected) + activeIDs = memSel.activeIDs + } + for stageIdx, st := range r.Pipeline.Stages { if flagged { // An earlier stage flagged → this stage is not attempted or billed. @@ -586,7 +698,7 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (* SnapshotID: snapID, Disposition: string(DispSkipped), FlagReason: string(flagReason), Detail: detail, }); err != nil { - return out, fmt.Errorf("pipeline: record skipped chunk_status: %w", err) + return out, activeIDs, fmt.Errorf("pipeline: record skipped chunk_status: %w", err) } out.Stages = append(out.Stages, StageResult{ Stage: st.Name, Role: st.Role, Model: st.Model, @@ -595,9 +707,17 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (* continue } - sr, err := r.runStage(ctx, st, stageIdx, snapID, ch, prev) + // Inject the glossary ONLY into the translator (шаг 4 scope): its src→dst records + // are matched against the source chunk. Other stages (the monolingual editor) get + // no injection in v1 (a dst-constraint injection for the editor is a fast-follow on + // this same surface). Empty injection → the plain 2-message layout. + injection := "" + if st.Role == roleTranslator { + injection = translatorInjection + } + sr, err := r.runStage(ctx, st, stageIdx, snapID, ch, prev, injection) if err != nil { - return out, err + return out, activeIDs, err } out.Stages = append(out.Stages, *sr) out.CostUSD += sr.CostUSD @@ -609,14 +729,82 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (* prev = sr.Text } + // Post-check the FINAL, exported output (E1): the reader sees the last stage's text + // (the editor's), not the translator draft — the monolingual editor can drift an + // approved term the translator got right, so checking the draft alone would miss it. + // Runs on the fresh OR fully-resumed chunk (both carry the final text through `prev`), + // so it is resume-reproducible. In the default FLAGGER mode a miss is recorded only in + // the retrieval-state (observability); with the opt-in gate a miss flags the chunk + // (glossary_miss). Skipped when a stage already flagged (no usable output to check). + var postMisses []postcheckMiss + outputChecked := false + if r.memory != nil && !flagged && prev != "" { + outputChecked = true + postMisses = r.memory.postcheck(memSel.injected, prev) + if r.Pipeline.Gates.Glossary.PostcheckGate && len(postMisses) > 0 { + flagged = true + flagReason = FlagGlossaryMiss + r.Log.WarnContext(ctx, "glossary post-check gate flagged the chunk", + "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "misses", len(postMisses)) + } + } + + // Persist the per-chunk retrieval-state (registry gate #4: convert silent memory + // degradation into a loud, visible record). Deterministic + idempotent, so a resume + // re-derives the identical row. Only when a glossary is materialized. + if r.memory != nil { + if err := r.persistRetrievalState(snapID, ch, memSel, postMisses, outputChecked); err != nil { + return out, activeIDs, err + } + } + if flagged { out.Disposition = DispFlagged out.FlagReason = flagReason - out.FinalText = "" // garbage/refusal never propagates to the next chunk or export + out.FinalText = "" // garbage/refusal/glossary-miss never propagates to the next chunk or export } else { out.FinalText = prev } - return out, nil + return out, activeIDs, nil +} + +// persistRetrievalState writes the per-chunk observability record from the deterministic +// selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED +// records (what the model saw); spoiler/eviction are the dropped-and-logged totals; +// post-check misses are recorded only when the translator actually produced text. +func (r *Runner) persistRetrievalState(snapID string, ch Chunk, sel memorySelection, misses []postcheckMiss, outputChecked bool) error { + rs := store.RetrievalState{ + BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, SnapshotID: snapID, + } + for _, p := range sel.injected { + if p.via == "sticky" { + rs.NSticky++ + } else { + rs.NExactHits++ + } + if p.disp == memAmbiguous { + rs.NAmbiguousFlagged++ + } + } + rs.NSpoilerBlocked = len(sel.rejected) + rs.NEvicted = len(sel.evicted) + if outputChecked { + rs.NPostcheckMiss = len(misses) + if len(misses) > 0 { + if b, err := json.Marshal(misses); err == nil { + rs.PostcheckDetail = string(b) + } + } + } + ids := make([]string, 0, len(sel.activeIDs)) + for id := range sel.activeIDs { + ids = append(ids, id) + } + sort.Strings(ids) + if b, err := json.Marshal(ids); err == nil { + rs.InjectedIDs = string(b) + } + return r.Store.UpsertRetrievalState(rs) } // runStage runs ONE stage of ONE chunk to a terminal disposition. It first @@ -626,7 +814,7 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (* // subset with a doubled budget up to the regenerate cap, then flag. Returns an // error ONLY on an infra failure; a bad completion is a disposition, never an // error. -func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, snapID string, ch Chunk, prev string) (*StageResult, error) { +func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, snapID string, ch Chunk, prev, injection string) (*StageResult, error) { job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID) if err != nil { return nil, err @@ -649,7 +837,7 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn // fast-path be CONTENT-VERIFIED: the source bytes are not in the snapshot, so // a positional chunk_status row must be checked against the current rendered // content, else an edited source would serve a stale, divergent translation. - msgs, err := Messages(r.templates[st.Name], RenderVars{Book: r.Book, Text: ch.Text, Draft: prev}) + msgs, err := MessagesWithInjection(r.templates[st.Name], RenderVars{Book: r.Book, Text: ch.Text, Draft: prev}, injection) if err != nil { return nil, err } diff --git a/backend/internal/pipeline/runner_memory_test.go b/backend/internal/pipeline/runner_memory_test.go new file mode 100644 index 0000000..16ddab0 --- /dev/null +++ b/backend/internal/pipeline/runner_memory_test.go @@ -0,0 +1,245 @@ +package pipeline + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// runner_memory_test.go: the шаг-4 acceptance — the memory bank v2 mechanism end-to-end +// on a seeded glossary against the mock provider (injection → translate → post-check → +// retrieval-state → resume $0 → resnapshot on an approved change). Per §8, acceptance is +// "the mechanism is correct on a seeded glossary", not "≥98% on a real book". + +const suzukiSeed = ` +terms: + - src: 鈴木 + dst: Судзуки + type: name + status: approved + decl: { invariant: true, forms: ["Судзуки"] } +` + +// suzukiSource is one ja chunk that names 鈴木 (so the glossary fires). +const suzukiSource = "鈴木は静かな図書館へ行った。" + +type wireMsg struct { + Role string + Content string +} + +func bodyMessages(t *testing.T, body string) []wireMsg { + t.Helper() + var req struct { + Messages []wireMsg `json:"messages"` + } + if err := json.Unmarshal([]byte(body), &req); err != nil { + t.Fatalf("bad request body: %v\n%s", err, body) + } + return req.Messages +} + +// translatorBody returns the recorded draft(translator) request body. +func translatorBody(t *testing.T, rec *reqRec) string { + t.Helper() + for _, b := range rec.all() { + if !isEditBody(b) { + return b + } + } + t.Fatal("no translator request recorded") + return "" +} + +func TestRunnerMemoryInjectionAndPostcheck(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isEditBody(body) { + return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД про Судзуки.", "stop" + } + return "Судзуки пошёл в тихую библиотеку.", "stop" // renders the approved dst + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed}) + ctx := context.Background() + + r := newRunner(t, bookPath) + res, err := r.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + if res.Flagged != 0 { + t.Fatalf("flagger default must not flag a correct render: flagged=%d", res.Flagged) + } + + // The TRANSLATOR request carries the injection as its own system message between the + // stable prefix and the user chunk; the dst appears only via injection (not in the ja + // source), so its presence proves the injection landed on the wire. + tb := translatorBody(t, rec) + msgs := bodyMessages(t, tb) + if len(msgs) != 3 { + t.Fatalf("translator must have 3 messages (system, injection, user), got %d: %+v", len(msgs), msgs) + } + if msgs[1].Role != "system" || !strings.Contains(msgs[1].Content, "Судзуки") || !strings.Contains(msgs[1].Content, "ГЛОССАРИЙ") { + t.Errorf("injection message wrong: %+v", msgs[1]) + } + if strings.Contains(msgs[2].Content, "ГЛОССАРИЙ") { + t.Errorf("glossary leaked into the user (ch.Text) message: %q", msgs[2].Content) + } + // The EDITOR gets no injection in v1. + for _, b := range rec.all() { + if isEditBody(b) && strings.Contains(b, "ГЛОССАРИЙ") { + t.Error("editor received a glossary injection (not in v1 scope)") + } + } + + // retrieval-state: one exact hit, post-check ran with 0 misses. + rs, err := r.Store.GetRetrievalState("test-book", 1, 0) + if err != nil || rs == nil { + t.Fatalf("retrieval-state missing: %v", err) + } + if rs.NExactHits != 1 || rs.NPostcheckMiss != 0 { + t.Errorf("retrieval-state wrong: exact=%d miss=%d", rs.NExactHits, rs.NPostcheckMiss) + } + callsRun1 := rec.count() + r.Close() + + // Resume: no new provider calls, $0 (the deterministic memory path re-derives for free). + r2 := newRunner(t, bookPath) + defer r2.Close() + res2, err := r2.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + if rec.count() != callsRun1 || res2.TotalUSD != 0 { + t.Fatalf("resume must be $0 with no new calls: calls=%d (run1=%d) usd=%v", rec.count(), callsRun1, res2.TotalUSD) + } + // retrieval-state re-derived identically. + rs2, _ := r2.Store.GetRetrievalState("test-book", 1, 0) + if rs2 == nil || rs2.NExactHits != 1 || rs2.NPostcheckMiss != 0 { + t.Errorf("retrieval-state not stable on resume: %+v", rs2) + } +} + +func TestRunnerMemoryFlaggerRecordsMissWithoutFlagging(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isEditBody(body) { + return "ОТРЕДАКТИРОВАННЫЙ.", "stop" + } + return "Некто пошёл в библиотеку.", "stop" // DROPS the approved name → a post-check miss + }) + defer srv.Close() + // Default flagger mode (postcheckGate: false). + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed}) + + r := newRunner(t, bookPath) + defer r.Close() + res, err := r.TranslateBook(context.Background()) + if err != nil { + t.Fatal(err) + } + // Flagger mode: the chunk is NOT flagged (translation still exported), but the miss IS + // recorded (silent degradation converted to a loud, visible record). + if res.Flagged != 0 { + t.Errorf("flagger mode must not flag the chunk: flagged=%d", res.Flagged) + } + rs, _ := r.Store.GetRetrievalState("test-book", 1, 0) + if rs == nil || rs.NPostcheckMiss != 1 { + t.Fatalf("post-check miss not recorded: %+v", rs) + } + if !strings.Contains(rs.PostcheckDetail, "鈴木") { + t.Errorf("post-check detail should name the missed src: %q", rs.PostcheckDetail) + } +} + +func TestRunnerMemoryPostcheckGateFlags(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isEditBody(body) { + return "ОТРЕДАКТИРОВАННЫЙ.", "stop" + } + return "Некто пошёл в библиотеку.", "stop" // drops the name + }) + defer srv.Close() + // Opt-in hard gate. + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true}) + + r := newRunner(t, bookPath) + defer r.Close() + res, err := r.TranslateBook(context.Background()) + if err != nil { + t.Fatal(err) + } + if res.Flagged != 1 { + t.Fatalf("gate mode must flag the missed chunk: flagged=%d", res.Flagged) + } + if res.Chunks[0].FlagReason != FlagGlossaryMiss { + t.Errorf("flag reason = %q, want glossary_miss", res.Chunks[0].FlagReason) + } + // The post-check runs on the FINAL exported output, so both stages ran (each ok in + // chunk_status), but the CHUNK is flagged and its text is not exported. + if res.Chunks[0].FinalText != "" { + t.Errorf("a glossary-flagged chunk must not export text, got %q", res.Chunks[0].FinalText) + } + var editorRan bool + for _, st := range res.Chunks[0].Stages { + if st.Role == "editor" && st.Disposition == DispOK { + editorRan = true + } + } + if !editorRan { + t.Error("editor stage should have run (post-check validates the final/editor output, not the draft)") + } +} + +func TestRunnerMemoryResnapshotOnApprovedChange(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isEditBody(body) { + return "ОТРЕДАКТИРОВАННЫЙ про Судзуки.", "stop" + } + return "Судзуки пошёл в библиотеку.", "stop" + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed}) + ctx := context.Background() + + r1 := newRunner(t, bookPath) + if _, err := r1.TranslateBook(ctx); err != nil { + t.Fatal(err) + } + r1.Close() + callsAfterRun1 := rec.count() + + // Change the APPROVED dst in the seed → the frozen materialization changes → the + // snapshot changes → a resume without --resnapshot must fail loud (F1: no silent + // divergent re-pay). + seedPath := filepath.Join(filepath.Dir(bookPath), "glossary-seed.yaml") + if err := os.WriteFile(seedPath, []byte(strings.Replace(suzukiSeed, "Судзуки", "Сузуки", -1)), 0o644); err != nil { + t.Fatal(err) + } + r2 := newRunner(t, bookPath) + _, err := r2.TranslateBook(ctx) + r2.Close() + if err == nil || !strings.Contains(err.Error(), "resnapshot") { + t.Fatalf("changed approved glossary must fail loud mentioning --resnapshot, got: %v", err) + } + if rec.count() != callsAfterRun1 { + t.Fatalf("denied resume must not call the provider: calls=%d (after run1=%d)", rec.count(), callsAfterRun1) + } + + // With --resnapshot the book re-translates under the new approved glossary. + r3 := newRunner(t, bookPath) + defer r3.Close() + r3.Resnapshot = true + if _, err := r3.TranslateBook(ctx); err != nil { + t.Fatal(err) + } + if rec.count() <= callsAfterRun1 { + t.Fatal("resnapshot run must re-call the provider for the re-translation") + } +} diff --git a/backend/internal/pipeline/runner_test.go b/backend/internal/pipeline/runner_test.go index de37272..1683b7f 100644 --- a/backend/internal/pipeline/runner_test.go +++ b/backend/internal/pipeline/runner_test.go @@ -108,6 +108,8 @@ type projectOpts struct { minMaxTokens int bookUSD float64 gatesYAML string // optional gates:/... block appended to pipeline.yaml + glossarySeed string // optional glossary seed YAML content; "" = no glossary + postcheckGate bool // enable the memory post-check hard gate (assumes gatesYAML is empty) } func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string { @@ -142,15 +144,20 @@ models: price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 } `, time.Now().UTC().Format("2006-01-02"), providerURL)) + gatesBlock := o.gatesYAML + if o.postcheckGate { + gatesBlock += "\ngates:\n glossary:\n postcheck_gate: true\n" + } writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(` core: C1 version: 1 defaults: { max_output_ratio: 2.0, min_max_tokens: %d } retries: { regenerate_before_escalate: %d } +context: { glossary_injection: selective, glossary_token_budget: 800 } stages: - { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" } - { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" } -%s`, o.minMaxTokens, o.regenerate, o.gatesYAML)) +%s`, o.minMaxTokens, o.regenerate, gatesBlock)) sourceName := "source.txt" if len(o.epub) > 0 { @@ -159,6 +166,11 @@ stages: } else { writeFile(t, filepath.Join(dir, "source.txt"), o.source) } + glossaryLine := "" + if o.glossarySeed != "" { + writeFile(t, filepath.Join(dir, "glossary-seed.yaml"), o.glossarySeed) + glossaryLine = "glossary_seed: glossary-seed.yaml" + } writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(` book_id: test-book title: Тест @@ -173,8 +185,9 @@ footnotes: minimal pipeline: pipeline.yaml models: models.yaml source_file: %s +%s ceilings: { book_usd: %g, day_usd: 2.0 } -`, sourceName, o.bookUSD)) +`, sourceName, glossaryLine, o.bookUSD)) return filepath.Join(dir, "book.yaml") } diff --git a/backend/internal/store/glossary.go b/backend/internal/store/glossary.go new file mode 100644 index 0000000..0b72f20 --- /dev/null +++ b/backend/internal/store/glossary.go @@ -0,0 +1,322 @@ +package store + +import ( + "database/sql" + "errors" +) + +// glossary.go: durable storage for the memory bank v2 glossary (schema v5). Dumb +// storage, mirroring ruby.go/chunkstatus.go: the pipeline owns aggregation, +// classification and matching; this layer only replaces and reads back. A book's +// glossary is a deterministic function of its inputs (seed file + classified ruby), +// so ReplaceGlossary rewrites the WHOLE book set in one transaction — re-seeding the +// identical inputs rewrites identical rows, a seed edit converges every column, and a +// removed term disappears instead of lingering (the same full-replace idempotency the +// ruby step adopted after external-review #6). + +// GlossaryEntry is one term row plus its aliases. Strings are stored raw; the +// pipeline normalizes for matching (§3.6). Decl is a JSON blob the post-check reads. +type GlossaryEntry struct { + ID int64 + BookID string + Src string + Dst string + Type string + Sense string + Gender string + Speech string + Decl string // JSON {"invariant":bool,"forms":[...]} + TranslitPolicy string + FirstPerson string + NicknameTranslation string + SinceCh int + UntilCh int + Status string // auto|draft|approved + AllowShort bool + Source string // seed|ruby|auto + RubyReading string + RubyClass string + Confidence int + Note string + Aliases []GlossaryAlias +} + +// GlossaryAlias is one alias-graph edge (an alternative source surface + its type). +type GlossaryAlias struct { + Alias string + AliasType string +} + +// GlossaryRevision is one append-only editorial-time record (B1). +type GlossaryRevision struct { + Src string + Sense string + OldDst string + NewDst string + EditorialTS string + Reason string +} + +// RetrievalState is one per-chunk observability record (registry gate #4). +type RetrievalState struct { + BookID string + Chapter int + ChunkIdx int + SnapshotID string + NExactHits int + NSticky int + NAmbiguousFlagged int + NSpoilerBlocked int + NEvicted int + EmbeddingTierUsed int + NPostcheckMiss int + PostcheckDetail string // JSON + InjectedIDs string // JSON +} + +// ReplaceGlossary replaces a book's ENTIRE glossary (rows + aliases) with the given +// entries in ONE transaction, and appends a B1 revision record for every term whose +// approved rendering (dst) changed since the previous set. Scoped to book_id, so other +// books are untouched. Each entry's BookID must equal bookID. term_id linkage is +// established within the tx: a glossary row is inserted, its rowid taken, then its +// aliases inserted against that id (ids are fresh each replace — never hashed). +func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error { + ctx, cancel := opContext() + defer cancel() + tx, err := s.w.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + // Snapshot the prior (src,sense,window)→dst to detect editorial-time changes (B1). + prior := map[[4]any]string{} + rows, err := tx.QueryContext(ctx, `SELECT src, sense, since_ch, until_ch, dst FROM glossary WHERE book_id = ?`, bookID) + if err != nil { + return err + } + for rows.Next() { + var src, sense, dst string + var since, until int + if err := rows.Scan(&src, &sense, &since, &until, &dst); err != nil { + rows.Close() + return err + } + prior[[4]any{src, sense, since, until}] = dst + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + + if _, err := tx.ExecContext(ctx, `DELETE FROM glossary_aliases WHERE book_id = ?`, bookID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM glossary WHERE book_id = ?`, bookID); err != nil { + return err + } + + for _, e := range entries { + res, err := tx.ExecContext(ctx, ` + INSERT INTO glossary ( + book_id, src, dst, type, sense, gender, speech, decl, + translit_policy, first_person, nickname_translation, + since_ch, until_ch, status, allow_short, source, + ruby_reading, ruby_class, confidence, note + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + e.BookID, e.Src, e.Dst, e.Type, e.Sense, e.Gender, e.Speech, e.Decl, + e.TranslitPolicy, e.FirstPerson, e.NicknameTranslation, + e.SinceCh, e.UntilCh, e.Status, boolToInt(e.AllowShort), e.Source, + e.RubyReading, e.RubyClass, e.Confidence, e.Note) + if err != nil { + return err + } + termID, err := res.LastInsertId() + if err != nil { + return err + } + for _, a := range e.Aliases { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO glossary_aliases (book_id, term_id, alias, alias_type) + VALUES (?, ?, ?, ?)`, e.BookID, termID, a.Alias, a.AliasType); err != nil { + return err + } + } + // B1: a term that existed with a non-empty dst and now renders differently + // (also non-empty) is an editorial change → append the revision journal row. + if old, ok := prior[[4]any{e.Src, e.Sense, e.SinceCh, e.UntilCh}]; ok && old != "" && e.Dst != "" && old != e.Dst { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO glossary_revisions (book_id, src, sense, old_dst, new_dst, reason) + VALUES (?, ?, ?, ?, ?, ?)`, e.BookID, e.Src, e.Sense, old, e.Dst, "seed-replace"); err != nil { + return err + } + } + } + return tx.Commit() +} + +// GlossaryForBook returns every glossary entry for a book with its aliases, in a +// DETERMINISTIC order (src, sense, since_ch, until_ch, status, dst) with aliases sorted +// — the stable materialization the hot-path matcher and the F1 memoryVersion() consume. +func (s *Store) GlossaryForBook(bookID string) ([]GlossaryEntry, error) { + ctx, cancel := opContext() + defer cancel() + rows, err := s.r.QueryContext(ctx, ` + SELECT id, src, dst, type, sense, gender, speech, decl, + translit_policy, first_person, nickname_translation, + since_ch, until_ch, status, allow_short, source, + ruby_reading, ruby_class, confidence, note + FROM glossary WHERE book_id = ? + ORDER BY src, sense, since_ch, until_ch, status, dst`, bookID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []GlossaryEntry + byID := map[int64]int{} // glossary.id → index in out + for rows.Next() { + e := GlossaryEntry{BookID: bookID} + var allowShort int + if err := rows.Scan(&e.ID, &e.Src, &e.Dst, &e.Type, &e.Sense, &e.Gender, &e.Speech, &e.Decl, + &e.TranslitPolicy, &e.FirstPerson, &e.NicknameTranslation, + &e.SinceCh, &e.UntilCh, &e.Status, &allowShort, &e.Source, + &e.RubyReading, &e.RubyClass, &e.Confidence, &e.Note); err != nil { + return nil, err + } + e.AllowShort = allowShort != 0 + byID[e.ID] = len(out) + out = append(out, e) + } + if err := rows.Err(); err != nil { + return nil, err + } + // Aliases in a second pass, ordered so each entry's alias list is deterministic. + arows, err := s.r.QueryContext(ctx, ` + SELECT term_id, alias, alias_type FROM glossary_aliases + WHERE book_id = ? ORDER BY term_id, alias`, bookID) + if err != nil { + return nil, err + } + defer arows.Close() + for arows.Next() { + var termID int64 + var a GlossaryAlias + if err := arows.Scan(&termID, &a.Alias, &a.AliasType); err != nil { + return nil, err + } + if idx, ok := byID[termID]; ok { + out[idx].Aliases = append(out[idx].Aliases, a) + } + } + return out, arows.Err() +} + +// GlossaryRevisionsForBook returns the editorial-time journal (B1) for a book, +// newest first, for the report / a human audit. +func (s *Store) GlossaryRevisionsForBook(bookID string) ([]GlossaryRevision, error) { + ctx, cancel := opContext() + defer cancel() + rows, err := s.r.QueryContext(ctx, ` + SELECT src, sense, old_dst, new_dst, editorial_ts, reason + FROM glossary_revisions WHERE book_id = ? + ORDER BY id DESC`, bookID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []GlossaryRevision + for rows.Next() { + var r GlossaryRevision + if err := rows.Scan(&r.Src, &r.Sense, &r.OldDst, &r.NewDst, &r.EditorialTS, &r.Reason); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// UpsertRetrievalState writes (or overwrites) the per-chunk retrieval-state record. +// Overwrite is the resolve semantics: recomputing the deterministic matcher over the +// same chunk reproduces the same row (self-heals on resume). +func (s *Store) UpsertRetrievalState(rs RetrievalState) error { + ctx, cancel := opContext() + defer cancel() + _, err := s.w.ExecContext(ctx, ` + INSERT INTO retrieval_state ( + book_id, chapter, chunk_idx, snapshot_id, + n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted, + embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids, updated_at + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now')) + ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET + snapshot_id = excluded.snapshot_id, + n_exact_hits = excluded.n_exact_hits, + n_sticky = excluded.n_sticky, + n_ambiguous_flagged = excluded.n_ambiguous_flagged, + n_spoiler_blocked = excluded.n_spoiler_blocked, + n_evicted = excluded.n_evicted, + embedding_tier_used = excluded.embedding_tier_used, + n_postcheck_miss = excluded.n_postcheck_miss, + postcheck_detail = excluded.postcheck_detail, + injected_ids = excluded.injected_ids, + updated_at = excluded.updated_at`, + rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID, + rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted, + rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs) + return err +} + +// GetRetrievalState returns the record for a chunk, or nil if none. +func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*RetrievalState, error) { + ctx, cancel := opContext() + defer cancel() + rs := RetrievalState{BookID: bookID, Chapter: chapter, ChunkIdx: chunkIdx} + err := s.r.QueryRowContext(ctx, ` + SELECT snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, + n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids + FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`, + bookID, chapter, chunkIdx).Scan( + &rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, + &rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &rs, nil +} + +// RetrievalStatesForBook returns every retrieval-state record for a book, ordered for +// a stable report (the memory section of tmctl report). +func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error) { + ctx, cancel := opContext() + defer cancel() + rows, err := s.r.QueryContext(ctx, ` + SELECT chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, + n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids + FROM retrieval_state WHERE book_id = ? + ORDER BY chapter, chunk_idx`, bookID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []RetrievalState + for rows.Next() { + rs := RetrievalState{BookID: bookID} + if err := rows.Scan(&rs.Chapter, &rs.ChunkIdx, &rs.SnapshotID, &rs.NExactHits, &rs.NSticky, + &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed, + &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs); err != nil { + return nil, err + } + out = append(out, rs) + } + return out, rows.Err() +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} diff --git a/backend/internal/store/glossary_test.go b/backend/internal/store/glossary_test.go new file mode 100644 index 0000000..fb15d14 --- /dev/null +++ b/backend/internal/store/glossary_test.go @@ -0,0 +1,153 @@ +package store + +import "testing" + +func gEntry(src, dst, status string, aliases ...string) GlossaryEntry { + e := GlossaryEntry{BookID: "book", Src: src, Dst: dst, Status: status, Source: "seed"} + for _, a := range aliases { + e.Aliases = append(e.Aliases, GlossaryAlias{Alias: a, AliasType: "прозвище"}) + } + return e +} + +func TestReplaceGlossaryIdempotentAndScoped(t *testing.T) { + s, _ := openTemp(t) + entries := []GlossaryEntry{ + gEntry("阿Q", "А-кью", "approved"), + gEntry("四叔", "Четвёртый дядюшка", "approved", "鲁四老爷"), + } + if err := s.ReplaceGlossary("book", entries); err != nil { + t.Fatal(err) + } + // Another book is untouched. + if err := s.ReplaceGlossary("other", []GlossaryEntry{{BookID: "other", Src: "x", Dst: "икс", Status: "approved"}}); err != nil { + t.Fatal(err) + } + + got1, err := s.GlossaryForBook("book") + if err != nil { + t.Fatal(err) + } + if len(got1) != 2 { + t.Fatalf("want 2 entries, got %d", len(got1)) + } + // Re-replacing the identical set yields the identical content (idempotent). + if err := s.ReplaceGlossary("book", entries); err != nil { + t.Fatal(err) + } + got2, err := s.GlossaryForBook("book") + if err != nil { + t.Fatal(err) + } + if len(got2) != 2 { + t.Fatalf("re-replace changed count: %d", len(got2)) + } + // The alias survived the replace and attached to the right term. + var sishu *GlossaryEntry + for i := range got2 { + if got2[i].Src == "四叔" { + sishu = &got2[i] + } + } + if sishu == nil || len(sishu.Aliases) != 1 || sishu.Aliases[0].Alias != "鲁四老爷" { + t.Fatalf("alias not preserved: %+v", sishu) + } + // other book still has its one row. + other, _ := s.GlossaryForBook("other") + if len(other) != 1 { + t.Fatalf("scope leak: other book has %d rows", len(other)) + } +} + +func TestReplaceGlossaryRemovesDroppedRows(t *testing.T) { + s, _ := openTemp(t) + if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("阿Q", "А-кью", "approved"), gEntry("四叔", "дядюшка", "approved")}); err != nil { + t.Fatal(err) + } + // Drop one term: full replace must make it disappear (no phantom). + if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("阿Q", "А-кью", "approved")}); err != nil { + t.Fatal(err) + } + got, _ := s.GlossaryForBook("book") + if len(got) != 1 || got[0].Src != "阿Q" { + t.Fatalf("dropped term still present: %+v", got) + } +} + +func TestGlossaryRevisionLogged(t *testing.T) { + s, _ := openTemp(t) + if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("王胡", "Бородатый Ван", "approved")}); err != nil { + t.Fatal(err) + } + // Change the approved rendering: a B1 editorial revision must be journaled. + if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("王胡", "Ван Ху", "approved")}); err != nil { + t.Fatal(err) + } + revs, err := s.GlossaryRevisionsForBook("book") + if err != nil { + t.Fatal(err) + } + if len(revs) != 1 || revs[0].OldDst != "Бородатый Ван" || revs[0].NewDst != "Ван Ху" { + t.Fatalf("revision not logged correctly: %+v", revs) + } + // Re-replacing identical does NOT add a second revision. + if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("王胡", "Ван Ху", "approved")}); err != nil { + t.Fatal(err) + } + revs, _ = s.GlossaryRevisionsForBook("book") + if len(revs) != 1 { + t.Fatalf("idempotent replace added a spurious revision: %d", len(revs)) + } +} + +func TestUniqueSrcSenseWindow(t *testing.T) { + s, _ := openTemp(t) + // Same src+sense+window twice → UNIQUE violation (B2). + dup := []GlossaryEntry{gEntry("道", "путь", "approved"), gEntry("道", "дао", "approved")} + if err := s.ReplaceGlossary("book", dup); err == nil { + t.Fatal("expected UNIQUE(src,sense,window) violation for duplicate src+sense+window") + } + // Same src, different SENSE → allowed (polysemy, A3). + ok := []GlossaryEntry{ + {BookID: "book", Src: "道", Dst: "путь", Sense: "way", Status: "approved"}, + {BookID: "book", Src: "道", Dst: "дао", Sense: "dao", Status: "approved"}, + } + if err := s.ReplaceGlossary("book", ok); err != nil { + t.Fatalf("distinct senses should be allowed: %v", err) + } + got, _ := s.GlossaryForBook("book") + if len(got) != 2 { + t.Fatalf("want 2 polysemy rows, got %d", len(got)) + } +} + +func TestRetrievalStateUpsert(t *testing.T) { + s, _ := openTemp(t) + rs := RetrievalState{BookID: "book", Chapter: 1, ChunkIdx: 0, SnapshotID: "snap", + NExactHits: 3, NAmbiguousFlagged: 1, NSpoilerBlocked: 1, NPostcheckMiss: 1, + PostcheckDetail: `[{"src":"阿Q","dst":"А-кью"}]`, InjectedIDs: `["ah_q"]`} + if err := s.UpsertRetrievalState(rs); err != nil { + t.Fatal(err) + } + got, err := s.GetRetrievalState("book", 1, 0) + if err != nil || got == nil { + t.Fatalf("get: %v %v", got, err) + } + if got.NExactHits != 3 || got.NSpoilerBlocked != 1 || got.NPostcheckMiss != 1 { + t.Fatalf("roundtrip mismatch: %+v", got) + } + // Overwrite (resolve semantics on resume): new counts win. + rs.NExactHits = 5 + rs.NPostcheckMiss = 0 + if err := s.UpsertRetrievalState(rs); err != nil { + t.Fatal(err) + } + got, _ = s.GetRetrievalState("book", 1, 0) + if got.NExactHits != 5 || got.NPostcheckMiss != 0 { + t.Fatalf("overwrite failed: %+v", got) + } + all, _ := s.RetrievalStatesForBook("book") + if len(all) != 1 { + t.Fatalf("want 1 retrieval-state row, got %d", len(all)) + } +} diff --git a/backend/internal/store/migrate.go b/backend/internal/store/migrate.go index 156dd83..ae1d215 100644 --- a/backend/internal/store/migrate.go +++ b/backend/internal/store/migrate.go @@ -161,6 +161,98 @@ var migrations = []string{ ); CREATE INDEX IF NOT EXISTS ruby_readings_book_idx ON ruby_readings (book_id); `, + // v5: the memory bank v2 glossary (шаг 4, registry architecture/06 §Гейты + D7). + // The DETERMINISTIC substrate the hot-path matcher, the injection surface and the + // post-check are built on. Rows are the frozen-at-job-start injectable pool whose + // approved subset the F1 memoryVersion() hashes into the snapshot. Like ruby_readings + // this is "dumb storage" (aggregation/classification live in the pipeline); the + // pipeline REPLACES a book's whole glossary from its deterministic inputs (seed file + // + classified ruby), so re-seeding is idempotent and a seed edit converges every + // column. No cross-table FK (mirrors the ruby/chunkstatus dumb-storage style): both + // child tables are scoped by book_id and replaced in the same transaction, term_id is + // a logical link (glossary.id is a fresh autoincrement each replace and is NEVER hashed + // into memoryVersion — only content columns are, ORDER BY-stable). + ` + CREATE TABLE IF NOT EXISTS glossary ( + id INTEGER PRIMARY KEY, + book_id TEXT NOT NULL, + src TEXT NOT NULL, -- source key (raw; normalized in Go for matching, §3.6) + dst TEXT NOT NULL DEFAULT '', -- approved Russian translation (LEMMA/base form); '' for a ruby candidate with no dst yet + type TEXT NOT NULL DEFAULT '', -- name|term|title|… alias-graph node type (B3) + sense TEXT NOT NULL DEFAULT '', -- A3 polysemy disambiguator (part of the uniqueness key) + gender TEXT NOT NULL DEFAULT '', -- male|female|hidden|'' (C2; hidden = gender-avoidant rendering until reveal) + speech TEXT NOT NULL DEFAULT '', -- ты/вы register note (series-bible-lite; the transition JOURNAL is deferred) + decl TEXT NOT NULL DEFAULT '', -- JSON {"invariant":bool,"forms":[...]}: the dst declension forms the post-check accepts (E1/E2 safety, decl = grammaticality mechanism, not a market feature) + translit_policy TEXT NOT NULL DEFAULT '', -- B5 western-name-via-katakana (field only in v1; validator = Phase 2) + first_person TEXT NOT NULL DEFAULT '', -- D7 first_person field (field only) + nickname_translation TEXT NOT NULL DEFAULT '', -- D7 (keep in v1: a column + a "translate once" lock) + since_ch INTEGER NOT NULL DEFAULT 0, -- spoiler window start; 0 = valid from the beginning (C1 hard reject-gate below since_ch) + until_ch INTEGER NOT NULL DEFAULT 0, -- spoiler window end; 0 = no end (valid forever) + status TEXT NOT NULL DEFAULT 'auto',-- auto|draft|approved (term status machine; only approved is CONFIRMED-injected) + allow_short INTEGER NOT NULL DEFAULT 0, -- rare escape hatch: permit a below-min_key_len key (A3 single-key ban override) + source TEXT NOT NULL DEFAULT '', -- provenance: seed|ruby|auto (who created the row) + ruby_reading TEXT NOT NULL DEFAULT '', -- ruby-seeded rows: the furigana reading (bridge to Polivanov dst validation, B6 Phase 2 — NOT a dst) + ruby_class TEXT NOT NULL DEFAULT '', -- name|gloss|ambiguous (ruby classifier §8: phonetic-name → name-lock candidate, unrelated reading → gloss/footnote, else flag) + confidence INTEGER NOT NULL DEFAULT 0, -- ruby occurrences count = confidence prior (NOT a direct promotion to approved) + note TEXT NOT NULL DEFAULT '', + -- B2: at most one row per (source key, sense, spoiler window). since_ch/until_ch are + -- NOT NULL with default 0, so the SQLite "NULLs are distinct in UNIQUE" gotcha cannot + -- silently admit a duplicate. A term whose approved rendering changes across a spoiler + -- boundary is two rows with different windows (legitimately distinct), not a collision. + UNIQUE (book_id, src, sense, since_ch, until_ch) + ); + CREATE INDEX IF NOT EXISTS glossary_book_idx ON glossary (book_id, status); + + CREATE TABLE IF NOT EXISTS glossary_aliases ( + id INTEGER PRIMARY KEY, + book_id TEXT NOT NULL, + term_id INTEGER NOT NULL, -- logical link to glossary.id within the SAME replace tx (no hard FK — dumb storage, scoped+replaced by book_id) + alias TEXT NOT NULL, -- an alternative source surface (raw; normalized in Go for matching) + alias_type TEXT NOT NULL DEFAULT '', -- имя|цзы|хао|прозвище|титул (B3 alias-graph node type) + UNIQUE (book_id, term_id, alias) + ); + CREATE INDEX IF NOT EXISTS glossary_aliases_term_idx ON glossary_aliases (term_id); + CREATE INDEX IF NOT EXISTS glossary_aliases_book_idx ON glossary_aliases (book_id); + + -- B1 editorial-time axis, ORTHOGONAL to the spoiler window (since_ch/until_ch is the + -- STORY timeline; this is the EDITORIAL timeline). Append-only journal of approved-dst + -- changes so a stale first-translation-wins rendering is DETECTABLE (DelTA PNR generates + -- stale/drift; this is the substrate that catches it). Not injected → not in any hash; + -- datetime('now') here is fine (telemetry, like request_log), never feeds memoryVersion. + CREATE TABLE IF NOT EXISTS glossary_revisions ( + id INTEGER PRIMARY KEY, + book_id TEXT NOT NULL, + src TEXT NOT NULL, + sense TEXT NOT NULL DEFAULT '', + old_dst TEXT NOT NULL DEFAULT '', + new_dst TEXT NOT NULL, + editorial_ts TEXT NOT NULL DEFAULT (datetime('now')), + reason TEXT NOT NULL DEFAULT '' + ); + CREATE INDEX IF NOT EXISTS glossary_revisions_book_idx ON glossary_revisions (book_id, src); + + -- Per-chunk retrieval-state (registry gate #4 / A1/F2/G2/E1): observability of memory + -- degradation FROM DAY ONE — the mechanism that converts silent degradation into loud. + -- Recomputed deterministically each run (the matcher is $0), so it self-heals on resume; + -- NOT wire-load-bearing (it records what the injection did, it does not change the request). + CREATE TABLE IF NOT EXISTS retrieval_state ( + book_id TEXT NOT NULL, + chapter INTEGER NOT NULL, + chunk_idx INTEGER NOT NULL, + snapshot_id TEXT NOT NULL, + n_exact_hits INTEGER NOT NULL DEFAULT 0, -- entries matched by an exact key/alias in this chunk + n_sticky INTEGER NOT NULL DEFAULT 0, -- entries carried by scene-inertia (A5), not re-matched here + n_ambiguous_flagged INTEGER NOT NULL DEFAULT 0, -- injected AMBIGUOUS (auto/draft) → "unverified" + forced post-check (A2) + n_spoiler_blocked INTEGER NOT NULL DEFAULT 0, -- matched but hard-rejected by the since_ch/until_ch window (C1) + n_evicted INTEGER NOT NULL DEFAULT 0, -- dropped by the token budget (F2 — logged, never silent) + embedding_tier_used INTEGER NOT NULL DEFAULT 0, -- always 0 in v1 (no embeddings on the hot path, Р3) + n_postcheck_miss INTEGER NOT NULL DEFAULT 0, -- E1 flags: a confirmed src matched but no accepted dst form is in the output + postcheck_detail TEXT NOT NULL DEFAULT '', -- JSON [{src,dst}] of the misses, for the report / human + injected_ids TEXT NOT NULL DEFAULT '', -- JSON of exact-matched entity keys (audit + sticky reconstruction) + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (book_id, chapter, chunk_idx) + ); + `, } // migrate runs all pending migrations on the write pool, one transaction per diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index a27df24..b6e9c89 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -430,6 +430,30 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор Отсеянная находка (oversize-хвост <500) осталась документированной границей §7b. Дальше — банк памяти v2 (шаг 4): реестр рисков `architecture/06` + ruby-контракт готовы (F1-materialization + горячий путь + post-check). Онбординг-промт след. сессии — `docs/BACKEND_SESSION_PROMPT_MEMORY.md`. +### 2026-07-05 — Шаг 4: банк памяти v2 (глоссарий + горячий путь + инъекция + F1 + post-check) + +**Главный блокер качества (консистентность имён/терминов) реализован — веха A–E на seeded-глоссарии** (граница вехи подтверждена оркестратором §8: механизм на ручном+ruby-сиде; автопопуляция/адъюдикация/резюме/series-bible — отдельные вехи). Всё зелёное `go build/vet/test ./... -race` (125 тестов в pipeline), `tmctl report` $0. Ратифицированный порядок реестра `architecture/06` §Гейты соблюдён. + +**A4 нормализация (`memnorm.go` + `data/trad2simp.txt`):** NFKC + trad→simp (вендорённая таблица, **версия = хеш содержимого таблицы** → дополнение = громкий `--resnapshot`, не тихая мини-карта) + катакана→хирагана + lower для источника; NFC + lower + ё→е + **схлопывание whitespace + унификация тире** для ru-цели. Симметрично ключ↔текст. Полный OpenCC оффлайн недоступен (нет данных на стенде) → курированный высокочастотный набор, все referenced/live-bug кейсы (爺→爷 и др.) покрыты и заассерчены; **завершение полного OpenCC — единственный отслеживаемый data-drop перед zh-приёмкой** (приёмочная книга D9 — ja→ru, где trad→simp не несущий). + +**A схема (миграция v5):** `glossary` (D7 + `sense`/ось редакционного времени `glossary_revisions`/`UNIQUE(src,sense,window)`/`min_key_len`-запрет одиночных ключей/`allow_short`) + `glossary_aliases` (alias-граф) + `glossary_revisions` (B1 append-only) + `retrieval_state` (гейт #4). Idempotent full-replace (как ruby). `store/glossary.go`. + +**B горячий путь (`memory.go`):** frozen `MemoryBank` из store-строк; **Aho-Corasick** по нормализованным ключам; запрет одиночных ключей (A3); longest-match containment; спойлер-hard-reject (C1, `since_ch/until_ch`, 0=безграничен); sticky scene-inertia (A5); токен-бюджет с логируемым вытеснением (F2); трёхсторонний disposition (A2 confirmed/ambiguous/reject). Детерминизм: 0 map-order в выводе, T1–T10 портированы (**не accept-регэкспы**), ловушки `retrieval_bench` 0/N, 200× determinism. **НЕ FTS5/вектор/эмбеддинги** (Р3). + +**C инъекция (`render.go`):** `MessagesWithInjection` — код-собранный `system`-месседж глоссария МЕЖДУ стабильным префиксом (кэш-граница) и user-чанком (вариант «а» оркестратора, шаблон не трогаем); НЕ в `ch.Text` (coverage/`{{text}}` чисты); свёрнут в `request_hash`/`content_hash`. + +**D закрыт F1 (`runner.go`):** `memoryVersion()` = content-hash замороженных **approved**-строк + версии норм/матч-алгоритмов (D8, не version-counter). Смена approved → громкий `--resnapshot` (e2e доказал `1e7b…`→`552c…`). id не хешируется; auto/draft вне хеша (их инъекция ловится `content_hash`). + +**E post-check (`mempostcheck.go`) + E1-замер:** decl-осознанный (**наивный регэксп = 18–36% ложных, research/14**), whole-word матч по записанным `decl`-формам, sticky исключён, слепой post-replace запрещён (E2). **Флаггер по умолчанию** (миссы → `retrieval_state`, видны в `tmctl report`), **жёсткий гейт opt-in** (`gates.glossary.postcheck_gate`, как coverage — флип после валидации владельцем). **E1 измерен как Go-тест на реальном склонённом русском (канон Рогова): full-decl 0% ложных, base-only 67%** — эмпирическое подтверждение, что stored-decl надёжен ТОЛЬКО при полноте (ответ на развилку stored-decl-в-Go vs pymorphy-сайдкар Ф2: держим флаггером, сайдкар — валидированный фолбэк если реальные книги зашумят). + +**Ruby→глоссарий (`memseed.go`):** классификатор `name_candidate`/`gloss_candidate`/`ambiguous` — **никогда не авто-лочит** (блайнд-лок = mistranslation; 強敵→とも неотличим от имени оффлайн → кандидат человеку). Ruby → auto-кандидаты (src=base, reading=мост к Поливанову B6, без dst), дедуп по base, скип manual-src. + +**Интеграция + config:** `TranslateBook`: ingest→persistRuby→seedGlossary(replace+materialize)→snapshot→loop с sticky-окном (ресет на границе главы, пересобирается детерминированно на resume). `book.yaml: glossary_seed`. `tmctl report` — секция ПАМЯТЬ (агрегаты + пост-check миссы). Пост-check валидирует **финальный (редакторский) вывод** — то, что экспортируется, не только черновик. + +**Агентское адверсариальное селфревью (§6, многоагентный Workflow):** 7 дименсий → 10 находок → независимая refute-by-default верификация с прогоном реального кода → **8 CONFIRMED (все воспроизведены), 2 REFUTED** (документированные размены: min-key-2 гомографы; редакторский drift — закрыт пост-check'ом финала). Все 8 исправлены + регресс-тесты **мутационно-проверены** (реверт → падает именно его тест): #1 empty-dst не матчится; #2/#8 whitespace-wrap/тире в post-check; #3 whole-word матч (дропнутый «Ван» не маскируется «караван»); #4 gate-mode `memoryVersion` сворачивает draft-decl (иначе resume молча флипал чанк); #5 дубль-alias в сиде дедупится (не крашит книгу); #6 спойлер-фильтр перед longest-match (спойлер-блокнутый длинный ключ не глотает валидное короткое имя); #7 floor-free токен-бюджет. + +**Заложено/отложено (реестр §5, поля/место есть, реализация — Ф2/веха+):** Палладий/Поливанов B6 (reading как мост заложен), гейт фактичности резюме D1, эмбеддинг-слой A7/D2, морфо-гейт глагольного рода C3 (pymorphy-сайдкар), бэкап файла F4, автопопуляция/адъюдикация G1/F5, series-bible ты/вы-журнал, инъекция глоссария в редактор (dst-констрейнты) — fast-follow на той же поверхности. Ставка (селективная инъекция vs длинный контекст) гейтится in-house eval'ом пилота Ф2.5 (параллельно, не блокер). Правки только в коде `backend/` + этот журнал; `architecture/06`/`research/*` не трогались. + ## Полигон (секция параллельной сессии — записи добавлять сюда)