Land memory bank v2 (glossary hot path, injection surface, F1 materialization, decl-aware post-check) with 8 self-review fixes
This commit is contained in:
Claude (backend session)
parent
baf6ae31bd
commit
aa00339abb
23 changed files with 3886 additions and 43 deletions
|
|
@ -180,6 +180,40 @@ func report(cfgPath string) error {
|
||||||
f.Chapter, f.ChunkIdx, f.Stage, f.Disposition, f.FlagReason, f.Attempts, f.CostUSD, f.Detail)
|
f.Chapter, f.ChunkIdx, f.Stage, f.Disposition, f.FlagReason, f.Attempts, f.CostUSD, f.Detail)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Memory section (шаг 4): the per-chunk retrieval-state, surfaced so silent glossary
|
||||||
|
// degradation is LOUD. The aggregate line always prints when a glossary is in use;
|
||||||
|
// each chunk with a post-check miss is listed (the flagger-mode signal — the model
|
||||||
|
// ignored an approved term, or we injected the wrong dst) with the offending src→dst.
|
||||||
|
states, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(states) > 0 {
|
||||||
|
var injected, sticky, ambiguous, spoiler, evicted, misses int
|
||||||
|
for _, rs := range states {
|
||||||
|
injected += rs.NExactHits
|
||||||
|
sticky += rs.NSticky
|
||||||
|
ambiguous += rs.NAmbiguousFlagged
|
||||||
|
spoiler += rs.NSpoilerBlocked
|
||||||
|
evicted += rs.NEvicted
|
||||||
|
misses += rs.NPostcheckMiss
|
||||||
|
}
|
||||||
|
fmt.Printf("\n=== ПАМЯТЬ (retrieval-state) ===\n")
|
||||||
|
fmt.Printf("инъекций(точных)=%d sticky=%d ambiguous=%d спойлер-блок=%d вытеснено=%d post-check-промахов=%d\n",
|
||||||
|
injected, sticky, ambiguous, spoiler, evicted, misses)
|
||||||
|
printed := false
|
||||||
|
for _, rs := range states {
|
||||||
|
if rs.NPostcheckMiss == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !printed {
|
||||||
|
fmt.Printf("%-4s %-6s %6s %s\n", "ch", "chunk", "misses", "detail (src→dst не найден в выводе)")
|
||||||
|
printed = true
|
||||||
|
}
|
||||||
|
fmt.Printf("%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NPostcheckMiss, rs.PostcheckDetail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
committed, reserved, err := r.Store.SpentUSD(r.Book.BookID)
|
committed, reserved, err := r.Store.SpentUSD(r.Book.BookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,13 @@ type Book struct {
|
||||||
ModelsFile string `yaml:"models"`
|
ModelsFile string `yaml:"models"`
|
||||||
SourceFile string `yaml:"source_file"` // Фаза 0: один файл = один чанк
|
SourceFile string `yaml:"source_file"` // Фаза 0: один файл = один чанк
|
||||||
ProjectDB string `yaml:"project_db"` // default: <book_id>.db рядом с book.yaml
|
ProjectDB string `yaml:"project_db"` // default: <book_id>.db рядом с book.yaml
|
||||||
Ceilings BookCap `yaml:"ceilings"`
|
// GlossarySeed is the optional path to the manual glossary seed YAML (memory v2,
|
||||||
|
// шаг 4). Its curated approved/draft terms + the classified ruby readings are the
|
||||||
|
// deterministic inputs the glossary is REPLACED from each run; editing it changes the
|
||||||
|
// materialized approved set → a loud --resnapshot (F1). Empty = ruby-seed only (or an
|
||||||
|
// empty glossary → the injection is inert, a safe no-op).
|
||||||
|
GlossarySeed string `yaml:"glossary_seed"`
|
||||||
|
Ceilings BookCap `yaml:"ceilings"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// BookCap are the ledger admission limits (Р7: потолок $ на книгу/день).
|
// BookCap are the ledger admission limits (Р7: потолок $ на книгу/день).
|
||||||
|
|
@ -70,6 +76,7 @@ func LoadBook(path string) (*Book, error) {
|
||||||
b.Pipeline = resolve(b.Pipeline)
|
b.Pipeline = resolve(b.Pipeline)
|
||||||
b.ModelsFile = resolve(b.ModelsFile)
|
b.ModelsFile = resolve(b.ModelsFile)
|
||||||
b.SourceFile = resolve(b.SourceFile)
|
b.SourceFile = resolve(b.SourceFile)
|
||||||
|
b.GlossarySeed = resolve(b.GlossarySeed)
|
||||||
if b.ProjectDB == "" {
|
if b.ProjectDB == "" {
|
||||||
b.ProjectDB = filepath.Join(dir, b.BookID+".db")
|
b.ProjectDB = filepath.Join(dir, b.BookID+".db")
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -98,6 +105,11 @@ func LoadBook(path string) (*Book, error) {
|
||||||
} else if _, err := os.Stat(b.SourceFile); err != nil {
|
} else if _, err := os.Stat(b.SourceFile); err != nil {
|
||||||
bad("source_file %s is not readable: %v", b.SourceFile, err)
|
bad("source_file %s is not readable: %v", b.SourceFile, err)
|
||||||
}
|
}
|
||||||
|
if b.GlossarySeed != "" {
|
||||||
|
if _, err := os.Stat(b.GlossarySeed); err != nil {
|
||||||
|
bad("glossary_seed %s is not readable: %v", b.GlossarySeed, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
if b.Ceilings.BookUSD <= 0 && b.Ceilings.DayUSD <= 0 {
|
if b.Ceilings.BookUSD <= 0 && b.Ceilings.DayUSD <= 0 {
|
||||||
bad("ceilings: at least one of book_usd/day_usd must be set (ledger без потолка запрещён Р7)")
|
bad("ceilings: at least one of book_usd/day_usd must be set (ledger без потолка запрещён Р7)")
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -81,6 +81,19 @@ type Stage struct {
|
||||||
// Gates is the QA-gate config skeleton (пороги полигона; исполнение — Фаза 1).
|
// Gates is the QA-gate config skeleton (пороги полигона; исполнение — Фаза 1).
|
||||||
type Gates struct {
|
type Gates struct {
|
||||||
Coverage CoverageGate `yaml:"coverage"`
|
Coverage CoverageGate `yaml:"coverage"`
|
||||||
|
Glossary GlossaryGate `yaml:"glossary"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GlossaryGate controls the memory-bank post-check (E1). The post-check ALWAYS runs
|
||||||
|
// (observability — it records misses into the retrieval-state), converting silent
|
||||||
|
// glossary drift into a loud, visible signal. PostcheckGate promotes a miss from a
|
||||||
|
// mere record to a DISPOSITION flag (the chunk is flagged, downstream stages skip):
|
||||||
|
// opt-in (default false = flagger), flipped on only AFTER the owner validates the
|
||||||
|
// false-flag rate on real chapters (E1) — exactly the coverage gate's opt-in discipline
|
||||||
|
// (D12 Q4). A naive/under-filled decl false-flags (research/14 §2), so a premature hard
|
||||||
|
// gate would flag-storm and train editors to ignore it.
|
||||||
|
type GlossaryGate struct {
|
||||||
|
PostcheckGate bool `yaml:"postcheck_gate"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CoverageGate v1 (Р7): метрика — символы без пробелов; нижняя граница —
|
// CoverageGate v1 (Р7): метрика — символы без пробелов; нижняя граница —
|
||||||
|
|
|
||||||
563
backend/internal/pipeline/data/trad2simp.txt
Normal file
563
backend/internal/pipeline/data/trad2simp.txt
Normal file
|
|
@ -0,0 +1,563 @@
|
||||||
|
# Traditional → Simplified Chinese character map (A4 normalization, memory bank v2).
|
||||||
|
#
|
||||||
|
# Format: one mapping per line, "<trad> <simp>" (whitespace-separated, single rune
|
||||||
|
# each). '#' comments and blank lines are ignored. Loaded once via go:embed and
|
||||||
|
# applied character-by-character in memnorm.go, SYMMETRICALLY to glossary keys and
|
||||||
|
# to chunk text — so a key written in Simplified still matches a Traditional source
|
||||||
|
# surface (and vice versa), closing the "тихо пусто" orthography hole (registry A4).
|
||||||
|
#
|
||||||
|
# PROVENANCE / COMPLETENESS (read before trusting): this is a curated HIGH-FREQUENCY
|
||||||
|
# set, NOT the full OpenCC TSCharacters table. It was authored offline (no OpenCC/
|
||||||
|
# Unihan data available on the stand) with accuracy prioritized over size — a WRONG
|
||||||
|
# trad→simp mapping silently corrupts matching, so only high-confidence single-char
|
||||||
|
# mappings are included; phrase-level OpenCC (one-to-many, e.g. 後→后 vs 后→后) is
|
||||||
|
# deliberately out of scope for v1. The table's content is versioned by
|
||||||
|
# memoryNormVersion (memnorm.go), folded into the snapshot via memoryVersion() — so
|
||||||
|
# COMPLETING it with the full OpenCC data is a data-file swap that fires a loud
|
||||||
|
# --resnapshot, never a silent behaviour change. The acceptance book is ja→ru, where
|
||||||
|
# trad→simp is not load-bearing (NFKC + kana-fold + ruby carry ja); this table matters
|
||||||
|
# for zh sources and is the one tracked data-completion item before a zh acceptance run.
|
||||||
|
# Known live-bug cases from research/14 (爺→爷 etc.) and every retrieval_bench trap
|
||||||
|
# character are covered and asserted in memnorm_test.go.
|
||||||
|
|
||||||
|
# --- referenced by the eval specs / research/14 live bugs (must stay covered) ---
|
||||||
|
魯 鲁
|
||||||
|
鎮 镇
|
||||||
|
趙 赵
|
||||||
|
蕭 萧
|
||||||
|
煉 炼
|
||||||
|
門 门
|
||||||
|
闆 板
|
||||||
|
莊 庄
|
||||||
|
錢 钱
|
||||||
|
陳 陈
|
||||||
|
萬 万
|
||||||
|
舊 旧
|
||||||
|
臉 脸
|
||||||
|
爺 爷
|
||||||
|
羅 罗
|
||||||
|
莊 庄
|
||||||
|
|
||||||
|
# --- 訁 (speech) radical ---
|
||||||
|
說 说
|
||||||
|
話 话
|
||||||
|
語 语
|
||||||
|
讀 读
|
||||||
|
誰 谁
|
||||||
|
課 课
|
||||||
|
談 谈
|
||||||
|
請 请
|
||||||
|
謝 谢
|
||||||
|
詩 诗
|
||||||
|
詞 词
|
||||||
|
試 试
|
||||||
|
記 记
|
||||||
|
訪 访
|
||||||
|
許 许
|
||||||
|
論 论
|
||||||
|
訴 诉
|
||||||
|
譯 译
|
||||||
|
議 议
|
||||||
|
護 护
|
||||||
|
譽 誉
|
||||||
|
讚 赞
|
||||||
|
變 变
|
||||||
|
讓 让
|
||||||
|
認 认
|
||||||
|
識 识
|
||||||
|
訊 讯
|
||||||
|
訂 订
|
||||||
|
計 计
|
||||||
|
訓 训
|
||||||
|
設 设
|
||||||
|
詳 详
|
||||||
|
誠 诚
|
||||||
|
語 语
|
||||||
|
誤 误
|
||||||
|
說 说
|
||||||
|
誦 诵
|
||||||
|
調 调
|
||||||
|
諒 谅
|
||||||
|
談 谈
|
||||||
|
謎 谜
|
||||||
|
講 讲
|
||||||
|
謙 谦
|
||||||
|
謠 谣
|
||||||
|
證 证
|
||||||
|
評 评
|
||||||
|
|
||||||
|
# --- 貝 (shell/money) radical ---
|
||||||
|
貝 贝
|
||||||
|
財 财
|
||||||
|
貨 货
|
||||||
|
貧 贫
|
||||||
|
貪 贪
|
||||||
|
責 责
|
||||||
|
貫 贯
|
||||||
|
貴 贵
|
||||||
|
買 买
|
||||||
|
費 费
|
||||||
|
貼 贴
|
||||||
|
賀 贺
|
||||||
|
貿 贸
|
||||||
|
資 资
|
||||||
|
賊 贼
|
||||||
|
賓 宾
|
||||||
|
賦 赋
|
||||||
|
賠 赔
|
||||||
|
賣 卖
|
||||||
|
賤 贱
|
||||||
|
賬 账
|
||||||
|
賭 赌
|
||||||
|
賴 赖
|
||||||
|
購 购
|
||||||
|
賽 赛
|
||||||
|
贈 赠
|
||||||
|
贏 赢
|
||||||
|
賺 赚
|
||||||
|
賞 赏
|
||||||
|
賢 贤
|
||||||
|
質 质
|
||||||
|
贖 赎
|
||||||
|
貸 贷
|
||||||
|
賃 赁
|
||||||
|
賄 贿
|
||||||
|
賂 赂
|
||||||
|
|
||||||
|
# --- 車 (vehicle) radical ---
|
||||||
|
車 车
|
||||||
|
輪 轮
|
||||||
|
軍 军
|
||||||
|
較 较
|
||||||
|
輕 轻
|
||||||
|
轉 转
|
||||||
|
輸 输
|
||||||
|
輩 辈
|
||||||
|
輝 辉
|
||||||
|
軟 软
|
||||||
|
軌 轨
|
||||||
|
轟 轰
|
||||||
|
輔 辅
|
||||||
|
輛 辆
|
||||||
|
載 载
|
||||||
|
轎 轿
|
||||||
|
|
||||||
|
# --- 門 (gate) radical ---
|
||||||
|
開 开
|
||||||
|
閉 闭
|
||||||
|
問 问
|
||||||
|
間 间
|
||||||
|
閃 闪
|
||||||
|
閒 闲
|
||||||
|
閑 闲
|
||||||
|
閣 阁
|
||||||
|
閩 闽
|
||||||
|
閱 阅
|
||||||
|
闊 阔
|
||||||
|
關 关
|
||||||
|
闖 闯
|
||||||
|
闕 阙
|
||||||
|
悶 闷
|
||||||
|
聞 闻
|
||||||
|
|
||||||
|
# --- 食 (food) radical ---
|
||||||
|
飛 飞
|
||||||
|
風 风
|
||||||
|
飯 饭
|
||||||
|
飲 饮
|
||||||
|
餓 饿
|
||||||
|
館 馆
|
||||||
|
養 养
|
||||||
|
餘 余
|
||||||
|
餅 饼
|
||||||
|
饑 饥
|
||||||
|
饞 馋
|
||||||
|
飄 飘
|
||||||
|
|
||||||
|
# --- 金 (metal) radical ---
|
||||||
|
鐘 钟
|
||||||
|
鐵 铁
|
||||||
|
銀 银
|
||||||
|
銅 铜
|
||||||
|
鋼 钢
|
||||||
|
錯 错
|
||||||
|
鎖 锁
|
||||||
|
鏡 镜
|
||||||
|
鑰 钥
|
||||||
|
鑽 钻
|
||||||
|
鍋 锅
|
||||||
|
鍵 键
|
||||||
|
錄 录
|
||||||
|
針 针
|
||||||
|
釘 钉
|
||||||
|
鈴 铃
|
||||||
|
鉛 铅
|
||||||
|
銷 销
|
||||||
|
鋒 锋
|
||||||
|
鏈 链
|
||||||
|
鑼 锣
|
||||||
|
鈔 钞
|
||||||
|
鑲 镶
|
||||||
|
|
||||||
|
# --- 頁 (head/page) radical ---
|
||||||
|
頁 页
|
||||||
|
頂 顶
|
||||||
|
項 项
|
||||||
|
順 顺
|
||||||
|
須 须
|
||||||
|
預 预
|
||||||
|
頑 顽
|
||||||
|
頓 顿
|
||||||
|
頗 颇
|
||||||
|
領 领
|
||||||
|
頸 颈
|
||||||
|
頻 频
|
||||||
|
顆 颗
|
||||||
|
題 题
|
||||||
|
額 额
|
||||||
|
顏 颜
|
||||||
|
願 愿
|
||||||
|
類 类
|
||||||
|
顧 顾
|
||||||
|
顯 显
|
||||||
|
顫 颤
|
||||||
|
|
||||||
|
# --- 馬 (horse) radical ---
|
||||||
|
馬 马
|
||||||
|
駕 驾
|
||||||
|
駛 驶
|
||||||
|
駐 驻
|
||||||
|
駝 驼
|
||||||
|
騎 骑
|
||||||
|
騙 骗
|
||||||
|
驕 骄
|
||||||
|
驗 验
|
||||||
|
驚 惊
|
||||||
|
驢 驴
|
||||||
|
駭 骇
|
||||||
|
駱 骆
|
||||||
|
騰 腾
|
||||||
|
|
||||||
|
# --- 鳥 (bird) radical ---
|
||||||
|
鳥 鸟
|
||||||
|
鴨 鸭
|
||||||
|
鵝 鹅
|
||||||
|
鴻 鸿
|
||||||
|
鵬 鹏
|
||||||
|
鶴 鹤
|
||||||
|
鷹 鹰
|
||||||
|
鷗 鸥
|
||||||
|
鴉 鸦
|
||||||
|
鳴 鸣
|
||||||
|
鴿 鸽
|
||||||
|
|
||||||
|
# --- 魚 (fish) radical ---
|
||||||
|
魚 鱼
|
||||||
|
鮮 鲜
|
||||||
|
鯨 鲸
|
||||||
|
鯉 鲤
|
||||||
|
鱗 鳞
|
||||||
|
|
||||||
|
# --- 糸 (silk/thread) radical ---
|
||||||
|
紀 纪
|
||||||
|
約 约
|
||||||
|
紅 红
|
||||||
|
紙 纸
|
||||||
|
級 级
|
||||||
|
納 纳
|
||||||
|
紛 纷
|
||||||
|
純 纯
|
||||||
|
紗 纱
|
||||||
|
綱 纲
|
||||||
|
網 网
|
||||||
|
綿 绵
|
||||||
|
線 线
|
||||||
|
練 练
|
||||||
|
組 组
|
||||||
|
細 细
|
||||||
|
終 终
|
||||||
|
結 结
|
||||||
|
絕 绝
|
||||||
|
給 给
|
||||||
|
絡 络
|
||||||
|
統 统
|
||||||
|
綠 绿
|
||||||
|
維 维
|
||||||
|
綜 综
|
||||||
|
緊 紧
|
||||||
|
緒 绪
|
||||||
|
編 编
|
||||||
|
緩 缓
|
||||||
|
締 缔
|
||||||
|
緣 缘
|
||||||
|
縣 县
|
||||||
|
縫 缝
|
||||||
|
縮 缩
|
||||||
|
績 绩
|
||||||
|
繩 绳
|
||||||
|
繫 系
|
||||||
|
繪 绘
|
||||||
|
繼 继
|
||||||
|
續 续
|
||||||
|
纏 缠
|
||||||
|
紋 纹
|
||||||
|
綁 绑
|
||||||
|
繳 缴
|
||||||
|
經 经
|
||||||
|
絲 丝
|
||||||
|
繡 绣
|
||||||
|
|
||||||
|
# --- water / fire / misc very-high-frequency ---
|
||||||
|
漢 汉
|
||||||
|
灣 湾
|
||||||
|
濟 济
|
||||||
|
潔 洁
|
||||||
|
澤 泽
|
||||||
|
濕 湿
|
||||||
|
灑 洒
|
||||||
|
滅 灭
|
||||||
|
溫 温
|
||||||
|
準 准
|
||||||
|
淚 泪
|
||||||
|
減 减
|
||||||
|
滾 滚
|
||||||
|
滿 满
|
||||||
|
漁 渔
|
||||||
|
漸 渐
|
||||||
|
潑 泼
|
||||||
|
濁 浊
|
||||||
|
瀉 泻
|
||||||
|
決 决
|
||||||
|
沒 没
|
||||||
|
沖 冲
|
||||||
|
況 况
|
||||||
|
淨 净
|
||||||
|
涼 凉
|
||||||
|
凍 冻
|
||||||
|
熱 热
|
||||||
|
煩 烦
|
||||||
|
燒 烧
|
||||||
|
燈 灯
|
||||||
|
燦 灿
|
||||||
|
爛 烂
|
||||||
|
營 营
|
||||||
|
爐 炉
|
||||||
|
煙 烟
|
||||||
|
熾 炽
|
||||||
|
燭 烛
|
||||||
|
氣 气
|
||||||
|
氫 氢
|
||||||
|
|
||||||
|
# --- 犭 (animal) radical ---
|
||||||
|
獨 独
|
||||||
|
獲 获
|
||||||
|
獸 兽
|
||||||
|
獻 献
|
||||||
|
獄 狱
|
||||||
|
猶 犹
|
||||||
|
狹 狭
|
||||||
|
獅 狮
|
||||||
|
獵 猎
|
||||||
|
豬 猪
|
||||||
|
|
||||||
|
# --- 阝 (mound/city) radical ---
|
||||||
|
陰 阴
|
||||||
|
陽 阳
|
||||||
|
階 阶
|
||||||
|
隊 队
|
||||||
|
隨 随
|
||||||
|
險 险
|
||||||
|
隱 隐
|
||||||
|
際 际
|
||||||
|
陸 陆
|
||||||
|
陝 陕
|
||||||
|
陣 阵
|
||||||
|
鄉 乡
|
||||||
|
鄰 邻
|
||||||
|
鄭 郑
|
||||||
|
鄧 邓
|
||||||
|
|
||||||
|
# --- 木 (wood) radical ---
|
||||||
|
條 条
|
||||||
|
極 极
|
||||||
|
樓 楼
|
||||||
|
標 标
|
||||||
|
樣 样
|
||||||
|
樹 树
|
||||||
|
橋 桥
|
||||||
|
機 机
|
||||||
|
檢 检
|
||||||
|
櫃 柜
|
||||||
|
欄 栏
|
||||||
|
權 权
|
||||||
|
檔 档
|
||||||
|
樸 朴
|
||||||
|
橫 横
|
||||||
|
檻 槛
|
||||||
|
櫥 橱
|
||||||
|
楊 杨
|
||||||
|
榮 荣
|
||||||
|
構 构
|
||||||
|
槍 枪
|
||||||
|
樑 梁
|
||||||
|
櫻 樱
|
||||||
|
樂 乐
|
||||||
|
業 业
|
||||||
|
東 东
|
||||||
|
|
||||||
|
# --- structural / very common standalone ---
|
||||||
|
國 国
|
||||||
|
學 学
|
||||||
|
會 会
|
||||||
|
對 对
|
||||||
|
時 时
|
||||||
|
見 见
|
||||||
|
這 这
|
||||||
|
們 们
|
||||||
|
個 个
|
||||||
|
來 来
|
||||||
|
為 为
|
||||||
|
麼 么
|
||||||
|
頭 头
|
||||||
|
過 过
|
||||||
|
還 还
|
||||||
|
進 进
|
||||||
|
種 种
|
||||||
|
應 应
|
||||||
|
關 关
|
||||||
|
點 点
|
||||||
|
動 动
|
||||||
|
於 于
|
||||||
|
實 实
|
||||||
|
發 发
|
||||||
|
現 现
|
||||||
|
長 长
|
||||||
|
兒 儿
|
||||||
|
東 东
|
||||||
|
與 与
|
||||||
|
舉 举
|
||||||
|
興 兴
|
||||||
|
覺 觉
|
||||||
|
觀 观
|
||||||
|
歡 欢
|
||||||
|
勸 劝
|
||||||
|
農 农
|
||||||
|
辦 办
|
||||||
|
務 务
|
||||||
|
勞 劳
|
||||||
|
勢 势
|
||||||
|
勝 胜
|
||||||
|
勵 励
|
||||||
|
單 单
|
||||||
|
嚴 严
|
||||||
|
嘗 尝
|
||||||
|
嘆 叹
|
||||||
|
嚇 吓
|
||||||
|
嚮 向
|
||||||
|
圖 图
|
||||||
|
團 团
|
||||||
|
圓 圆
|
||||||
|
園 园
|
||||||
|
圍 围
|
||||||
|
囑 嘱
|
||||||
|
專 专
|
||||||
|
將 将
|
||||||
|
尋 寻
|
||||||
|
導 导
|
||||||
|
屬 属
|
||||||
|
層 层
|
||||||
|
屢 屡
|
||||||
|
廣 广
|
||||||
|
廠 厂
|
||||||
|
廢 废
|
||||||
|
廟 庙
|
||||||
|
廚 厨
|
||||||
|
廈 厦
|
||||||
|
廳 厅
|
||||||
|
華 华
|
||||||
|
葉 叶
|
||||||
|
蓋 盖
|
||||||
|
蒼 苍
|
||||||
|
薦 荐
|
||||||
|
薩 萨
|
||||||
|
藍 蓝
|
||||||
|
藝 艺
|
||||||
|
藥 药
|
||||||
|
蘇 苏
|
||||||
|
蘭 兰
|
||||||
|
蘋 苹
|
||||||
|
蟲 虫
|
||||||
|
傷 伤
|
||||||
|
傳 传
|
||||||
|
債 债
|
||||||
|
傾 倾
|
||||||
|
僅 仅
|
||||||
|
僕 仆
|
||||||
|
僑 侨
|
||||||
|
價 价
|
||||||
|
儀 仪
|
||||||
|
億 亿
|
||||||
|
儉 俭
|
||||||
|
儲 储
|
||||||
|
償 偿
|
||||||
|
優 优
|
||||||
|
側 侧
|
||||||
|
偉 伟
|
||||||
|
偵 侦
|
||||||
|
師 师
|
||||||
|
帥 帅
|
||||||
|
幣 币
|
||||||
|
帳 帐
|
||||||
|
幫 帮
|
||||||
|
歲 岁
|
||||||
|
歸 归
|
||||||
|
歷 历
|
||||||
|
曆 历
|
||||||
|
歐 欧
|
||||||
|
殺 杀
|
||||||
|
毀 毁
|
||||||
|
愛 爱
|
||||||
|
節 节
|
||||||
|
範 范
|
||||||
|
築 筑
|
||||||
|
簡 简
|
||||||
|
籃 篮
|
||||||
|
籠 笼
|
||||||
|
籤 签
|
||||||
|
篤 笃
|
||||||
|
簾 帘
|
||||||
|
籌 筹
|
||||||
|
醫 医
|
||||||
|
產 产
|
||||||
|
麗 丽
|
||||||
|
麵 面
|
||||||
|
麥 麦
|
||||||
|
黃 黄
|
||||||
|
黨 党
|
||||||
|
龍 龙
|
||||||
|
鳳 凤
|
||||||
|
龜 龟
|
||||||
|
齒 齿
|
||||||
|
齊 齐
|
||||||
|
書 书
|
||||||
|
畫 画
|
||||||
|
邊 边
|
||||||
|
飛 飞
|
||||||
|
決 决
|
||||||
|
擊 击
|
||||||
|
擔 担
|
||||||
|
據 据
|
||||||
|
擁 拥
|
||||||
|
撲 扑
|
||||||
|
擴 扩
|
||||||
|
攔 拦
|
||||||
|
掃 扫
|
||||||
|
撿 捡
|
||||||
|
擋 挡
|
||||||
|
掛 挂
|
||||||
|
換 换
|
||||||
|
損 损
|
||||||
|
搶 抢
|
||||||
|
攜 携
|
||||||
|
擾 扰
|
||||||
|
撐 撑
|
||||||
|
|
@ -72,6 +72,15 @@ const (
|
||||||
FlagExcisionSuspect FlagReason = "excision_suspect"
|
FlagExcisionSuspect FlagReason = "excision_suspect"
|
||||||
FlagHardBlock FlagReason = "hard_block"
|
FlagHardBlock FlagReason = "hard_block"
|
||||||
FlagUpstreamNotOK FlagReason = "upstream_not_ok"
|
FlagUpstreamNotOK FlagReason = "upstream_not_ok"
|
||||||
|
|
||||||
|
// FlagGlossaryMiss is the memory-bank post-check verdict (E1, шаг 4): an approved
|
||||||
|
// term's src fired in the chunk but no accepted dst form appears in the output — the
|
||||||
|
// model ignored the glossary, or we injected the wrong dst and it obeyed. Emitted
|
||||||
|
// ONLY when the opt-in glossary post-check gate is enabled (config); in the default
|
||||||
|
// flagger mode a miss is recorded in the retrieval-state, not a disposition. NOT
|
||||||
|
// retryable (a same-model retry re-produces the same rendering) and NOT auto-
|
||||||
|
// escalatable in v1 (the L3 targeted re-ask is a Phase-2 remedy, research/14 §9).
|
||||||
|
FlagGlossaryMiss FlagReason = "glossary_miss"
|
||||||
)
|
)
|
||||||
|
|
||||||
// classifierVersion versions the INTRINSIC classify() verdict logic — the refusal
|
// classifierVersion versions the INTRINSIC classify() verdict logic — the refusal
|
||||||
|
|
|
||||||
182
backend/internal/pipeline/memnorm.go
Normal file
182
backend/internal/pipeline/memnorm.go
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
_ "embed"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
"golang.org/x/text/unicode/norm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// memnorm.go: the memory-bank NORMALIZATION artifact (registry A4 — the most likely
|
||||||
|
// "тихо пусто" bug: a glossary key is present in the chunk but in a different
|
||||||
|
// orthographic form, so the record silently fails to match). Two pure, deterministic,
|
||||||
|
// VERSIONED normalizers, applied SYMMETRICALLY to both sides of every comparison:
|
||||||
|
//
|
||||||
|
// normalizeSourceKey — zh/ja source surfaces (glossary keys/aliases AND chunk text):
|
||||||
|
// NFKC (fold half/full-width, e.g. ﾃ→テ, Ｑ→Q) → trad→simp (per-char, the
|
||||||
|
// embedded table) → katakana→hiragana fold → Unicode lower. So a key stored in
|
||||||
|
// Simplified still matches a Traditional source surface (魯鎮 ↔ 鲁镇), a full-width
|
||||||
|
// Ｑ matches a half-width Q, and スズキ matches すずき.
|
||||||
|
// normalizeTargetForm — Russian dst forms (stored decl forms AND model output) for
|
||||||
|
// the post-check substring test: NFC → Unicode lower → ё→е fold.
|
||||||
|
//
|
||||||
|
// The whole artifact is versioned by memoryNormVersion, which is FOLDED INTO the
|
||||||
|
// snapshot via memoryVersion() (runner.go): a change to the algorithm OR to the
|
||||||
|
// embedded trad→simp table is a loud --resnapshot, never a silent match-behaviour
|
||||||
|
// change on already-checkpointed chunks. Determinism is load-bearing — the injected
|
||||||
|
// glossary block is a pure function of (frozen glossary, normalized chunk), and it
|
||||||
|
// enters request_hash via the rendered messages.
|
||||||
|
|
||||||
|
//go:embed data/trad2simp.txt
|
||||||
|
var trad2simpRaw string
|
||||||
|
|
||||||
|
// memoryNormAlgoVersion tags the normalization ALGORITHM (the pipeline of steps and
|
||||||
|
// their order). Bump it on an algorithm change. The trad→simp TABLE content is
|
||||||
|
// versioned SEPARATELY by hashing trad2simpRaw into memoryNormVersion below, so
|
||||||
|
// editing/completing the data file also invalidates — drift-proof (D8): you cannot
|
||||||
|
// forget to bump a version when you change the table, because the table's bytes ARE
|
||||||
|
// the version.
|
||||||
|
const memoryNormAlgoVersion = "memnorm-v1-nfkc+trad+kana+lower/nfc+lower+yofold"
|
||||||
|
|
||||||
|
var (
|
||||||
|
// trad2simp is the parsed Traditional→Simplified single-char map (data/trad2simp.txt).
|
||||||
|
trad2simp map[rune]rune
|
||||||
|
// memoryNormVersion is the snapshot-load-bearing version of the entire
|
||||||
|
// normalization artifact: the algorithm tag + a content hash of the embedded
|
||||||
|
// trad→simp table. memoryVersion() (runner.go) folds it into the job snapshot.
|
||||||
|
memoryNormVersion string
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
trad2simp = parseTradTable(trad2simpRaw)
|
||||||
|
h := sha256.Sum256([]byte(memoryNormAlgoVersion + "\x00" + trad2simpRaw))
|
||||||
|
memoryNormVersion = memoryNormAlgoVersion + "-" + hex.EncodeToString(h[:])[:12]
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseTradTable parses the embedded "<trad> <simp>" table. Comment ('#') and blank
|
||||||
|
// lines are ignored; a non-empty content line that is NOT exactly two
|
||||||
|
// single-rune whitespace-separated fields is a corrupt table → panic at init (a
|
||||||
|
// loud build/test failure, caught the moment any test in the package runs, rather
|
||||||
|
// than a silently truncated map that would re-open the A4 hole). The table is
|
||||||
|
// authored in-repo and asserted by memnorm_test.go, so a panic here means the data
|
||||||
|
// file was corrupted, not bad user input.
|
||||||
|
func parseTradTable(raw string) map[rune]rune {
|
||||||
|
m := make(map[rune]rune, 1024)
|
||||||
|
for i, line := range strings.Split(raw, "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) != 2 {
|
||||||
|
panic(fmt.Sprintf("pipeline: trad2simp table line %d: want 2 fields, got %d (%q)", i+1, len(fields), line))
|
||||||
|
}
|
||||||
|
tr := []rune(fields[0])
|
||||||
|
si := []rune(fields[1])
|
||||||
|
if len(tr) != 1 || len(si) != 1 {
|
||||||
|
panic(fmt.Sprintf("pipeline: trad2simp table line %d: each side must be a single rune (%q → %q)", i+1, fields[0], fields[1]))
|
||||||
|
}
|
||||||
|
// A later duplicate wins deterministically (the file may list a char under
|
||||||
|
// two families); assert the mapping is consistent to catch a typo where the
|
||||||
|
// same trad char is mapped to two different simp chars.
|
||||||
|
if prev, ok := m[tr[0]]; ok && prev != si[0] {
|
||||||
|
panic(fmt.Sprintf("pipeline: trad2simp table line %d: %q maps to both %q and %q", i+1, fields[0], string(prev), fields[1]))
|
||||||
|
}
|
||||||
|
m[tr[0]] = si[0]
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeSourceKey normalizes a zh/ja SOURCE surface (a glossary key/alias OR the
|
||||||
|
// chunk text) for exact matching. Pure and deterministic; the pipeline is
|
||||||
|
// NFKC → trad→simp (per rune) → katakana→hiragana → Unicode lower. Applied to BOTH
|
||||||
|
// keys and text so orthographic variants match (A4). Kana folding mirrors the eval
|
||||||
|
// spec (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to
|
||||||
|
// hiragana; the prolonged-sound mark ー (U+30FC) and half-width forms are already
|
||||||
|
// handled (the latter by NFKC).
|
||||||
|
func normalizeSourceKey(s string) string {
|
||||||
|
s = norm.NFKC.String(s)
|
||||||
|
var b strings.Builder
|
||||||
|
b.Grow(len(s))
|
||||||
|
for _, r := range s {
|
||||||
|
if m, ok := trad2simp[r]; ok {
|
||||||
|
r = m
|
||||||
|
}
|
||||||
|
if r >= 0x30A1 && r <= 0x30F6 { // katakana → hiragana
|
||||||
|
r -= 0x60
|
||||||
|
}
|
||||||
|
b.WriteRune(unicode.ToLower(r))
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeTargetForm normalizes a Russian TARGET form (a stored decl form OR the
|
||||||
|
// model output) for the post-check whole-word test. Applied SYMMETRICALLY to both
|
||||||
|
// sides so a correctly-rendered term is NEVER a false MISS (over-flagging is the
|
||||||
|
// failure mode research/14 §2 warns about). Steps, each closing a self-review finding:
|
||||||
|
// - NFC;
|
||||||
|
// - dash/hyphen variants (en-dash, NB-hyphen, minus, soft hyphen, …) → ASCII '-', so
|
||||||
|
// «А–кью»/«А‑кью» match a stored «А-кью» (self-review #8);
|
||||||
|
// - runs of ANY Unicode whitespace (newline, NBSP U+00A0, ideographic space, …) →
|
||||||
|
// one ASCII space, then trim, so a multi-word name wrapped across a line break
|
||||||
|
// («Бородатого\nВана») matches a stored «Бородатого Вана» (self-review #2);
|
||||||
|
// - Unicode lower + ё→е fold (the ёфикатор's е/ё variation).
|
||||||
|
// Pure and deterministic.
|
||||||
|
func normalizeTargetForm(s string) string {
|
||||||
|
s = norm.NFC.String(s)
|
||||||
|
var b strings.Builder
|
||||||
|
b.Grow(len(s))
|
||||||
|
prevSpace := false
|
||||||
|
for _, r := range s {
|
||||||
|
if isDashVariant(r) {
|
||||||
|
r = '-'
|
||||||
|
}
|
||||||
|
if unicode.IsSpace(r) {
|
||||||
|
if !prevSpace {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
prevSpace = true
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
prevSpace = false
|
||||||
|
r = unicode.ToLower(r)
|
||||||
|
if r == 'ё' {
|
||||||
|
r = 'е'
|
||||||
|
}
|
||||||
|
b.WriteRune(r)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(b.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// isDashVariant reports whether r is a hyphen/dash/minus variant that should fold to
|
||||||
|
// ASCII '-' for the post-check (Unicode dash punctuation + the math minus + soft hyphen).
|
||||||
|
func isDashVariant(r rune) bool {
|
||||||
|
switch r {
|
||||||
|
case '‐', '‑', '‒', '–', '—', '―', '−', '':
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// significantLen counts the "significant" characters of a normalized key — CJK
|
||||||
|
// ideographs/kana plus letters (Latin/Cyrillic) — for the single-key ban (registry
|
||||||
|
// A3): a key whose significant length is below minKeyLen may not fire on its own
|
||||||
|
// (a lone Han/kana/letter like 炎/气/钱 matches unrelated text on the wrong sense).
|
||||||
|
// Digits and punctuation do not count (so "阿Q" = 2, but a bare "Q" = 1). Mirrors the
|
||||||
|
// eval spec's cjk_len.
|
||||||
|
func significantLen(normalized string) int {
|
||||||
|
n := 0
|
||||||
|
for _, r := range normalized {
|
||||||
|
switch {
|
||||||
|
case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul):
|
||||||
|
n++
|
||||||
|
case unicode.IsLetter(r):
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
116
backend/internal/pipeline/memnorm_test.go
Normal file
116
backend/internal/pipeline/memnorm_test.go
Normal file
|
|
@ -0,0 +1,116 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestTradTableCoversKnownCases asserts the embedded trad→simp table covers every
|
||||||
|
// character referenced by the eval specs / research/14 live bugs. A mutation
|
||||||
|
// (deleting a mapping line) fails exactly here — the guard against silently
|
||||||
|
// regressing the A4 hole (爺→爷 was research/14's live "тихо пусто" bug).
|
||||||
|
func TestTradTableCoversKnownCases(t *testing.T) {
|
||||||
|
want := map[rune]rune{
|
||||||
|
'魯': '鲁', '鎮': '镇', '趙': '赵', '蕭': '萧', '煉': '炼',
|
||||||
|
'門': '门', '闆': '板', '莊': '庄', '錢': '钱', '陳': '陈',
|
||||||
|
'萬': '万', '舊': '旧', '臉': '脸', '爺': '爷', '羅': '罗',
|
||||||
|
}
|
||||||
|
for tr, si := range want {
|
||||||
|
if got, ok := trad2simp[tr]; !ok || got != si {
|
||||||
|
t.Errorf("trad2simp[%q] = %q, ok=%v; want %q", string(tr), string(got), ok, string(si))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(trad2simp) < 200 {
|
||||||
|
t.Errorf("trad2simp table has only %d entries — expected the curated high-frequency set (≥200)", len(trad2simp))
|
||||||
|
}
|
||||||
|
if memoryNormVersion == "" || !strings.HasPrefix(memoryNormVersion, memoryNormAlgoVersion) {
|
||||||
|
t.Errorf("memoryNormVersion=%q must start with the algo tag %q", memoryNormVersion, memoryNormAlgoVersion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSourceKey(t *testing.T) {
|
||||||
|
cases := []struct{ name, in, want string }{
|
||||||
|
{"trad→simp two-char", "魯鎮", "鲁镇"},
|
||||||
|
{"trad→simp name", "趙太爺", "赵太爷"},
|
||||||
|
{"fullwidth latin lowered", "阿Ｑ", "阿q"},
|
||||||
|
{"halfwidth latin lowered", "阿Q", "阿q"},
|
||||||
|
{"katakana→hiragana", "スズキ", "すずき"},
|
||||||
|
{"halfwidth katakana via NFKC", "ｽｽｷ", "すすき"}, // NFKC folds half→full katakana, then kana-fold to hiragana
|
||||||
|
{"plain han unchanged", "送灶", "送灶"},
|
||||||
|
{"mixed", "萧炎", "萧炎"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
if got := normalizeSourceKey(c.in); got != c.want {
|
||||||
|
t.Errorf("normalizeSourceKey(%q) = %q; want %q", c.in, got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNormalizeSymmetry is the A4 invariant: a key written in one orthography
|
||||||
|
// matches a chunk written in another, because normalization is applied to BOTH.
|
||||||
|
func TestNormalizeSymmetry(t *testing.T) {
|
||||||
|
// glossary key in Simplified, chunk text in Traditional
|
||||||
|
key := normalizeSourceKey("鲁镇") // simplified
|
||||||
|
text := normalizeSourceKey("魯鎮的冬天很冷。") // traditional
|
||||||
|
if !strings.Contains(text, key) {
|
||||||
|
t.Errorf("normalized simplified key %q not found in normalized traditional text %q", key, text)
|
||||||
|
}
|
||||||
|
// katakana name key vs hiragana surface
|
||||||
|
k2 := normalizeSourceKey("スズキ")
|
||||||
|
t2 := normalizeSourceKey("すずきさんが来た。")
|
||||||
|
if !strings.Contains(t2, k2) {
|
||||||
|
t.Errorf("normalized katakana key %q not found in normalized hiragana text %q", k2, t2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeTargetForm(t *testing.T) {
|
||||||
|
cases := []struct{ in, want string }{
|
||||||
|
{"А-кью", "а-кью"},
|
||||||
|
{"Вэйчжуан", "вэйчжуан"},
|
||||||
|
{"Тёмный страж", "темный страж"}, // NFC → lower → ё→е fold
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := normalizeTargetForm(c.in); got != c.want {
|
||||||
|
t.Errorf("normalizeTargetForm(%q) = %q; want %q", c.in, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// ё and е must fold to the SAME normalized form (symmetric — no false miss)
|
||||||
|
if normalizeTargetForm("сёстры") != normalizeTargetForm("сестры") {
|
||||||
|
t.Error("ё→е fold is not symmetric: сёстры vs сестры differ after normalization")
|
||||||
|
}
|
||||||
|
// self-review #2: internal whitespace runs (incl. newline/NBSP) collapse to one space.
|
||||||
|
for _, v := range []string{"Бородатого\nВана", "Бородатого Вана", "Бородатого Вана"} {
|
||||||
|
if got := normalizeTargetForm(v); got != "бородатого вана" {
|
||||||
|
t.Errorf("whitespace not canonicalized: %q → %q", v, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// self-review #8: dash/hyphen variants fold to ASCII '-'.
|
||||||
|
for _, v := range []string{"А–кью", "А‑кью", "А−кью"} {
|
||||||
|
if got := normalizeTargetForm(v); got != "а-кью" {
|
||||||
|
t.Errorf("dash variant not unified: %q → %q", v, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSignificantLen(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
in string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"阿q", 2}, // han + latin letter
|
||||||
|
{"炎", 1}, // single han — banned as a standalone key
|
||||||
|
{"气", 1}, // single han
|
||||||
|
{"钱", 1}, // single han (research/14 homograph)
|
||||||
|
{"送灶", 2}, // bigram — allowed
|
||||||
|
{"すずき", 3}, // three kana
|
||||||
|
{"q", 1}, // bare latin letter
|
||||||
|
{"а-кью", 4}, // cyrillic letters, hyphen not counted: а к ь ю = 4
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := significantLen(c.in); got != c.want {
|
||||||
|
t.Errorf("significantLen(%q) = %d; want %d", c.in, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
592
backend/internal/pipeline/memory.go
Normal file
592
backend/internal/pipeline/memory.go
Normal file
|
|
@ -0,0 +1,592 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
"textmachine/backend/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// memory.go: the DETERMINISTIC hot path of the memory bank v2 (registry §Контракт
|
||||||
|
// горячего пути / research/13 Q3). $0, no LLM, no embeddings, no FTS5 — a multi-pattern
|
||||||
|
// exact match (Aho-Corasick) of glossary keys/aliases over the NORMALIZED chunk, with
|
||||||
|
// the single-key ban (A3), longest-match whole-entity replacement, the spoiler
|
||||||
|
// hard-reject window (C1), sticky scene-inertia (A5), a priority token budget with a
|
||||||
|
// logged eviction (F2), and the three-way injection disposition (A2). Pure and
|
||||||
|
// deterministic (no map-order output, no time/rand): the injected block is a pure
|
||||||
|
// function of (frozen bank, normalized chunk, chapter, sticky, budget), and it enters
|
||||||
|
// request_hash via the rendered messages — so a resumed chunk reproduces it for free.
|
||||||
|
//
|
||||||
|
// The registry's principle drives every choice here: "лучше ничего, чем мусор" and
|
||||||
|
// PRECISION over recall. A wrong injection (a homograph / short alias firing on the
|
||||||
|
// wrong sense) is the MAIN source of silent degradation (2510.00829) — worse than an
|
||||||
|
// empty match, which is a safe fallback to the model's base behaviour. So the matcher
|
||||||
|
// errs toward NOT firing, and the post-check (memcheck) makes what did fire observable.
|
||||||
|
|
||||||
|
// memoryMatchVersion versions the matcher + injection-selection ALGORITHM (min-key
|
||||||
|
// ban, longest-match containment, spoiler gate, sticky, budget priority order). Folded
|
||||||
|
// into memoryVersion() → a change is a loud --resnapshot (it shifts the injected bytes
|
||||||
|
// → the wire → a resumed checkpoint's content). Sibling of memoryNormVersion.
|
||||||
|
const memoryMatchVersion = "memmatch-v1-longest+minkey2+spoiler+sticky+budget+postcheck-declaware"
|
||||||
|
|
||||||
|
// defaultMinKeyLen is the minimum significant length (significantLen) a key must have
|
||||||
|
// to fire on its own (A3 single-key ban): a lone Han/kana/letter (炎/气/钱) matches
|
||||||
|
// unrelated text on the wrong sense — empirically confirmed (research/14). A per-entry
|
||||||
|
// allow_short overrides it (rare, guarded). Per-language tuning (registry min_key_len
|
||||||
|
// per-язык) is a later knob; v1 uses the eval spec's global MIN_KEY=2.
|
||||||
|
const defaultMinKeyLen = 2
|
||||||
|
|
||||||
|
// stickyDepth is the scene-inertia window (A5): entries exact-matched in the previous
|
||||||
|
// N chunks of the SAME chapter are carried into a pronominal chunk that names nobody.
|
||||||
|
// A code rule (Р2), reset at each chapter boundary (a new chapter is a scene change).
|
||||||
|
const stickyDepth = 2
|
||||||
|
|
||||||
|
// injectionDisposition is the three-way per-record decision (registry A2) — the core
|
||||||
|
// mechanism that converts silent degradation into loud. Distinct from the chunk×stage
|
||||||
|
// Disposition (disposition.go): that is ok|flagged|skipped for a completion; this is
|
||||||
|
// how much to TRUST an injected glossary record.
|
||||||
|
type injectionDisposition string
|
||||||
|
|
||||||
|
const (
|
||||||
|
memConfirmed injectionDisposition = "confirmed" // exact key/alias, approved, spoiler-valid → authoritative
|
||||||
|
memAmbiguous injectionDisposition = "ambiguous" // auto/draft term → inject "unverified" + forced post-check
|
||||||
|
memReject injectionDisposition = "reject" // spoiler-window violation → dropped and logged (C1 safety)
|
||||||
|
)
|
||||||
|
|
||||||
|
// declInfo is the parsed decl column: the dst declension forms the post-check accepts.
|
||||||
|
type declInfo struct {
|
||||||
|
Invariant bool `json:"invariant"`
|
||||||
|
Forms []string `json:"forms"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// memoryEntry is one frozen, materialized glossary entry with its matchable keys
|
||||||
|
// pre-normalized. Immutable for a job.
|
||||||
|
type memoryEntry struct {
|
||||||
|
id string // stable unique id: src\x1f sense\x1f since\x1f until (= the UNIQUE key)
|
||||||
|
src string // raw source key
|
||||||
|
dst string // raw approved translation (may be "" for a ruby candidate)
|
||||||
|
status string // auto|draft|approved
|
||||||
|
sense string
|
||||||
|
sinceCh int
|
||||||
|
untilCh int
|
||||||
|
// normKeys are the normalized source surfaces (src + aliases) ELIGIBLE to fire
|
||||||
|
// (significantLen ≥ minKeyLen OR allow_short). A key too short is dropped here, so
|
||||||
|
// it is never in the automaton — the single-key ban is structural, not a runtime skip.
|
||||||
|
normKeys []string
|
||||||
|
// declForms are the normalized-target dst forms the post-check accepts. Empty →
|
||||||
|
// the post-check falls back to the single normalized dst (the naive base form that
|
||||||
|
// research/14 shows false-flags on inflection — the reason full decl matters).
|
||||||
|
declForms []string
|
||||||
|
declInvariant bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// MemoryBank is a book's frozen glossary materialized for one job: the entries, the
|
||||||
|
// Aho-Corasick automaton over all eligible keys, and the F1 memoryVersion. Built once
|
||||||
|
// (materializeMemory) before the chunk loop and never mutated.
|
||||||
|
type MemoryBank struct {
|
||||||
|
entries []memoryEntry
|
||||||
|
ac *ahoCorasick
|
||||||
|
keyOwners map[string][]int // normalized key → indices of entries that contributed it
|
||||||
|
version string // memoryVersion(): hash of frozen APPROVED rows + algo versions (F1/D8)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pickedEntry is one selected record for a chunk with its firing key and disposition.
|
||||||
|
type pickedEntry struct {
|
||||||
|
entry *memoryEntry
|
||||||
|
via string // the matching key, or "sticky"
|
||||||
|
disp injectionDisposition
|
||||||
|
}
|
||||||
|
|
||||||
|
// memorySelection is the hot path's output for one chunk.
|
||||||
|
type memorySelection struct {
|
||||||
|
injected []pickedEntry // in budget priority order (what the model sees)
|
||||||
|
rejected []pickedEntry // spoiler-window rejects (C1, logged)
|
||||||
|
evicted []pickedEntry // dropped by the token budget (F2, logged)
|
||||||
|
activeIDs map[string]bool // exact-matched ids (NOT sticky) → the next chunk's sticky_prev
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *MemoryBank) Version() string { return b.version }
|
||||||
|
|
||||||
|
// materializeMemory builds a MemoryBank from the book's stored glossary rows (ORDER
|
||||||
|
// BY-stable — GlossaryForBook). Pure and deterministic. It computes memoryVersion as a
|
||||||
|
// content hash of the frozen APPROVED rows (D8: content-hash, not a version-counter —
|
||||||
|
// drift-proof) plus the normalization + matcher algorithm versions, so ANY change to
|
||||||
|
// the approved glossary OR to the deterministic machinery is a loud --resnapshot (F1).
|
||||||
|
func materializeMemory(rows []store.GlossaryEntry, gateOn bool) *MemoryBank {
|
||||||
|
b := &MemoryBank{keyOwners: map[string][]int{}}
|
||||||
|
var allKeys []string
|
||||||
|
seenKey := map[string]bool{}
|
||||||
|
|
||||||
|
for _, row := range rows {
|
||||||
|
e := memoryEntry{
|
||||||
|
id: row.Src + "\x1f" + row.Sense + "\x1f" + strconv.Itoa(row.SinceCh) + "\x1f" + strconv.Itoa(row.UntilCh),
|
||||||
|
src: row.Src,
|
||||||
|
dst: row.Dst,
|
||||||
|
status: row.Status,
|
||||||
|
sense: row.Sense,
|
||||||
|
sinceCh: row.SinceCh,
|
||||||
|
untilCh: row.UntilCh,
|
||||||
|
}
|
||||||
|
// Eligible source surfaces (src + aliases) under the single-key ban. A record
|
||||||
|
// with NO dst yet (a ruby auto-candidate) is NOT matchable: it renders nothing
|
||||||
|
// (renderGlossaryBlock/postcheck both skip empty dst), so admitting its keys would
|
||||||
|
// let it consume the token budget, evict a renderable line, and inflate the
|
||||||
|
// retrieval-state exact-hit count for a record the model never sees (self-review
|
||||||
|
// #1). It stays in the store for future promotion, just inert on the hot path.
|
||||||
|
var surfaces []string
|
||||||
|
if strings.TrimSpace(row.Dst) != "" {
|
||||||
|
surfaces = append(surfaces, row.Src)
|
||||||
|
for _, a := range row.Aliases {
|
||||||
|
surfaces = append(surfaces, a.Alias)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range surfaces {
|
||||||
|
nk := normalizeSourceKey(s)
|
||||||
|
if nk == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if significantLen(nk) < defaultMinKeyLen && !row.AllowShort {
|
||||||
|
continue // A3: a lone Han/kana/letter never fires on its own
|
||||||
|
}
|
||||||
|
e.normKeys = append(e.normKeys, nk)
|
||||||
|
}
|
||||||
|
// Parse decl forms for the post-check (normalized target side).
|
||||||
|
if row.Decl != "" {
|
||||||
|
var d declInfo
|
||||||
|
if err := json.Unmarshal([]byte(row.Decl), &d); err == nil {
|
||||||
|
e.declInvariant = d.Invariant
|
||||||
|
for _, f := range d.Forms {
|
||||||
|
if nf := normalizeTargetForm(f); nf != "" {
|
||||||
|
e.declForms = append(e.declForms, nf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
idx := len(b.entries)
|
||||||
|
b.entries = append(b.entries, e)
|
||||||
|
for _, nk := range e.normKeys {
|
||||||
|
b.keyOwners[nk] = append(b.keyOwners[nk], idx)
|
||||||
|
if !seenKey[nk] {
|
||||||
|
seenKey[nk] = true
|
||||||
|
allKeys = append(allKeys, nk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(allKeys) // deterministic automaton construction
|
||||||
|
b.ac = buildAC(allKeys)
|
||||||
|
b.version = computeMemoryVersion(rows, gateOn)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeMemoryVersion is the F1 content-hash: the frozen rows (ORDER BY-stable) plus the
|
||||||
|
// normalization and matcher algorithm versions. Approved-only per D8/§8 when the post-check
|
||||||
|
// hard gate is OFF (auto/draft injected-content changes are caught at the per-chunk
|
||||||
|
// content_hash level; their decl changes affect only the recomputed retrieval-state).
|
||||||
|
// When the gate is ON, the resolved chunk disposition depends on the decl forms of ALL
|
||||||
|
// injected records (approved AND auto/draft), and those are in NEITHER content_hash (decl
|
||||||
|
// is not injected) NOR the approved-only hash — so a decl edit would silently flip a
|
||||||
|
// resumed chunk's disposition (self-review #4). So gateOn folds every row's content
|
||||||
|
// (incl. decl + status) into the hash, making any such edit a loud --resnapshot. The
|
||||||
|
// glossary.id autoincrement is deliberately EXCLUDED (fresh each replace); only content
|
||||||
|
// columns are hashed.
|
||||||
|
func computeMemoryVersion(rows []store.GlossaryEntry, gateOn bool) string {
|
||||||
|
h := sha256.New()
|
||||||
|
h.Write([]byte("tm-memory-v2\x00"))
|
||||||
|
h.Write([]byte(memoryNormVersion + "\x00" + memoryMatchVersion + "\x00"))
|
||||||
|
h.Write([]byte("gate:" + strconv.FormatBool(gateOn) + "\x00"))
|
||||||
|
for _, r := range rows {
|
||||||
|
if r.Status != "approved" && !gateOn {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// A fixed, length-prefixed field layout so no content can forge a boundary.
|
||||||
|
writeField(h, r.Status)
|
||||||
|
writeField(h, r.Src)
|
||||||
|
writeField(h, r.Dst)
|
||||||
|
writeField(h, r.Sense)
|
||||||
|
writeField(h, r.Type)
|
||||||
|
writeField(h, r.Gender)
|
||||||
|
writeField(h, r.Decl)
|
||||||
|
writeField(h, strconv.Itoa(r.SinceCh))
|
||||||
|
writeField(h, strconv.Itoa(r.UntilCh))
|
||||||
|
writeField(h, strconv.FormatBool(r.AllowShort))
|
||||||
|
writeField(h, strconv.Itoa(len(r.Aliases)))
|
||||||
|
for _, a := range r.Aliases { // GlossaryForBook returns aliases ORDER BY alias
|
||||||
|
writeField(h, a.Alias)
|
||||||
|
writeField(h, a.AliasType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(h.Sum(nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeField(h interface{ Write([]byte) (int, error) }, s string) {
|
||||||
|
var lb [8]byte
|
||||||
|
n := uint64(len(s))
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
lb[i] = byte(n >> (8 * i))
|
||||||
|
}
|
||||||
|
h.Write(lb[:])
|
||||||
|
h.Write([]byte(s))
|
||||||
|
}
|
||||||
|
|
||||||
|
// glossaryLineTokens is the injected token cost of one record's "src → dst" line — the
|
||||||
|
// unit the token budget (config glossary_token_budget) spends. It is FLOOR-FREE (the raw
|
||||||
|
// cjk + other/3 estimate, NOT EstimateTokens' 16-token minimum): the floor is for sizing
|
||||||
|
// a whole request's max_tokens, but applying it PER glossary line ~1.85×-overcounts a
|
||||||
|
// block of short name lines and needlessly evicts records that fit (self-review #7). The
|
||||||
|
// header's cost is small and constant; omitting it keeps the unit purely per-record.
|
||||||
|
// Shared by Select's budget and the eviction test so the two never diverge.
|
||||||
|
func glossaryLineTokens(e *memoryEntry) int {
|
||||||
|
cjk, other := 0, 0
|
||||||
|
for _, r := range e.src + " → " + e.dst {
|
||||||
|
switch {
|
||||||
|
case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul):
|
||||||
|
cjk++
|
||||||
|
case unicode.IsSpace(r):
|
||||||
|
default:
|
||||||
|
other++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cjk + other/3
|
||||||
|
}
|
||||||
|
|
||||||
|
// Select is the hot path for ONE chunk: match → spoiler-reject → disposition → sticky
|
||||||
|
// → priority token budget. Pure and deterministic. stickyPrev is the set of ids
|
||||||
|
// exact-matched in the prior chunk(s) of the same chapter (A5). budgetTokens ≤ 0 means
|
||||||
|
// unbounded; otherwise records are kept in priority order while the cumulative injected
|
||||||
|
// token cost stays within budget — the rest are EVICTED (dropped but logged, F2; the
|
||||||
|
// budget may be underfilled, "лучше ничего, чем мусор").
|
||||||
|
func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]bool, budgetTokens int) memorySelection {
|
||||||
|
ntext := []rune(normalizeSourceKey(chunk))
|
||||||
|
occ := b.ac.matches(ntext)
|
||||||
|
// Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when
|
||||||
|
// the longer key belongs to a spoiler-VALID entry at this chapter (self-review #6). A
|
||||||
|
// spoiler-blocked longer key must not suppress a valid shorter name nested inside it.
|
||||||
|
occ = b.suppressContained(occ, chapter)
|
||||||
|
|
||||||
|
// Map surviving key occurrences → entries, recording the LONGEST firing key per entry.
|
||||||
|
matchedVia := map[int]string{}
|
||||||
|
for _, m := range occ {
|
||||||
|
k := b.ac.keys[m.keyIdx]
|
||||||
|
for _, ei := range b.keyOwners[k] {
|
||||||
|
if cur, ok := matchedVia[ei]; !ok || len([]rune(k)) > len([]rune(cur)) {
|
||||||
|
matchedVia[ei] = k
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sel := memorySelection{activeIDs: map[string]bool{}}
|
||||||
|
hits := map[string]pickedEntry{} // id → picked (exact)
|
||||||
|
|
||||||
|
// Iterate entries in their frozen (ORDER BY-stable) order — never map order.
|
||||||
|
for ei := range b.entries {
|
||||||
|
via, ok := matchedVia[ei]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e := &b.entries[ei]
|
||||||
|
if blocked := spoilerBlocked(e, chapter); blocked {
|
||||||
|
sel.rejected = append(sel.rejected, pickedEntry{e, via, memReject})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hits[e.id] = pickedEntry{e, via, dispositionFor(e)}
|
||||||
|
sel.activeIDs[e.id] = true // exact-matched ids feed the next chunk's sticky (NOT sticky-carried)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sticky scene-inertia: carry prior-chunk exact matches not re-matched here, unless
|
||||||
|
// the spoiler window blocks them (spoiler beats sticky).
|
||||||
|
for ei := range b.entries {
|
||||||
|
e := &b.entries[ei]
|
||||||
|
if !stickyPrev[e.id] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, already := hits[e.id]; already {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if spoilerBlocked(e, chapter) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hits[e.id] = pickedEntry{e, "sticky", dispositionFor(e)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deterministic priority order for the budget: confirmed>ambiguous, exact>sticky,
|
||||||
|
// approved>auto. Collect in frozen-entry order (NOT map order) then stable-sort, so
|
||||||
|
// ties keep the deterministic base order.
|
||||||
|
order := make([]pickedEntry, 0, len(hits))
|
||||||
|
for ei := range b.entries {
|
||||||
|
if p, ok := hits[b.entries[ei].id]; ok {
|
||||||
|
order = append(order, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(order, func(i, j int) bool {
|
||||||
|
return priorityRank(order[i]) < priorityRank(order[j])
|
||||||
|
})
|
||||||
|
|
||||||
|
if budgetTokens > 0 {
|
||||||
|
used, cut := 0, len(order)
|
||||||
|
for i := range order {
|
||||||
|
used += glossaryLineTokens(order[i].entry)
|
||||||
|
if used > budgetTokens {
|
||||||
|
cut = i // this record and everything after it overflow the budget
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sel.injected = order[:cut]
|
||||||
|
sel.evicted = order[cut:] // F2: dropped, but LOGGED via the retrieval-state
|
||||||
|
} else {
|
||||||
|
sel.injected = order
|
||||||
|
}
|
||||||
|
return sel
|
||||||
|
}
|
||||||
|
|
||||||
|
// spoilerBlocked reports whether the entry's since_ch/until_ch window excludes this
|
||||||
|
// chapter (C1: a hard reject gate — a fact the current chapter must not know yet).
|
||||||
|
func spoilerBlocked(e *memoryEntry, chapter int) bool {
|
||||||
|
if e.sinceCh > 0 && chapter < e.sinceCh {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if e.untilCh > 0 && chapter > e.untilCh {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// dispositionFor maps a term's status to its injection disposition (A2): approved →
|
||||||
|
// CONFIRMED (authoritative); auto/draft → AMBIGUOUS (inject "unverified" + forced
|
||||||
|
// post-check).
|
||||||
|
func dispositionFor(e *memoryEntry) injectionDisposition {
|
||||||
|
if e.status == "approved" {
|
||||||
|
return memConfirmed
|
||||||
|
}
|
||||||
|
return memAmbiguous
|
||||||
|
}
|
||||||
|
|
||||||
|
// priorityRank orders the budget: confirmed before ambiguous, exact before sticky,
|
||||||
|
// approved before auto/draft (F2 eviction keeps the most trustworthy records).
|
||||||
|
func priorityRank(p pickedEntry) int {
|
||||||
|
rank := 0
|
||||||
|
if p.disp != memConfirmed {
|
||||||
|
rank |= 1 << 2
|
||||||
|
}
|
||||||
|
if p.via == "sticky" {
|
||||||
|
rank |= 1 << 1
|
||||||
|
}
|
||||||
|
if p.entry.status != "approved" {
|
||||||
|
rank |= 1 << 0
|
||||||
|
}
|
||||||
|
return rank
|
||||||
|
}
|
||||||
|
|
||||||
|
// glossaryBlockHeader introduces the injected glossary block. The layout mirrors the
|
||||||
|
// SakuraLLM/GalTransl convention "src → dst", with unverified (ambiguous) records
|
||||||
|
// tagged so the model — and a human reviewer — see they are not authoritative (A2).
|
||||||
|
const glossaryBlockHeader = "ГЛОССАРИЙ (используй эти утверждённые переводы имён и терминов последовательно; строки с пометкой ⟨проверить⟩ — неподтверждённые кандидаты):"
|
||||||
|
|
||||||
|
// renderGlossaryBlock serializes the selected records into the injection message
|
||||||
|
// (§C). Records with no dst yet (ruby candidates) are skipped — a "src → " line
|
||||||
|
// carries nothing. Returns "" when nothing renders, so an empty selection injects NO
|
||||||
|
// message at all ("лучше ничего, чем мусор"). Deterministic: the injected order is the
|
||||||
|
// budget priority order fixed by Select.
|
||||||
|
func renderGlossaryBlock(injected []pickedEntry) string {
|
||||||
|
var lines []string
|
||||||
|
for _, p := range injected {
|
||||||
|
if strings.TrimSpace(p.entry.dst) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
line := p.entry.src + " → " + p.entry.dst
|
||||||
|
if p.disp != memConfirmed {
|
||||||
|
line += " ⟨проверить⟩"
|
||||||
|
}
|
||||||
|
lines = append(lines, line)
|
||||||
|
}
|
||||||
|
if len(lines) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return glossaryBlockHeader + "\n" + strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Aho-Corasick multi-pattern automaton over runes ----------------------------
|
||||||
|
|
||||||
|
type acMatch struct {
|
||||||
|
keyIdx int
|
||||||
|
start, end int // rune indices [start, end)
|
||||||
|
}
|
||||||
|
|
||||||
|
type acNode struct {
|
||||||
|
next map[rune]int
|
||||||
|
fail int
|
||||||
|
out []int // key indices whose pattern ends at this node (incl. via fail links)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ahoCorasick struct {
|
||||||
|
nodes []acNode
|
||||||
|
keys []string
|
||||||
|
klen []int // rune length of each key
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildAC constructs the automaton from the (already sorted, unique) normalized keys.
|
||||||
|
// Deterministic in RESULT regardless of map iteration order: fail links and outputs are
|
||||||
|
// a function of the key set, not the BFS visit order.
|
||||||
|
func buildAC(keys []string) *ahoCorasick {
|
||||||
|
ac := &ahoCorasick{keys: keys, klen: make([]int, len(keys))}
|
||||||
|
ac.nodes = []acNode{{next: map[rune]int{}}} // root = node 0
|
||||||
|
for i, k := range keys {
|
||||||
|
kr := []rune(k)
|
||||||
|
ac.klen[i] = len(kr)
|
||||||
|
cur := 0
|
||||||
|
for _, r := range kr {
|
||||||
|
nxt, ok := ac.nodes[cur].next[r]
|
||||||
|
if !ok {
|
||||||
|
nxt = len(ac.nodes)
|
||||||
|
ac.nodes = append(ac.nodes, acNode{next: map[rune]int{}})
|
||||||
|
ac.nodes[cur].next[r] = nxt
|
||||||
|
}
|
||||||
|
cur = nxt
|
||||||
|
}
|
||||||
|
ac.nodes[cur].out = append(ac.nodes[cur].out, i)
|
||||||
|
}
|
||||||
|
// BFS to compute fail links; propagate outputs down fail chains one level (each
|
||||||
|
// node inherits its fail node's already-complete output set).
|
||||||
|
var queue []int
|
||||||
|
for _, c := range ac.nodes[0].next {
|
||||||
|
ac.nodes[c].fail = 0
|
||||||
|
queue = append(queue, c)
|
||||||
|
}
|
||||||
|
for len(queue) > 0 {
|
||||||
|
cur := queue[0]
|
||||||
|
queue = queue[1:]
|
||||||
|
for r, nxt := range ac.nodes[cur].next {
|
||||||
|
queue = append(queue, nxt)
|
||||||
|
f := ac.nodes[cur].fail
|
||||||
|
for f != 0 {
|
||||||
|
if _, ok := ac.nodes[f].next[r]; ok {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
f = ac.nodes[f].fail
|
||||||
|
}
|
||||||
|
if fn, ok := ac.nodes[f].next[r]; ok && fn != nxt {
|
||||||
|
ac.nodes[nxt].fail = fn
|
||||||
|
} else {
|
||||||
|
ac.nodes[nxt].fail = 0
|
||||||
|
}
|
||||||
|
ac.nodes[nxt].out = append(ac.nodes[nxt].out, ac.nodes[ac.nodes[nxt].fail].out...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ac
|
||||||
|
}
|
||||||
|
|
||||||
|
// matches returns every key occurrence in text (rune indices), sorted deterministically.
|
||||||
|
func (ac *ahoCorasick) matches(text []rune) []acMatch {
|
||||||
|
var out []acMatch
|
||||||
|
cur := 0
|
||||||
|
for i, r := range text {
|
||||||
|
for cur != 0 {
|
||||||
|
if _, ok := ac.nodes[cur].next[r]; ok {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
cur = ac.nodes[cur].fail
|
||||||
|
}
|
||||||
|
if nxt, ok := ac.nodes[cur].next[r]; ok {
|
||||||
|
cur = nxt
|
||||||
|
} else {
|
||||||
|
cur = 0
|
||||||
|
}
|
||||||
|
for _, ki := range ac.nodes[cur].out {
|
||||||
|
out = append(out, acMatch{keyIdx: ki, start: i - ac.klen[ki] + 1, end: i + 1})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(a, b int) bool {
|
||||||
|
if out[a].start != out[b].start {
|
||||||
|
return out[a].start < out[b].start
|
||||||
|
}
|
||||||
|
if out[a].end != out[b].end {
|
||||||
|
return out[a].end < out[b].end
|
||||||
|
}
|
||||||
|
return out[a].keyIdx < out[b].keyIdx
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// suppressContained drops an occurrence fully contained inside a STRICTLY longer one
|
||||||
|
// (longest-match, whole-entity replacement — A3), where the longer one belongs to a
|
||||||
|
// spoiler-VALID entry at this chapter. Gating the SUPPRESSOR on spoiler validity closes
|
||||||
|
// self-review #6: a spoiler-blocked longer key (e.g. 林动的父亲, until_ch=3, read at ch10)
|
||||||
|
// must not silently drop a valid shorter name nested inside it (林动) — the shorter name
|
||||||
|
// survives and injects, the longer one is separately spoiler-rejected+logged. Equal-length
|
||||||
|
// overlaps are both kept (genuine surface ambiguity, surfaced by the injectivity check).
|
||||||
|
// O(n²) over the few matches in a chunk.
|
||||||
|
func (b *MemoryBank) suppressContained(ms []acMatch, chapter int) []acMatch {
|
||||||
|
validSuppressor := func(m acMatch) bool {
|
||||||
|
for _, ei := range b.keyOwners[b.ac.keys[m.keyIdx]] {
|
||||||
|
if !spoilerBlocked(&b.entries[ei], chapter) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var kept []acMatch
|
||||||
|
for i, m := range ms {
|
||||||
|
contained := false
|
||||||
|
for j, o := range ms {
|
||||||
|
if i == j {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start) && validSuppressor(o) {
|
||||||
|
contained = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !contained {
|
||||||
|
kept = append(kept, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return kept
|
||||||
|
}
|
||||||
|
|
||||||
|
// injectivityCollisions reports approved dst collisions (B2): two distinct source
|
||||||
|
// terms mapped to the SAME dst (one Russian surface for two entities → the reader
|
||||||
|
// cannot tell them apart), returned as human-readable strings for a load-time warning.
|
||||||
|
// A pure diagnostic; it does not reject (some collisions are legitimate, e.g. a title
|
||||||
|
// shared by rank tiers), so the caller logs it, not aborts.
|
||||||
|
func injectivityCollisions(rows []store.GlossaryEntry) []string {
|
||||||
|
bySurface := map[string][]string{}
|
||||||
|
for _, r := range rows {
|
||||||
|
if r.Status != "approved" || strings.TrimSpace(r.Dst) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := normalizeTargetForm(r.Dst)
|
||||||
|
bySurface[key] = append(bySurface[key], r.Src)
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
// Deterministic order: iterate a sorted key list, not the map.
|
||||||
|
keys := make([]string, 0, len(bySurface))
|
||||||
|
for k := range bySurface {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
srcs := bySurface[k]
|
||||||
|
if len(distinct(srcs)) > 1 {
|
||||||
|
sort.Strings(srcs)
|
||||||
|
out = append(out, "dst "+strconv.Quote(k)+" ← "+strings.Join(distinct(srcs), ", "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func distinct(ss []string) []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, s := range ss {
|
||||||
|
if !seen[s] {
|
||||||
|
seen[s] = true
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
177
backend/internal/pipeline/memory_e1_test.go
Normal file
177
backend/internal/pipeline/memory_e1_test.go
Normal file
|
|
@ -0,0 +1,177 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"textmachine/backend/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// memory_e1_test.go: the E1 MEASUREMENT the onboarding contract requires BEFORE the
|
||||||
|
// post-check may be trusted as a hard gate (research/13 §2 "post-check в русском —
|
||||||
|
// несущий И невалидированный"; research/14 §2 quantified it at 18–36% false flags for a
|
||||||
|
// naive matcher). It measures the decl-aware post-check's false-flag rate on REAL
|
||||||
|
// inflected Russian (the adaptive_levers/adaptive_reask canon: 阿Q ch.5–9, Rogov),
|
||||||
|
// empirically — not "примерно" (the agent's explicit ask).
|
||||||
|
//
|
||||||
|
// It answers two things the agent flagged:
|
||||||
|
// 1. Stored-decl WORKS when the forms are complete → 0 false flags on a correctly
|
||||||
|
// rendered, inflected chunk (the arm that lets the post-check be a real signal).
|
||||||
|
// 2. Stored-decl is NOT free-reliable: with only the base form (an under-filled decl,
|
||||||
|
// the OOV-translit risk — one cheap LLM call may miss cases) the SAME chunk yields
|
||||||
|
// false flags. This is why v1 keeps the post-check a FLAGGER, not a hard gate: if
|
||||||
|
// the seed under-fills decl, it must degrade to observability, not block chunks.
|
||||||
|
//
|
||||||
|
// The inflection cases are the exact ones research/14 measured a naive matcher losing:
|
||||||
|
// «Вэйчжуане» (loc, a SUFFIX inflection substring-matching handles), «Бородатого Вана»
|
||||||
|
// (a modifier that inflects INTERNALLY — substring of the base fails), «сюцая»
|
||||||
|
// (stem-final й→я — the base is not a substring), «Маленького Дэна» (internal).
|
||||||
|
|
||||||
|
// e1Entity is one canon entity with its full decl set.
|
||||||
|
type e1Entity struct {
|
||||||
|
src string
|
||||||
|
baseDst string
|
||||||
|
fullDecl []string
|
||||||
|
}
|
||||||
|
|
||||||
|
var e1Canon = []e1Entity{
|
||||||
|
{"阿Q", "А-кью", []string{"А-кью"}}, // hyphenated translit, invariant
|
||||||
|
{"未庄", "Вэйчжуан", []string{"Вэйчжуан", "Вэйчжуане", "Вэйчжуана", "Вэйчжуану", "Вэйчжуаном"}},
|
||||||
|
{"王胡", "Бородатый Ван", []string{"Бородатый Ван", "Бородатого Вана", "Бородатому Вану", "Бородатым Ваном", "Бородатом Ване"}},
|
||||||
|
{"秀才", "сюцай", []string{"сюцай", "сюцая", "сюцаю", "сюцае", "сюцаем"}},
|
||||||
|
{"小D", "Маленький Дэн", []string{"Маленький Дэн", "Маленького Дэна", "Маленькому Дэну", "Маленьким Дэном", "Маленьком Дэне"}},
|
||||||
|
{"邹七嫂", "тётушка Цзоу Седьмая", []string{"тётушка Цзоу Седьмая", "тётушки Цзоу Седьмой", "тётушку Цзоу Седьмую"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
// e1Source contains every src key so all six entities are injected.
|
||||||
|
const e1Source = "阿Q在未庄遇见王胡和秀才，又碰到小D，邹七嫂在一旁看着。"
|
||||||
|
|
||||||
|
// e1CorrectOutput is a publisher-quality rendering: every entity present but INFLECTED
|
||||||
|
// (the case a boundary regexp / a base-only matcher false-flags).
|
||||||
|
const e1CorrectOutput = "В Вэйчжуане Бородатого Вана и сюцая знали все. " +
|
||||||
|
"А-кью повстречал Маленького Дэна у реки; тётушка Цзоу Седьмая молча прошла мимо."
|
||||||
|
|
||||||
|
func e1Glossary(useFullDecl bool) []store.GlossaryEntry {
|
||||||
|
var out []store.GlossaryEntry
|
||||||
|
for _, e := range e1Canon {
|
||||||
|
g := gl(e.src, e.baseDst, "", "approved")
|
||||||
|
if useFullDecl {
|
||||||
|
g.Decl = declJSON(false, e.fullDecl...)
|
||||||
|
} else {
|
||||||
|
g.Decl = declJSON(true, e.baseDst) // naive: only the base form
|
||||||
|
}
|
||||||
|
out = append(out, g)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestE1PostcheckFalseFlagRate(t *testing.T) {
|
||||||
|
// Arm A — full decl: the post-check must NOT false-flag a correct inflected output.
|
||||||
|
bFull := bankFrom(e1Glossary(true))
|
||||||
|
selFull := bFull.Select(e1Source, 1, nil, 0)
|
||||||
|
if len(selFull.injected) != len(e1Canon) {
|
||||||
|
t.Fatalf("expected all %d canon entities injected, got %d", len(e1Canon), len(selFull.injected))
|
||||||
|
}
|
||||||
|
fullMiss := bFull.postcheck(selFull.injected, e1CorrectOutput)
|
||||||
|
if len(fullMiss) != 0 {
|
||||||
|
t.Errorf("STORED-DECL FALSE FLAGS on correct output: %d/%d — %v", len(fullMiss), len(selFull.injected), fullMiss)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Arm B — base only (under-filled decl): the SAME correct output now false-flags.
|
||||||
|
bBase := bankFrom(e1Glossary(false))
|
||||||
|
selBase := bBase.Select(e1Source, 1, nil, 0)
|
||||||
|
baseMiss := bBase.postcheck(selBase.injected, e1CorrectOutput)
|
||||||
|
|
||||||
|
fullRate := float64(len(fullMiss)) / float64(len(selFull.injected))
|
||||||
|
baseRate := float64(len(baseMiss)) / float64(len(selBase.injected))
|
||||||
|
t.Logf("E1 false-flag rate on real inflected Russian: full-decl %.0f%% (%d/%d), base-only %.0f%% (%d/%d)",
|
||||||
|
fullRate*100, len(fullMiss), len(selFull.injected),
|
||||||
|
baseRate*100, len(baseMiss), len(selBase.injected))
|
||||||
|
|
||||||
|
// The measurement's verdict: decl-awareness is essential (full ≪ base), and the
|
||||||
|
// stored-decl matcher is trustworthy ONLY when complete. Base-only reproduces the
|
||||||
|
// research/14 failure — the reason the post-check stays a flagger until validated.
|
||||||
|
if len(baseMiss) == 0 {
|
||||||
|
t.Error("base-only decl produced 0 false flags — the E1 measurement lost its hard inflection cases; check the fixture")
|
||||||
|
}
|
||||||
|
if len(fullMiss) >= len(baseMiss) {
|
||||||
|
t.Errorf("full decl (%d) did not reduce false flags vs base (%d)", len(fullMiss), len(baseMiss))
|
||||||
|
}
|
||||||
|
// The specific research/14 internal-inflection cases MUST be the base-only misses.
|
||||||
|
for _, src := range []string{"王胡", "秀才", "小D"} {
|
||||||
|
if !hasMiss(baseMiss, src) {
|
||||||
|
t.Errorf("base-only decl did not false-flag %s (expected an internal-inflection false flag)", src)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPostcheckWhitespaceAndDashRobust covers self-review #2 (multi-word name wrapped
|
||||||
|
// across a newline / doubled space / NBSP) and #8 (hyphen/dash variants): a correctly
|
||||||
|
// rendered term in ANY of these forms must NOT be false-flagged.
|
||||||
|
func TestPostcheckWhitespaceAndDashRobust(t *testing.T) {
|
||||||
|
wang := gl("王胡", "Бородатый Ван", "", "approved")
|
||||||
|
wang.Decl = declJSON(false, "Бородатый Ван", "Бородатого Вана")
|
||||||
|
ahq := gl("阿Q", "А-кью", "", "approved")
|
||||||
|
ahq.Decl = declJSON(true, "А-кью")
|
||||||
|
b := bankFrom([]store.GlossaryEntry{wang, ahq})
|
||||||
|
sel := b.Select("王胡和阿Q在场。", 1, nil, 0)
|
||||||
|
|
||||||
|
robust := []string{
|
||||||
|
"Бородатого Вана и А-кью видели.", // baseline
|
||||||
|
"Бородатого\nВана и А-кью видели.", // #2 newline between the name's words
|
||||||
|
"Бородатого Вана и А-кью видели.", // #2 doubled space
|
||||||
|
"Бородатого Вана и А-кью видели.", // #2 NBSP
|
||||||
|
"Бородатого Вана и А–кью видели.", // #8 en-dash in А-кью
|
||||||
|
"Бородатого Вана и А‑кью видели.", // #8 non-breaking hyphen
|
||||||
|
}
|
||||||
|
for _, out := range robust {
|
||||||
|
if m := b.postcheck(sel.injected, out); len(m) != 0 {
|
||||||
|
t.Errorf("false flag on a correctly-rendered variant %q: %v", out, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPostcheckWholeWordNoMasking covers self-review #3: a DROPPED short translit name
|
||||||
|
// must not be masked by an unrelated common word that merely contains its letters.
|
||||||
|
func TestPostcheckWholeWordNoMasking(t *testing.T) {
|
||||||
|
van := gl("王", "Ван", "", "approved")
|
||||||
|
van.AllowShort = true // single-char src, allowed here to exercise the target-side check
|
||||||
|
van.Decl = declJSON(false, "Ван", "Вана", "Вану", "Ваном", "Ване")
|
||||||
|
b := bankFrom([]store.GlossaryEntry{van})
|
||||||
|
sel := b.Select("王走了。", 1, nil, 0)
|
||||||
|
if len(sel.injected) != 1 {
|
||||||
|
t.Fatalf("expected 王 injected, got %d", len(sel.injected))
|
||||||
|
}
|
||||||
|
// 王 is ABSENT from each output, yet a naive substring would find «ван» inside these.
|
||||||
|
for _, masked := range []string{"Мимо прошёл караван верблюдов.", "Он сел на диван и молчал.", "Иван Петрович кивнул."} {
|
||||||
|
if m := b.postcheck(sel.injected, masked); !hasMiss(m, "王") {
|
||||||
|
t.Errorf("post-check masked a dropped name by an unrelated word: %q → %v", masked, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A genuine rendering (inflected) is found as a whole word.
|
||||||
|
if m := b.postcheck(sel.injected, "К нему подошёл Ван."); len(m) != 0 {
|
||||||
|
t.Errorf("false flag on a correct whole-word rendering: %v", m)
|
||||||
|
}
|
||||||
|
if m := b.postcheck(sel.injected, "Он говорил с Ваном о делах."); len(m) != 0 {
|
||||||
|
t.Errorf("false flag on a correct inflected rendering: %v", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestE1PostcheckCatchesPoisoning: with complete decl, a WRONG rendering (poisoning /
|
||||||
|
// the model ignoring the glossary) is still a true-positive flag — the post-check's
|
||||||
|
// actual job. Precision does not come for free: the decl forms are inflections of the
|
||||||
|
// FULL dst phrase, so a foreign name that merely shares the head noun does NOT slip past.
|
||||||
|
func TestE1PostcheckCatchesPoisoning(t *testing.T) {
|
||||||
|
b := bankFrom(e1Glossary(true))
|
||||||
|
sel := b.Select("王胡走过来。", 1, nil, 0) // only 王胡 fires
|
||||||
|
if len(sel.injected) != 1 {
|
||||||
|
t.Fatalf("expected 1 injected, got %d", len(sel.injected))
|
||||||
|
}
|
||||||
|
// Correct inflected rendering → no flag.
|
||||||
|
if m := b.postcheck(sel.injected, "К нему подошёл Бородатый Ван."); len(m) != 0 {
|
||||||
|
t.Errorf("false flag on correct rendering: %v", m)
|
||||||
|
}
|
||||||
|
// Poisoned rendering (a different name) → flagged (true positive).
|
||||||
|
if m := b.postcheck(sel.injected, "К нему подошёл Ван Ху."); !hasMiss(m, "王胡") {
|
||||||
|
t.Errorf("post-check missed the poisoning (Ван Ху ≠ Бородатый Ван): %v", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
99
backend/internal/pipeline/memory_f1_test.go
Normal file
99
backend/internal/pipeline/memory_f1_test.go
Normal file
|
|
@ -0,0 +1,99 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"textmachine/backend/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// memory_f1_test.go covers the F1 content-hash (D8): memoryVersion is a pure function of
|
||||||
|
// the frozen APPROVED rows + the algorithm versions. A change to the approved glossary
|
||||||
|
// re-pins loudly; auto/draft churn does not; the row id never leaks in.
|
||||||
|
|
||||||
|
func TestMemoryVersionStableAndSensitive(t *testing.T) {
|
||||||
|
base := []store.GlossaryEntry{
|
||||||
|
gl("阿Q", "А-кью", "name", "approved"),
|
||||||
|
gl("鲁镇", "Лучжэнь", "", "approved"),
|
||||||
|
}
|
||||||
|
v0 := computeMemoryVersion(base, false)
|
||||||
|
|
||||||
|
// Identical rows → identical version (idempotent; drift-proof content hash).
|
||||||
|
if computeMemoryVersion(base, false) != v0 {
|
||||||
|
t.Fatal("memoryVersion is not stable for identical rows")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Changing an APPROVED dst → different version (a loud --resnapshot).
|
||||||
|
changed := []store.GlossaryEntry{
|
||||||
|
gl("阿Q", "А-Кью (изменено)", "name", "approved"),
|
||||||
|
gl("鲁镇", "Лучжэнь", "", "approved"),
|
||||||
|
}
|
||||||
|
if computeMemoryVersion(changed, false) == v0 {
|
||||||
|
t.Error("changing an approved dst did not change memoryVersion — F1 would silently regress")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adding an alias to an approved row → different version (aliases affect matching).
|
||||||
|
withAlias := []store.GlossaryEntry{
|
||||||
|
func() store.GlossaryEntry { e := gl("阿Q", "А-кью", "name", "approved"); e.Aliases = []store.GlossaryAlias{alias("阿桂")}; return e }(),
|
||||||
|
gl("鲁镇", "Лучжэнь", "", "approved"),
|
||||||
|
}
|
||||||
|
if computeMemoryVersion(withAlias, false) == v0 {
|
||||||
|
t.Error("adding an alias to an approved row did not change memoryVersion")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Changing an AUTO row → SAME version (approved-only, D8; auto changes are caught by
|
||||||
|
// the per-chunk content_hash instead).
|
||||||
|
withAuto := append(append([]store.GlossaryEntry{}, base...), gl("小D", "Малыш Дэ", "", "auto"))
|
||||||
|
if computeMemoryVersion(withAuto, false) != v0 {
|
||||||
|
t.Error("an auto row changed memoryVersion — it must hash only approved rows (D8)")
|
||||||
|
}
|
||||||
|
withAuto2 := append(append([]store.GlossaryEntry{}, base...), gl("小D", "Другой Дэ", "", "auto"))
|
||||||
|
if computeMemoryVersion(withAuto, false) != computeMemoryVersion(withAuto2, false) {
|
||||||
|
t.Error("editing an auto row changed memoryVersion — auto must not enter the F1 hash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMemoryVersionIgnoresID: the autoincrement glossary.id is fresh on every replace;
|
||||||
|
// it must NOT feed the hash (else a re-seed of identical content would false-resnapshot).
|
||||||
|
func TestMemoryVersionIgnoresID(t *testing.T) {
|
||||||
|
a := []store.GlossaryEntry{{BookID: "b", ID: 1, Src: "甲", Dst: "А", Status: "approved"}}
|
||||||
|
b := []store.GlossaryEntry{{BookID: "b", ID: 999, Src: "甲", Dst: "А", Status: "approved"}}
|
||||||
|
if computeMemoryVersion(a, false) != computeMemoryVersion(b, false) {
|
||||||
|
t.Error("memoryVersion depends on glossary.id — a re-seed would false-resnapshot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMaterializeMemoryVersionMatches: the bank's Version() equals computeMemoryVersion
|
||||||
|
// of its input rows (the runner hashes the same thing it matches on).
|
||||||
|
func TestMaterializeMemoryVersionMatches(t *testing.T) {
|
||||||
|
rows := specGlossary()
|
||||||
|
b := materializeMemory(rows, false)
|
||||||
|
if b.Version() != computeMemoryVersion(rows, false) {
|
||||||
|
t.Error("bank Version() diverges from computeMemoryVersion of its rows")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMemoryVersionGateFoldsUnapprovedDecl covers self-review #4: when the post-check
|
||||||
|
// hard gate is ON, editing a DRAFT/auto row's decl (which changes the gate's resolved
|
||||||
|
// disposition but is in NEITHER content_hash NOR the approved-only hash) MUST change
|
||||||
|
// memoryVersion — else a $0 resume silently flips a chunk. Gate OFF: it must NOT change
|
||||||
|
// it (D8 approved-only holds; decl affects only the recomputed retrieval-state).
|
||||||
|
func TestMemoryVersionGateFoldsUnapprovedDecl(t *testing.T) {
|
||||||
|
mk := func(declForm string) []store.GlossaryEntry {
|
||||||
|
e := gl("鈴木", "Судзуки", "", "draft")
|
||||||
|
e.Decl = declJSON(false, declForm)
|
||||||
|
return []store.GlossaryEntry{gl("阿Q", "А-кью", "", "approved"), e}
|
||||||
|
}
|
||||||
|
a := mk("Судзуки")
|
||||||
|
b := mk("Судзуку") // only the DRAFT decl form changed (dst unchanged, status draft)
|
||||||
|
|
||||||
|
if computeMemoryVersion(a, false) != computeMemoryVersion(b, false) {
|
||||||
|
t.Error("gate OFF: a draft decl change must NOT move memoryVersion (D8 approved-only)")
|
||||||
|
}
|
||||||
|
if computeMemoryVersion(a, true) == computeMemoryVersion(b, true) {
|
||||||
|
t.Error("gate ON: a draft decl change MUST move memoryVersion (else a resume silently flips the chunk)")
|
||||||
|
}
|
||||||
|
// Toggling the gate itself also changes the version (a mode change is a loud re-pin).
|
||||||
|
if computeMemoryVersion(a, false) == computeMemoryVersion(a, true) {
|
||||||
|
t.Error("toggling the post-check gate must change memoryVersion")
|
||||||
|
}
|
||||||
|
}
|
||||||
372
backend/internal/pipeline/memory_test.go
Normal file
372
backend/internal/pipeline/memory_test.go
Normal file
|
|
@ -0,0 +1,372 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"textmachine/backend/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// memory_test.go ports the executable spec eval/memory_hotpath.py (T1–T10) to Go
|
||||||
|
// unit tests, per the onboarding contract — but NOT the accept-REGEXPS verbatim
|
||||||
|
// (research/14: a \b-regexp is the 18–36% false-flag source). The selection tests
|
||||||
|
// (T1–T8) are the same cases; the post-check tests (T9/T10) use the decl-aware
|
||||||
|
// matcher (memory_e1_test.go measures its false-flag rate on real Russian).
|
||||||
|
|
||||||
|
func gl(src, dst, sense, status string) store.GlossaryEntry {
|
||||||
|
return store.GlossaryEntry{BookID: "b", Src: src, Dst: dst, Sense: sense, Status: status, Source: "seed"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func declJSON(invariant bool, forms ...string) string {
|
||||||
|
b, _ := json.Marshal(declInfo{Invariant: invariant, Forms: forms})
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func alias(a string) store.GlossaryAlias { return store.GlossaryAlias{Alias: a, AliasType: "прозвище"} }
|
||||||
|
|
||||||
|
// specGlossary mirrors eval/memory_hotpath.py's G (with decl forms replacing the
|
||||||
|
// toy accept-regexps).
|
||||||
|
func specGlossary() []store.GlossaryEntry {
|
||||||
|
ahq := gl("阿Q", "А-кью", "name", "approved")
|
||||||
|
ahq.Decl = declJSON(true, "А-кью", "А кью")
|
||||||
|
|
||||||
|
sishu := gl("四叔", "Четвёртый дядюшка", "char", "approved")
|
||||||
|
sishu.Aliases = []store.GlossaryAlias{alias("鲁四老爷")}
|
||||||
|
sishu.Decl = declJSON(false, "дядюшка", "дядюшки", "дядюшке", "дядюшку", "дядюшкой")
|
||||||
|
|
||||||
|
luzhen := gl("鲁镇", "Лучжэнь", "", "approved")
|
||||||
|
luzhen.Decl = declJSON(false, "Лучжэнь", "Лучжэня", "Лучжэне")
|
||||||
|
|
||||||
|
songzao := gl("送灶", "проводы бога очага", "festival", "approved")
|
||||||
|
songzao.Decl = declJSON(true, "бога очага")
|
||||||
|
|
||||||
|
xiaoyan := gl("萧炎", "Сяо Янь", "name", "approved")
|
||||||
|
xiaoyan.Aliases = []store.GlossaryAlias{alias("炎哥")}
|
||||||
|
xiaoyan.Decl = declJSON(true, "Сяо Янь")
|
||||||
|
|
||||||
|
xiuwei := gl("修为", "уровень совершенствования", "cultivation", "approved")
|
||||||
|
xiuwei.Decl = declJSON(false, "совершенствования", "совершенствование")
|
||||||
|
|
||||||
|
shadow := gl("影卫", "Тёмный страж", "spoiler", "approved")
|
||||||
|
shadow.SinceCh = 200
|
||||||
|
shadow.Decl = declJSON(false, "Тёмный страж", "Тёмного стража")
|
||||||
|
|
||||||
|
newname := gl("小D", "Малыш Дэ", "", "auto")
|
||||||
|
newname.Decl = declJSON(false, "Малыш Дэ", "Малыша Дэ")
|
||||||
|
|
||||||
|
return []store.GlossaryEntry{ahq, sishu, luzhen, songzao, xiaoyan, xiuwei, shadow, newname}
|
||||||
|
}
|
||||||
|
|
||||||
|
func bankFrom(entries []store.GlossaryEntry) *MemoryBank { return materializeMemory(entries, false) }
|
||||||
|
|
||||||
|
// injMap maps injected src → disposition for assertions.
|
||||||
|
func injMap(sel memorySelection) map[string]injectionDisposition {
|
||||||
|
m := map[string]injectionDisposition{}
|
||||||
|
for _, p := range sel.injected {
|
||||||
|
m[p.entry.src] = p.disp
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHotPathSpecT1toT10(t *testing.T) {
|
||||||
|
b := bankFrom(specGlossary())
|
||||||
|
const budget = 0
|
||||||
|
|
||||||
|
// T1: exact + alias select; alias 鲁四老爷 pulls 四叔.
|
||||||
|
sel := b.Select("阿Q走进鲁镇，遇见鲁四老爷。", 1, nil, budget)
|
||||||
|
m := injMap(sel)
|
||||||
|
if m["阿Q"] != memConfirmed || m["鲁镇"] != memConfirmed || m["四叔"] != memConfirmed {
|
||||||
|
t.Errorf("T1 exact+alias: %v", m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// T2: 送到灶 (literal) — 送灶 must NOT inject (bigram absent, 灶 single-key banned).
|
||||||
|
sel = b.Select("厨房里，老妈子把饭菜送到灶间去热一热。", 1, nil, budget)
|
||||||
|
if _, ok := injMap(sel)["送灶"]; ok {
|
||||||
|
t.Error("T2 wrong-sense 送灶 injected (A3 homograph)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// T3: 炎势 (flame) — 萧炎 must NOT inject.
|
||||||
|
sel = b.Select("山火蔓延，炎势冲天，村民连夜逃走。", 1, nil, budget)
|
||||||
|
if _, ok := injMap(sel)["萧炎"]; ok {
|
||||||
|
t.Error("T3 wrong-sense 炎 injected")
|
||||||
|
}
|
||||||
|
|
||||||
|
// T4: 修好 (repair) — 修为 must NOT inject.
|
||||||
|
sel = b.Select("工匠把石桥修好了。", 1, nil, budget)
|
||||||
|
if _, ok := injMap(sel)["修为"]; ok {
|
||||||
|
t.Error("T4 wrong-sense 修 injected")
|
||||||
|
}
|
||||||
|
|
||||||
|
// T5: trad→simp normalization — 魯鎮 matches key 鲁镇 (A4).
|
||||||
|
sel = b.Select("魯鎮的冬天很冷。", 1, nil, budget)
|
||||||
|
if _, ok := injMap(sel)["鲁镇"]; !ok {
|
||||||
|
t.Error("T5 trad→simp: 魯鎮 did not match 鲁镇")
|
||||||
|
}
|
||||||
|
|
||||||
|
// T6: pronominal chunk — sticky carries 四叔 from the previous chunk.
|
||||||
|
sishuID := "四叔\x1fchar\x1f0\x1f0"
|
||||||
|
sel = b.Select("他慢慢站起来，叹了口气。", 2, map[string]bool{sishuID: true}, budget)
|
||||||
|
var stickySishu *pickedEntry
|
||||||
|
for i := range sel.injected {
|
||||||
|
if sel.injected[i].entry.src == "四叔" {
|
||||||
|
stickySishu = &sel.injected[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if stickySishu == nil || stickySishu.via != "sticky" {
|
||||||
|
t.Errorf("T6 sticky scene-inertia failed: %+v", injMap(sel))
|
||||||
|
}
|
||||||
|
|
||||||
|
// T7: spoiler — 影卫 (since_ch=200) in chapter 5 → hard reject.
|
||||||
|
sel = b.Select("影卫出现了。", 5, nil, budget)
|
||||||
|
if _, ok := injMap(sel)["影卫"]; ok {
|
||||||
|
t.Error("T7 spoiler 影卫 injected in ch5")
|
||||||
|
}
|
||||||
|
if len(sel.rejected) != 1 || sel.rejected[0].entry.src != "影卫" {
|
||||||
|
t.Errorf("T7 spoiler not rejected+logged: %+v", sel.rejected)
|
||||||
|
}
|
||||||
|
// T7b: same entry passes after the reveal chapter.
|
||||||
|
sel = b.Select("影卫出现了。", 250, nil, budget)
|
||||||
|
if _, ok := injMap(sel)["影卫"]; !ok {
|
||||||
|
t.Error("T7b spoiler entry did not pass at ch250")
|
||||||
|
}
|
||||||
|
|
||||||
|
// T8: auto term 小D → ambiguous disposition.
|
||||||
|
sel = b.Select("小D也来了。", 1, nil, budget)
|
||||||
|
if injMap(sel)["小D"] != memAmbiguous {
|
||||||
|
t.Errorf("T8 auto-term not ambiguous: %v", injMap(sel))
|
||||||
|
}
|
||||||
|
|
||||||
|
// T9: post-check catches a missing dst (leak) and passes a present one.
|
||||||
|
sel = b.Select("阿Q走进鲁镇。", 1, nil, budget)
|
||||||
|
bad := b.postcheck(sel.injected, "Некто вошёл в Лучжэнь.") // А-кью dropped
|
||||||
|
good := b.postcheck(sel.injected, "А-кью вошёл в Лучжэнь.")
|
||||||
|
if !hasMiss(bad, "阿Q") {
|
||||||
|
t.Errorf("T9 post-check missed the leak: %+v", bad)
|
||||||
|
}
|
||||||
|
if len(good) != 0 {
|
||||||
|
t.Errorf("T9 post-check false-flagged a correct output: %+v", good)
|
||||||
|
}
|
||||||
|
|
||||||
|
// T10: post-check catches poisoning — injected dst absent, output used a foreign name.
|
||||||
|
sel = b.Select("阿Q走进鲁镇。", 1, nil, budget)
|
||||||
|
poison := b.postcheck(sel.injected, "Линь Чун вошёл в деревню Вэйчжуан.")
|
||||||
|
if !hasMiss(poison, "阿Q") {
|
||||||
|
t.Errorf("T10 post-check missed the poisoning: %+v", poison)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasMiss(misses []postcheckMiss, src string) bool {
|
||||||
|
for _, m := range misses {
|
||||||
|
if m.Src == src {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTrapCorpus ports the retrieval_bench wrong-sense traps: exact multi-char match
|
||||||
|
// must fire 0/N on homograph chunks that contain the KEY CHARACTER but not the term.
|
||||||
|
func TestTrapCorpus(t *testing.T) {
|
||||||
|
entries := []store.GlossaryEntry{
|
||||||
|
gl("送灶", "проводы бога очага", "festival", "approved"),
|
||||||
|
gl("修为", "уровень совершенствования", "cultivation", "approved"),
|
||||||
|
gl("萧炎", "Сяо Янь", "name", "approved"),
|
||||||
|
gl("斗气", "боевая ци", "term", "approved"),
|
||||||
|
gl("钱老爷", "господин Цянь", "name", "approved"), // multi-char; the bare 钱 homograph must not fire
|
||||||
|
}
|
||||||
|
b := bankFrom(entries)
|
||||||
|
traps := []struct{ name, text string }{
|
||||||
|
{"送到灶 literal", "厨房里冷了，老妈子把剩下的饭菜送到灶间去重新热一热。"},
|
||||||
|
{"修好 repair", "工匠们花了三天，终于把那条被山洪冲垮的石桥修好了。"},
|
||||||
|
{"炎势 flame", "山火蔓延，炎势冲天，浓烟滚滚，村民连夜收拾细软。"},
|
||||||
|
{"雾气 mist", "清晨的山谷里雾气很重，空气格外清新，露水打湿了草叶。"},
|
||||||
|
{"钱 money homograph", "他摸出几个铜钱，买了一碗热汤。"}, // 钱 present but 钱老爷 absent
|
||||||
|
{"Chekhov negative", "Дмитрий Гуров сидел в ялтинском павильоне и лениво пил чай."},
|
||||||
|
}
|
||||||
|
for _, tr := range traps {
|
||||||
|
t.Run(tr.name, func(t *testing.T) {
|
||||||
|
sel := b.Select(tr.text, 1, nil, 0)
|
||||||
|
if len(sel.injected) != 0 {
|
||||||
|
t.Errorf("trap fired %d injection(s), want 0: %v", len(sel.injected), injMap(sel))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLongestMatchContainment: a shorter key fully inside a longer key's span is
|
||||||
|
// suppressed (longest-match / whole-entity replacement, A3), but the shorter entity
|
||||||
|
// still fires when it stands alone.
|
||||||
|
func TestLongestMatchContainment(t *testing.T) {
|
||||||
|
entries := []store.GlossaryEntry{
|
||||||
|
gl("阿Q", "А-кью", "name", "approved"),
|
||||||
|
gl("阿Q正传", "«Подлинная история А-кью»", "title", "approved"),
|
||||||
|
}
|
||||||
|
b := bankFrom(entries)
|
||||||
|
// Inside the title: only the title fires.
|
||||||
|
sel := b.Select("我读了《阿Q正传》。", 1, nil, 0)
|
||||||
|
m := injMap(sel)
|
||||||
|
if _, ok := m["阿Q正传"]; !ok {
|
||||||
|
t.Errorf("title did not fire: %v", m)
|
||||||
|
}
|
||||||
|
if _, ok := m["阿Q"]; ok {
|
||||||
|
t.Errorf("short key 阿Q fired inside the longer title span: %v", m)
|
||||||
|
}
|
||||||
|
// Standalone: the character name fires.
|
||||||
|
sel = b.Select("阿Q走进村子。", 1, nil, 0)
|
||||||
|
if _, ok := injMap(sel)["阿Q"]; !ok {
|
||||||
|
t.Error("阿Q did not fire standalone")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSingleKeyBanAndAllowShort(t *testing.T) {
|
||||||
|
banned := gl("炎", "пламя-как-имя", "", "approved") // single Han → banned by default
|
||||||
|
b := bankFrom([]store.GlossaryEntry{banned})
|
||||||
|
if sel := b.Select("炎势冲天。", 1, nil, 0); len(sel.injected) != 0 {
|
||||||
|
t.Error("single-char key fired despite the A3 ban")
|
||||||
|
}
|
||||||
|
// allow_short escape hatch lets it fire.
|
||||||
|
banned.AllowShort = true
|
||||||
|
b = bankFrom([]store.GlossaryEntry{banned})
|
||||||
|
if sel := b.Select("炎势冲天。", 1, nil, 0); len(sel.injected) != 1 {
|
||||||
|
t.Error("allow_short did not re-enable the single-char key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBudgetEviction(t *testing.T) {
|
||||||
|
entries := []store.GlossaryEntry{
|
||||||
|
gl("甲甲", "Альфа", "", "approved"),
|
||||||
|
gl("乙乙", "Бета", "", "auto"), // ambiguous → lower priority
|
||||||
|
gl("丙丙", "Гамма", "", "approved"),
|
||||||
|
}
|
||||||
|
b := bankFrom(entries)
|
||||||
|
chunk := "甲甲、乙乙、丙丙都在。"
|
||||||
|
// Unbounded: all three fire, in priority order (the two approved before the auto).
|
||||||
|
all := b.Select(chunk, 1, nil, 0)
|
||||||
|
if len(all.injected) != 3 {
|
||||||
|
t.Fatalf("unbounded: want 3 injected, got %d", len(all.injected))
|
||||||
|
}
|
||||||
|
// A token budget that fits exactly the top TWO priority records (computed with the
|
||||||
|
// same cost helper Select uses, so the test never drifts from the implementation).
|
||||||
|
budget := glossaryLineTokens(all.injected[0].entry) + glossaryLineTokens(all.injected[1].entry)
|
||||||
|
sel := b.Select(chunk, 1, nil, budget)
|
||||||
|
if len(sel.injected) != 2 || len(sel.evicted) != 1 {
|
||||||
|
t.Fatalf("budget: injected=%d evicted=%d (budget=%d)", len(sel.injected), len(sel.evicted), budget)
|
||||||
|
}
|
||||||
|
// The auto (ambiguous) term is the one evicted (confirmed>ambiguous priority, F2).
|
||||||
|
if sel.evicted[0].entry.src != "乙乙" {
|
||||||
|
t.Errorf("eviction priority wrong: evicted %s (want the auto term 乙乙)", sel.evicted[0].entry.src)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSelectDeterminism runs Select many times over a multi-hit chunk and asserts the
|
||||||
|
// injected order is byte-identical — guards against a map-iteration leak (the
|
||||||
|
// determinism invariant: the injected bytes enter request_hash).
|
||||||
|
func TestSelectDeterminism(t *testing.T) {
|
||||||
|
b := bankFrom(specGlossary())
|
||||||
|
chunk := "阿Q走进鲁镇，遇见鲁四老爷，谈起修为。"
|
||||||
|
first := selKey(b.Select(chunk, 1, nil, 0))
|
||||||
|
for i := 0; i < 200; i++ {
|
||||||
|
if got := selKey(b.Select(chunk, 1, nil, 0)); got != first {
|
||||||
|
t.Fatalf("non-deterministic Select: run %d gave %q, first was %q", i, got, first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func selKey(sel memorySelection) string {
|
||||||
|
s := ""
|
||||||
|
for _, p := range sel.injected {
|
||||||
|
s += p.entry.src + ":" + string(p.disp) + "|"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEmptyDstNotMatchable covers self-review #1: a ruby candidate (dst="") must not be
|
||||||
|
// matchable — it renders nothing, so admitting it would waste budget, evict a real line,
|
||||||
|
// and inflate the exact-hit count.
|
||||||
|
func TestEmptyDstNotMatchable(t *testing.T) {
|
||||||
|
phantom := gl("甲甲", "", "", "auto") // ruby-style candidate, no dst
|
||||||
|
real := gl("乙乙", "Настоящий", "", "auto")
|
||||||
|
b := bankFrom([]store.GlossaryEntry{phantom, real})
|
||||||
|
// Tiny budget that would fit exactly one line: the phantom must not steal it.
|
||||||
|
budget := glossaryLineTokens(&memoryEntry{src: "乙乙", dst: "Настоящий"})
|
||||||
|
sel := b.Select("甲甲和乙乙都在场。", 1, nil, budget)
|
||||||
|
if len(sel.injected) != 1 || sel.injected[0].entry.src != "乙乙" {
|
||||||
|
t.Fatalf("empty-dst phantom polluted selection: injected=%v", injMap(sel))
|
||||||
|
}
|
||||||
|
if renderGlossaryBlock(sel.injected) == "" {
|
||||||
|
t.Error("a renderable line was evicted by an empty-dst phantom")
|
||||||
|
}
|
||||||
|
// Unbounded: the phantom never appears as an exact hit.
|
||||||
|
sel = b.Select("甲甲和乙乙都在场。", 1, nil, 0)
|
||||||
|
if _, ok := injMap(sel)["甲甲"]; ok {
|
||||||
|
t.Error("empty-dst phantom counted as an exact hit")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSpoilerBeforeSuppression covers self-review #6: a spoiler-blocked LONGER key must
|
||||||
|
// not suppress a valid SHORTER name nested inside it.
|
||||||
|
func TestSpoilerBeforeSuppression(t *testing.T) {
|
||||||
|
short := gl("林动", "Линь Дун", "", "approved")
|
||||||
|
long := gl("林动的父亲", "отец Линь Дуна", "", "approved")
|
||||||
|
long.UntilCh = 3 // a spoiler: valid only through chapter 3
|
||||||
|
b := bankFrom([]store.GlossaryEntry{short, long})
|
||||||
|
|
||||||
|
// Chapter 10: the longer key is spoiler-blocked; the shorter valid name must survive.
|
||||||
|
sel := b.Select("林动的父亲缓缓转身。", 10, nil, 0)
|
||||||
|
if _, ok := injMap(sel)["林动"]; !ok {
|
||||||
|
t.Errorf("valid shorter name dropped by a spoiler-blocked longer key: %v", injMap(sel))
|
||||||
|
}
|
||||||
|
if _, ok := injMap(sel)["林动的父亲"]; ok {
|
||||||
|
t.Error("spoiler-blocked longer key was injected")
|
||||||
|
}
|
||||||
|
if len(sel.rejected) != 1 || sel.rejected[0].entry.src != "林动的父亲" {
|
||||||
|
t.Errorf("spoiler-blocked longer key not logged as rejected: %+v", sel.rejected)
|
||||||
|
}
|
||||||
|
// Chapter 2 (longer key in window): longest-match still prefers the longer entity.
|
||||||
|
sel = b.Select("林动的父亲缓缓转身。", 2, nil, 0)
|
||||||
|
if _, ok := injMap(sel)["林动"]; ok {
|
||||||
|
t.Error("longest-match failed when the longer key is spoiler-valid")
|
||||||
|
}
|
||||||
|
if _, ok := injMap(sel)["林动的父亲"]; !ok {
|
||||||
|
t.Error("spoiler-valid longer key was not injected at ch2")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBudgetFloorFree covers self-review #7: the per-line budget cost is floor-free, so a
|
||||||
|
// block of short name lines is not ~1.85×-overcounted and needlessly evicted.
|
||||||
|
func TestBudgetFloorFree(t *testing.T) {
|
||||||
|
var entries []store.GlossaryEntry
|
||||||
|
srcs := []string{"甲甲", "乙乙", "丙丙", "丁丁", "戊戊", "己己", "庚庚", "辛辛"}
|
||||||
|
dsts := []string{"Ко", "Оцу", "Хэй", "Тэй", "Бо", "Ки", "Ко2", "Син"}
|
||||||
|
for i := range srcs {
|
||||||
|
entries = append(entries, gl(srcs[i], dsts[i], "", "approved"))
|
||||||
|
}
|
||||||
|
b := bankFrom(entries)
|
||||||
|
chunk := strings.Join(srcs, "、") + "都在场。"
|
||||||
|
all := b.Select(chunk, 1, nil, 0)
|
||||||
|
if len(all.injected) != 8 {
|
||||||
|
t.Fatalf("want 8 injected, got %d", len(all.injected))
|
||||||
|
}
|
||||||
|
// A budget equal to the EXACT sum of the (floor-free) line costs fits all 8.
|
||||||
|
budget := 0
|
||||||
|
for _, p := range all.injected {
|
||||||
|
budget += glossaryLineTokens(p.entry)
|
||||||
|
}
|
||||||
|
sel := b.Select(chunk, 1, nil, budget)
|
||||||
|
if len(sel.injected) != 8 || len(sel.evicted) != 0 {
|
||||||
|
t.Errorf("floor-free budget=%d should fit all 8: injected=%d evicted=%d", budget, len(sel.injected), len(sel.evicted))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInjectivityCollision(t *testing.T) {
|
||||||
|
entries := []store.GlossaryEntry{
|
||||||
|
{BookID: "b", Src: "甲", Dst: "Мастер", Status: "approved"},
|
||||||
|
{BookID: "b", Src: "乙", Dst: "Мастер", Status: "approved"}, // same dst → collision (B2)
|
||||||
|
{BookID: "b", Src: "丙", Dst: "Ученик", Status: "approved"},
|
||||||
|
}
|
||||||
|
cols := injectivityCollisions(entries)
|
||||||
|
if len(cols) != 1 {
|
||||||
|
t.Fatalf("want 1 dst collision, got %d: %v", len(cols), cols)
|
||||||
|
}
|
||||||
|
}
|
||||||
109
backend/internal/pipeline/mempostcheck.go
Normal file
109
backend/internal/pipeline/mempostcheck.go
Normal file
|
|
@ -0,0 +1,109 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mempostcheck.go: the post-check (registry E1/E2 / research/13 Q5) — the main, almost
|
||||||
|
// free detector of silent degradation. For every injected record whose SRC actually
|
||||||
|
// fired in THIS chunk, it asks: did an accepted DST form appear in the output? A miss
|
||||||
|
// catches BOTH failure modes — "the model ignored the glossary" (soft-following loses
|
||||||
|
// 17–36% of terms, 2310.05824) AND "we injected the wrong dst and the model obeyed it"
|
||||||
|
// (the owner's fear, 2510.00829). It is a post-CHECK, never a blind post-REPLACE (E2):
|
||||||
|
// forcing a dictionary form into an oblique Russian case breaks agreement (46% of
|
||||||
|
// constrained-model errors in en-cs are agreement, 2106.12398), so v1 only FLAGS.
|
||||||
|
//
|
||||||
|
// Decl-awareness is load-bearing (research/14 §2, quantified): a naive \b-regexp on the
|
||||||
|
// base form gives 18–36% FALSE flags on a hard chunk — inflected forms rendered
|
||||||
|
// CORRECTLY ("Вэйчжуане", "Бородатого Вана", "Дэна", "сюцая") that the boundary regexp
|
||||||
|
// misses. So the check matches against the STORED decl forms (filled at term-commit),
|
||||||
|
// not the bare lemma. BUT stored-decl reliability is itself a function of decl
|
||||||
|
// COMPLETENESS — an OOV translit name whose forms were under-filled re-introduces the
|
||||||
|
// same false flags. That is why v1 keeps the post-check a FLAGGER (observable, in the
|
||||||
|
// retrieval-state), NOT a hard disposition gate, until the false-flag rate is MEASURED
|
||||||
|
// on real inflected Russian (E1 — memory_e1_test.go). A hard gate flips on only after
|
||||||
|
// the owner validates precision (mirrors the coverage gate's opt-in, D12 Q4).
|
||||||
|
//
|
||||||
|
// Sticky records are DELIBERATELY excluded: a sticky entry's src is NOT in this chunk
|
||||||
|
// (that is why scene-inertia carries it), so its dst is legitimately absent from the
|
||||||
|
// output — post-checking it would false-flag every pronominal chunk. Only records whose
|
||||||
|
// key exactly fired here are expected in the output.
|
||||||
|
|
||||||
|
// postcheckMiss is one flagged term: an injected record whose src fired but whose dst
|
||||||
|
// (any accepted form) is absent from the output.
|
||||||
|
type postcheckMiss struct {
|
||||||
|
Src string `json:"src"`
|
||||||
|
Dst string `json:"dst"`
|
||||||
|
Disp string `json:"disp"` // confirmed | ambiguous (A2: ambiguous injections force a post-check)
|
||||||
|
}
|
||||||
|
|
||||||
|
// postcheck runs the decl-aware check over the injected records against the model
|
||||||
|
// output. Pure and deterministic. Returns the misses (empty = all fired terms present).
|
||||||
|
func (b *MemoryBank) postcheck(injected []pickedEntry, output string) []postcheckMiss {
|
||||||
|
nout := normalizeTargetForm(output)
|
||||||
|
var misses []postcheckMiss
|
||||||
|
noutRunes := []rune(nout)
|
||||||
|
for _, p := range injected {
|
||||||
|
if p.via == "sticky" { // sticky context is not expected in the output
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(p.entry.dst) == "" { // a ruby candidate with no dst yet — nothing to check
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !dstFormPresent(p.entry, noutRunes) {
|
||||||
|
misses = append(misses, postcheckMiss{Src: p.entry.src, Dst: p.entry.dst, Disp: string(p.disp)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return misses
|
||||||
|
}
|
||||||
|
|
||||||
|
// dstFormPresent reports whether any accepted dst form of the entry appears in the
|
||||||
|
// normalized output as a WHOLE WORD (bounded by non-letters or string edges), not a bare
|
||||||
|
// substring. The whole-word rule closes self-review #3: a DROPPED short translit name
|
||||||
|
// («Ван») must not be masked by an unrelated common word that merely contains its letters
|
||||||
|
// («караВАН», «диВАН», «ИВАН» — a different person). Decl-aware (the research/14
|
||||||
|
// requirement): the stored decl forms carry the inflections, so precision and recall are
|
||||||
|
// both satisfied when decl is complete; the base-dst fallback (empty decl) is the naive
|
||||||
|
// case the E1 measurement quantifies. noutRunes is the normalized output pre-decomposed.
|
||||||
|
func dstFormPresent(e *memoryEntry, noutRunes []rune) bool {
|
||||||
|
forms := e.declForms
|
||||||
|
if len(forms) == 0 {
|
||||||
|
forms = []string{normalizeTargetForm(e.dst)}
|
||||||
|
}
|
||||||
|
for _, f := range forms {
|
||||||
|
if f != "" && containsWholeWord(noutRunes, []rune(f)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// containsWholeWord reports whether form occurs in text with a letter-boundary on both
|
||||||
|
// ends (the char before the match is not a letter or is the start; likewise after). Runs
|
||||||
|
// on runes so Cyrillic boundaries are correct. O(len(text)·len(form)) — fine, the
|
||||||
|
// post-check runs once per chunk over a handful of short forms.
|
||||||
|
func containsWholeWord(text, form []rune) bool {
|
||||||
|
n := len(form)
|
||||||
|
if n == 0 || n > len(text) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := 0; i+n <= len(text); i++ {
|
||||||
|
if !runesEqual(text[i:i+n], form) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (i == 0 || !unicode.IsLetter(text[i-1])) && (i+n == len(text) || !unicode.IsLetter(text[i+n])) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func runesEqual(a, b []rune) bool {
|
||||||
|
for i := range a {
|
||||||
|
if a[i] != b[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
253
backend/internal/pipeline/memseed.go
Normal file
253
backend/internal/pipeline/memseed.go
Normal file
|
|
@ -0,0 +1,253 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
|
"textmachine/backend/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// memseed.go: the DETERMINISTIC seeding of the glossary from its two milestone inputs —
|
||||||
|
// a manual seed file (curated approved/draft terms) and the captured ruby readings
|
||||||
|
// (classified into auto candidates). No LLM (autopopulation/adjudication are a later
|
||||||
|
// milestone). The output is a []store.GlossaryEntry that seedGlossary REPLACES into the
|
||||||
|
// book (idempotent full-replace: same inputs → same rows, an edit converges every column).
|
||||||
|
|
||||||
|
// --- manual seed file (YAML) ----------------------------------------------------
|
||||||
|
|
||||||
|
type seedFile struct {
|
||||||
|
Terms []seedTerm `yaml:"terms"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type seedTerm struct {
|
||||||
|
Src string `yaml:"src"`
|
||||||
|
Dst string `yaml:"dst"`
|
||||||
|
Type string `yaml:"type"`
|
||||||
|
Sense string `yaml:"sense"`
|
||||||
|
Gender string `yaml:"gender"`
|
||||||
|
Speech string `yaml:"speech"`
|
||||||
|
Decl *seedDecl `yaml:"decl"`
|
||||||
|
TranslitPolicy string `yaml:"translit_policy"`
|
||||||
|
FirstPerson string `yaml:"first_person"`
|
||||||
|
NicknameTranslation string `yaml:"nickname_translation"`
|
||||||
|
SinceCh int `yaml:"since_ch"`
|
||||||
|
UntilCh int `yaml:"until_ch"`
|
||||||
|
Status string `yaml:"status"`
|
||||||
|
AllowShort bool `yaml:"allow_short"`
|
||||||
|
Note string `yaml:"note"`
|
||||||
|
Aliases []seedAlias `yaml:"aliases"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type seedDecl struct {
|
||||||
|
Invariant bool `yaml:"invariant"`
|
||||||
|
Forms []string `yaml:"forms"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type seedAlias struct {
|
||||||
|
Alias string `yaml:"alias"`
|
||||||
|
Type string `yaml:"type"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadGlossarySeed parses a glossary seed YAML into store entries. A manual seed is
|
||||||
|
// curated, so an empty status defaults to "approved" (the author downgrades explicitly
|
||||||
|
// with status: draft|auto). Validation is fail-loud: a term without src, or with an
|
||||||
|
// unknown status, is a config error (a silently-dropped term is exactly the A-class
|
||||||
|
// hole this bank exists to close).
|
||||||
|
func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("pipeline: read glossary seed %s: %w", path, err)
|
||||||
|
}
|
||||||
|
var sf seedFile
|
||||||
|
if err := yaml.Unmarshal(raw, &sf); err != nil {
|
||||||
|
return nil, fmt.Errorf("pipeline: parse glossary seed %s: %w", path, err)
|
||||||
|
}
|
||||||
|
var out []store.GlossaryEntry
|
||||||
|
var problems []string
|
||||||
|
termKeys := map[[4]string]bool{} // (src,sense,since,until) → detect a duplicate term key
|
||||||
|
for i, t := range sf.Terms {
|
||||||
|
if strings.TrimSpace(t.Src) == "" {
|
||||||
|
problems = append(problems, fmt.Sprintf("term %d: src is required", i))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// A duplicate (src, sense, window) would crash ReplaceGlossary on the store's
|
||||||
|
// UNIQUE constraint mid-run (aborting the whole book) — catch it here as a clear
|
||||||
|
// config error instead (self-review #5, sibling of the alias case below).
|
||||||
|
tk := [4]string{t.Src, t.Sense, fmt.Sprint(t.SinceCh), fmt.Sprint(t.UntilCh)}
|
||||||
|
if termKeys[tk] {
|
||||||
|
problems = append(problems, fmt.Sprintf("term %q: duplicate (src, sense=%q, since_ch=%d, until_ch=%d) — a term may appear once per spoiler window", t.Src, t.Sense, t.SinceCh, t.UntilCh))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
termKeys[tk] = true
|
||||||
|
status := t.Status
|
||||||
|
if status == "" {
|
||||||
|
status = "approved"
|
||||||
|
}
|
||||||
|
switch status {
|
||||||
|
case "auto", "draft", "approved":
|
||||||
|
default:
|
||||||
|
problems = append(problems, fmt.Sprintf("term %q: status must be auto|draft|approved, got %q", t.Src, status))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
decl := ""
|
||||||
|
if t.Decl != nil {
|
||||||
|
b, mErr := json.Marshal(declInfo{Invariant: t.Decl.Invariant, Forms: t.Decl.Forms})
|
||||||
|
if mErr != nil {
|
||||||
|
return nil, mErr
|
||||||
|
}
|
||||||
|
decl = string(b)
|
||||||
|
}
|
||||||
|
e := store.GlossaryEntry{
|
||||||
|
BookID: "", Src: t.Src, Dst: t.Dst, Type: t.Type, Sense: t.Sense,
|
||||||
|
Gender: t.Gender, Speech: t.Speech, Decl: decl,
|
||||||
|
TranslitPolicy: t.TranslitPolicy, FirstPerson: t.FirstPerson,
|
||||||
|
NicknameTranslation: t.NicknameTranslation,
|
||||||
|
SinceCh: t.SinceCh, UntilCh: t.UntilCh, Status: status,
|
||||||
|
AllowShort: t.AllowShort, Source: "seed", Note: t.Note,
|
||||||
|
}
|
||||||
|
// Dedup aliases within a term by surface (keep the first type). A repeated alias
|
||||||
|
// surface — a copy-paste, or the same name annotated with two types — would
|
||||||
|
// otherwise crash ReplaceGlossary on UNIQUE(book_id,term_id,alias) and abort the
|
||||||
|
// whole book run (self-review #5). Deduping is the forgiving fix: the surface is
|
||||||
|
// what matters for matching, and a redundant alias is a harmless authoring slip.
|
||||||
|
seenAlias := map[string]bool{}
|
||||||
|
for _, a := range t.Aliases {
|
||||||
|
if strings.TrimSpace(a.Alias) == "" {
|
||||||
|
problems = append(problems, fmt.Sprintf("term %q: an alias is empty", t.Src))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if seenAlias[a.Alias] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seenAlias[a.Alias] = true
|
||||||
|
e.Aliases = append(e.Aliases, store.GlossaryAlias{Alias: a.Alias, AliasType: a.Type})
|
||||||
|
}
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return nil, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- ruby → auto candidates -----------------------------------------------------
|
||||||
|
|
||||||
|
// rubyToCandidates classifies the captured ruby readings into auto glossary candidates
|
||||||
|
// (registry §Ruby-seed / §8). One candidate per BASE (the dominant reading by occurrence,
|
||||||
|
// ties broken lexically) so variant readings never collide on UNIQUE(src,sense,window)
|
||||||
|
// — the (base,reading) source of truth stays in ruby_readings. A base already covered by
|
||||||
|
// the manual seed is skipped (the curated entry wins). Every candidate is status=auto
|
||||||
|
// with NO dst: the reading is the Polivanov bridge (B6, Phase 2), not a dst — so nothing
|
||||||
|
// is auto-injected, and a human/batch-judge promotes it (never a blind name-lock, which
|
||||||
|
// would be a mistranslation). The ruby_class is a deterministic HINT for that promotion.
|
||||||
|
func rubyToCandidates(readings []store.RubyReading, manualSrcs map[string]bool) []store.GlossaryEntry {
|
||||||
|
// Group by base → the dominant (base,reading).
|
||||||
|
type best struct {
|
||||||
|
reading string
|
||||||
|
occ int
|
||||||
|
firstCh int
|
||||||
|
}
|
||||||
|
byBase := map[string]best{}
|
||||||
|
order := []string{}
|
||||||
|
for _, rr := range readings {
|
||||||
|
if manualSrcs[rr.Base] {
|
||||||
|
continue // the curated seed already owns this src
|
||||||
|
}
|
||||||
|
cur, ok := byBase[rr.Base]
|
||||||
|
if !ok {
|
||||||
|
byBase[rr.Base] = best{rr.Reading, rr.Occurrences, rr.FirstChapter}
|
||||||
|
order = append(order, rr.Base)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Dominant reading: higher occurrences wins; tie → lexically smaller reading.
|
||||||
|
if rr.Occurrences > cur.occ || (rr.Occurrences == cur.occ && rr.Reading < cur.reading) {
|
||||||
|
cur.reading, cur.occ = rr.Reading, rr.Occurrences
|
||||||
|
}
|
||||||
|
if rr.FirstChapter < cur.firstCh { // earliest appearance across readings
|
||||||
|
cur.firstCh = rr.FirstChapter
|
||||||
|
}
|
||||||
|
byBase[rr.Base] = cur
|
||||||
|
}
|
||||||
|
sort.Strings(order) // deterministic output regardless of input order
|
||||||
|
var out []store.GlossaryEntry
|
||||||
|
for _, base := range order {
|
||||||
|
b := byBase[base]
|
||||||
|
class := classifyRubyReading(base, b.reading)
|
||||||
|
typ := ""
|
||||||
|
if class == rubyClassName {
|
||||||
|
typ = "name"
|
||||||
|
}
|
||||||
|
out = append(out, store.GlossaryEntry{
|
||||||
|
Src: base, Dst: "", Type: typ, Status: "auto", Source: "ruby",
|
||||||
|
RubyReading: b.reading, RubyClass: class, SinceCh: b.firstCh,
|
||||||
|
Confidence: b.occ,
|
||||||
|
Note: "ruby auto-candidate (" + class + "); reading is the Polivanov bridge, not a dst — promote before use",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ruby classifier classes (deterministic HINTS for human promotion).
|
||||||
|
const (
|
||||||
|
rubyClassName = "name_candidate" // all-Han base + all-kana reading: the furigana-name SHAPE
|
||||||
|
rubyClassGloss = "gloss_candidate" // reading carries Han: likely an author double-reading → footnote, NOT a lock
|
||||||
|
rubyClassAmbiguous = "ambiguous" // anything else → flag to a human
|
||||||
|
)
|
||||||
|
|
||||||
|
// classifyRubyReading is the deterministic clear-case rule (§8). It NEVER auto-locks:
|
||||||
|
// the all-Han+all-kana shape is a name-lock CANDIDATE (it could still be a double-reading
|
||||||
|
// like 強敵→とも "friend", indistinguishable without a yomi dictionary — Phase 2), so it is
|
||||||
|
// flagged for human confirmation, not locked. A reading that carries kanji is a clear
|
||||||
|
// author's double-reading/gloss → a footnote candidate. Over-flagging is safe; a blind
|
||||||
|
// lock is a mistranslation. Pure and deterministic.
|
||||||
|
func classifyRubyReading(base, reading string) string {
|
||||||
|
if runesAnyHan(reading) {
|
||||||
|
return rubyClassGloss // the "reading" is itself a word (semantic gloss / double-reading)
|
||||||
|
}
|
||||||
|
if runesAllHan(base) && runesAllKana(reading) {
|
||||||
|
return rubyClassName
|
||||||
|
}
|
||||||
|
return rubyClassAmbiguous
|
||||||
|
}
|
||||||
|
|
||||||
|
func runesAllHan(s string) bool {
|
||||||
|
if s == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, r := range s {
|
||||||
|
if !unicode.Is(unicode.Han, r) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func runesAnyHan(s string) bool {
|
||||||
|
for _, r := range s {
|
||||||
|
if unicode.Is(unicode.Han, r) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func runesAllKana(s string) bool {
|
||||||
|
if s == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, r := range s {
|
||||||
|
if r == 'ー' || r == '・' { // prolonged-sound mark, middle dot — allowed within kana readings
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !unicode.Is(unicode.Hiragana, r) && !unicode.Is(unicode.Katakana, r) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
162
backend/internal/pipeline/memseed_test.go
Normal file
162
backend/internal/pipeline/memseed_test.go
Normal file
|
|
@ -0,0 +1,162 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"textmachine/backend/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeSeed(t *testing.T, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(t.TempDir(), "glossary.yaml")
|
||||||
|
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadGlossarySeed(t *testing.T) {
|
||||||
|
seed := writeSeed(t, `
|
||||||
|
terms:
|
||||||
|
- src: 阿Q
|
||||||
|
dst: А-кью
|
||||||
|
type: name
|
||||||
|
decl: { invariant: true, forms: ["А-кью"] }
|
||||||
|
aliases:
|
||||||
|
- { alias: 阿桂, type: прозвище }
|
||||||
|
- src: 王胡
|
||||||
|
dst: Бородатый Ван
|
||||||
|
status: draft
|
||||||
|
decl: { forms: ["Бородатый Ван", "Бородатого Вана"] }
|
||||||
|
`)
|
||||||
|
entries, err := loadGlossarySeed(seed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(entries) != 2 {
|
||||||
|
t.Fatalf("want 2 entries, got %d", len(entries))
|
||||||
|
}
|
||||||
|
// Empty status defaults to approved (a manual seed is curated).
|
||||||
|
if entries[0].Status != "approved" {
|
||||||
|
t.Errorf("default status = %q, want approved", entries[0].Status)
|
||||||
|
}
|
||||||
|
if entries[0].Source != "seed" {
|
||||||
|
t.Errorf("source = %q, want seed", entries[0].Source)
|
||||||
|
}
|
||||||
|
if len(entries[0].Aliases) != 1 || entries[0].Aliases[0].Alias != "阿桂" {
|
||||||
|
t.Errorf("alias not parsed: %+v", entries[0].Aliases)
|
||||||
|
}
|
||||||
|
if entries[0].Decl == "" {
|
||||||
|
t.Error("decl JSON not marshaled")
|
||||||
|
}
|
||||||
|
if entries[1].Status != "draft" {
|
||||||
|
t.Errorf("explicit status not honored: %q", entries[1].Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadGlossarySeedRejectsBad(t *testing.T) {
|
||||||
|
// Missing src → fail loud (a silently-dropped term is the A-class hole).
|
||||||
|
if _, err := loadGlossarySeed(writeSeed(t, "terms:\n - dst: без ключа\n")); err == nil {
|
||||||
|
t.Error("expected error for a term without src")
|
||||||
|
}
|
||||||
|
// Unknown status → fail loud.
|
||||||
|
if _, err := loadGlossarySeed(writeSeed(t, "terms:\n - src: 甲\n status: bogus\n")); err == nil {
|
||||||
|
t.Error("expected error for an unknown status")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSeedAliasDedupAndDuplicateTerm covers self-review #5: a duplicate alias within a
|
||||||
|
// term must be deduped (not crash the run on UNIQUE), and a duplicate (src,sense,window)
|
||||||
|
// term must fail loud (not crash mid-run).
|
||||||
|
func TestSeedAliasDedupAndDuplicateTerm(t *testing.T) {
|
||||||
|
// Duplicate alias surface (annotated with two types) → deduped, no error.
|
||||||
|
entries, err := loadGlossarySeed(writeSeed(t, `
|
||||||
|
terms:
|
||||||
|
- src: 四叔
|
||||||
|
dst: Четвёртый дядюшка
|
||||||
|
aliases:
|
||||||
|
- { alias: 鲁四老爷, type: имя }
|
||||||
|
- { alias: 鲁四老爷, type: титул }
|
||||||
|
`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("duplicate alias should be deduped, not error: %v", err)
|
||||||
|
}
|
||||||
|
if len(entries) != 1 || len(entries[0].Aliases) != 1 {
|
||||||
|
t.Fatalf("alias not deduped: %+v", entries)
|
||||||
|
}
|
||||||
|
// Duplicate (src, sense, window) → fail loud.
|
||||||
|
if _, err := loadGlossarySeed(writeSeed(t, `
|
||||||
|
terms:
|
||||||
|
- { src: 道, dst: путь }
|
||||||
|
- { src: 道, dst: дао }
|
||||||
|
`)); err == nil {
|
||||||
|
t.Error("duplicate (src,sense,window) term must fail loud")
|
||||||
|
}
|
||||||
|
// Same src, different SENSE → allowed (polysemy).
|
||||||
|
if _, err := loadGlossarySeed(writeSeed(t, `
|
||||||
|
terms:
|
||||||
|
- { src: 道, dst: путь, sense: way }
|
||||||
|
- { src: 道, dst: дао, sense: dao }
|
||||||
|
`)); err != nil {
|
||||||
|
t.Errorf("distinct senses should be allowed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClassifyRubyReading(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
base, reading, want string
|
||||||
|
}{
|
||||||
|
{"鈴木", "すずき", rubyClassName}, // all-Han base + all-kana reading → name candidate
|
||||||
|
{"強敵", "とも", rubyClassName}, // SAME shape (a double-reading!) — deliberately NOT distinguishable offline → still a CANDIDATE, never auto-locked
|
||||||
|
{"強敵", "きょうてき", rubyClassName}, // real reading, same class
|
||||||
|
{"泣蟲", "泣き虫", rubyClassGloss}, // reading carries kanji → a semantic gloss/double-reading → footnote, not a lock
|
||||||
|
{"ゲーム", "gēmu", rubyClassAmbiguous}, // base is kana → not a plain furigana-name
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := classifyRubyReading(c.base, c.reading); got != c.want {
|
||||||
|
t.Errorf("classifyRubyReading(%q,%q) = %q; want %q", c.base, c.reading, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRubyToCandidates(t *testing.T) {
|
||||||
|
readings := []store.RubyReading{
|
||||||
|
{BookID: "b", Base: "鈴木", Reading: "すずき", FirstChapter: 2, Occurrences: 9},
|
||||||
|
{BookID: "b", Base: "鈴木", Reading: "スズキ", FirstChapter: 5, Occurrences: 3}, // variant → deduped by base
|
||||||
|
{BookID: "b", Base: "田中", Reading: "たなか", FirstChapter: 1, Occurrences: 4},
|
||||||
|
{BookID: "b", Base: "図書館", Reading: "としょかん", FirstChapter: 1, Occurrences: 20},
|
||||||
|
}
|
||||||
|
// 図書館 is owned by the manual seed → its ruby candidate is skipped.
|
||||||
|
manual := map[string]bool{"図書館": true}
|
||||||
|
cands := rubyToCandidates(readings, manual)
|
||||||
|
|
||||||
|
if len(cands) != 2 {
|
||||||
|
t.Fatalf("want 2 candidates (鈴木, 田中; 図書館 skipped), got %d: %+v", len(cands), cands)
|
||||||
|
}
|
||||||
|
byBase := map[string]store.GlossaryEntry{}
|
||||||
|
for _, c := range cands {
|
||||||
|
byBase[c.Src] = c
|
||||||
|
if c.Status != "auto" || c.Source != "ruby" || c.Dst != "" {
|
||||||
|
t.Errorf("ruby candidate must be auto/ruby with no dst: %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 鈴木 deduped to the DOMINANT reading (すずき, 9 occ > スズキ, 3) with the earliest chapter.
|
||||||
|
suzuki := byBase["鈴木"]
|
||||||
|
if suzuki.RubyReading != "すずき" {
|
||||||
|
t.Errorf("dominant reading = %q, want すずき", suzuki.RubyReading)
|
||||||
|
}
|
||||||
|
if suzuki.SinceCh != 2 {
|
||||||
|
t.Errorf("since_ch = %d, want the earliest (2)", suzuki.SinceCh)
|
||||||
|
}
|
||||||
|
if suzuki.Confidence != 9 {
|
||||||
|
t.Errorf("confidence = %d, want 9 (occurrences of the dominant reading)", suzuki.Confidence)
|
||||||
|
}
|
||||||
|
if suzuki.RubyClass != rubyClassName {
|
||||||
|
t.Errorf("ruby_class = %q, want %q", suzuki.RubyClass, rubyClassName)
|
||||||
|
}
|
||||||
|
if _, skipped := byBase["図書館"]; skipped {
|
||||||
|
t.Error("図書館 should have been skipped (owned by the manual seed)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -150,11 +150,28 @@ func snippetStr(s string) string {
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
// Messages builds the wire-neutral message list for a stage. Layout по Р5:
|
// Messages builds the wire-neutral message list for a stage with no memory injection
|
||||||
// system (стабильный префикс, кэш-граница на последнем стабильном блоке) →
|
// (the Веха-0/2 behaviour). Layout по Р5: system (стабильный префикс, кэш-граница) →
|
||||||
// user (волатильный хвост: чанк/черновик). Инъекция глоссария/STM встанет
|
// user (волатильный хвост). Kept as the zero-injection convenience over
|
||||||
// между ними в Фазе 1, не меняя схемы.
|
// MessagesWithInjection so existing callers/tests are untouched.
|
||||||
func Messages(tpl *PromptTemplate, v RenderVars) ([]llm.Message, error) {
|
func Messages(tpl *PromptTemplate, v RenderVars) ([]llm.Message, error) {
|
||||||
|
return MessagesWithInjection(tpl, v, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// MessagesWithInjection builds the stage messages with the memory bank v2 injection
|
||||||
|
// surface (шаг 4 §C, orchestrator variant "а" — a code-assembled message, NOT a new
|
||||||
|
// RenderVars placeholder, so the fixed template stays stable). Layout по Р5/A6:
|
||||||
|
//
|
||||||
|
// system(стабильный префикс, CacheBoundary) → [injection: глоссарий/STM] → user(волатильный хвост)
|
||||||
|
//
|
||||||
|
// The injection is its OWN message placed AFTER the cache boundary (which stays on the
|
||||||
|
// stable system prefix — the DeepSeek byte-prefix cache still hits; Anthropic's removed,
|
||||||
|
// no cache_control concern). It is NOT part of ch.Text, so it never leaks into the
|
||||||
|
// coverage len_ratio or the {{text}} placeholder (the 3a ch.Text contract). Because the
|
||||||
|
// injection is a message, it is folded into request_hash/content_hash automatically — a
|
||||||
|
// changed injection is a changed request (a resumed chunk reproduces it deterministically
|
||||||
|
// from the frozen bank). An empty injection yields the exact 2-message list of Messages.
|
||||||
|
func MessagesWithInjection(tpl *PromptTemplate, v RenderVars, injection string) ([]llm.Message, error) {
|
||||||
sys, err := Render(tpl.System, v)
|
sys, err := Render(tpl.System, v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
@ -163,10 +180,13 @@ func Messages(tpl *PromptTemplate, v RenderVars) ([]llm.Message, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return []llm.Message{
|
msgs := make([]llm.Message, 0, 3)
|
||||||
{Role: "system", Content: sys, CacheBoundary: true},
|
msgs = append(msgs, llm.Message{Role: "system", Content: sys, CacheBoundary: true})
|
||||||
{Role: "user", Content: user},
|
if strings.TrimSpace(injection) != "" {
|
||||||
}, nil
|
msgs = append(msgs, llm.Message{Role: "system", Content: injection, CacheBoundary: false})
|
||||||
|
}
|
||||||
|
msgs = append(msgs, llm.Message{Role: "user", Content: user})
|
||||||
|
return msgs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RequestHash is the checkpoint/resume key (§3.1): a stable hash of everything
|
// RequestHash is the checkpoint/resume key (§3.1): a stable hash of everything
|
||||||
|
|
|
||||||
93
backend/internal/pipeline/render_memory_test.go
Normal file
93
backend/internal/pipeline/render_memory_test.go
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMessagesInjectionLayout(t *testing.T) {
|
||||||
|
tpl := writeTemplate(t, "Стабильный системный префикс.\n---USER---\nПереведи: {{text}}")
|
||||||
|
v := RenderVars{Book: testBook(), Text: "исходный чанк"}
|
||||||
|
injection := "ГЛОССАРИЙ:\n阿Q → А-кью"
|
||||||
|
|
||||||
|
msgs, err := MessagesWithInjection(tpl, v, injection)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(msgs) != 3 {
|
||||||
|
t.Fatalf("want 3 messages (system, injection, user), got %d", len(msgs))
|
||||||
|
}
|
||||||
|
// [0] stable system prefix carries the cache boundary.
|
||||||
|
if msgs[0].Role != "system" || !msgs[0].CacheBoundary {
|
||||||
|
t.Errorf("msg[0] must be the system prefix with CacheBoundary=true: %+v", msgs[0])
|
||||||
|
}
|
||||||
|
// [1] injection sits AFTER the boundary (volatile → CacheBoundary=false).
|
||||||
|
if msgs[1].Role != "system" || msgs[1].CacheBoundary || msgs[1].Content != injection {
|
||||||
|
t.Errorf("msg[1] must be the injection with CacheBoundary=false: %+v", msgs[1])
|
||||||
|
}
|
||||||
|
// [2] the volatile user chunk. It must contain ONLY the rendered chunk, never the
|
||||||
|
// glossary (ch.Text stays clean — the 3a contract; the coverage gate reads ch.Text).
|
||||||
|
if msgs[2].Role != "user" {
|
||||||
|
t.Errorf("msg[2] must be user: %+v", msgs[2])
|
||||||
|
}
|
||||||
|
if strings.Contains(msgs[2].Content, "ГЛОССАРИЙ") || strings.Contains(msgs[2].Content, "阿Q") {
|
||||||
|
t.Errorf("glossary leaked into the user (ch.Text) message: %q", msgs[2].Content)
|
||||||
|
}
|
||||||
|
if !strings.Contains(msgs[2].Content, "исходный чанк") {
|
||||||
|
t.Errorf("user message lost the chunk: %q", msgs[2].Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMessagesEmptyInjectionIdentical(t *testing.T) {
|
||||||
|
tpl := writeTemplate(t, "Системный префикс.\n---USER---\nТекст: {{text}}")
|
||||||
|
v := RenderVars{Book: testBook(), Text: "чанк"}
|
||||||
|
plain, _ := Messages(tpl, v)
|
||||||
|
withEmpty, _ := MessagesWithInjection(tpl, v, " ") // whitespace-only → no injection
|
||||||
|
if len(plain) != 2 || len(withEmpty) != 2 {
|
||||||
|
t.Fatalf("empty injection must yield 2 messages: %d vs %d", len(plain), len(withEmpty))
|
||||||
|
}
|
||||||
|
for i := range plain {
|
||||||
|
if plain[i] != withEmpty[i] {
|
||||||
|
t.Errorf("empty-injection message %d differs from plain: %+v vs %+v", i, withEmpty[i], plain[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestInjectionChangesRequestHash: the injection is folded into request_hash via the
|
||||||
|
// messages, so a different glossary is a different request (a changed approved term
|
||||||
|
// re-derives the chunk, never serves a stale checkpoint).
|
||||||
|
func TestInjectionChangesRequestHash(t *testing.T) {
|
||||||
|
tpl := writeTemplate(t, "Префикс.\n---USER---\n{{text}}")
|
||||||
|
v := RenderVars{Book: testBook(), Text: "чанк"}
|
||||||
|
none, _ := Messages(tpl, v)
|
||||||
|
inj, _ := MessagesWithInjection(tpl, v, "ГЛОССАРИЙ:\n阿Q → А-кью")
|
||||||
|
hNone := RequestHash("b", 1, 0, 0, "draft", "translator", "m", 0.3, "off", false, 1024, "snap", none)
|
||||||
|
hInj := RequestHash("b", 1, 0, 0, "draft", "translator", "m", 0.3, "off", false, 1024, "snap", inj)
|
||||||
|
if hNone == hInj {
|
||||||
|
t.Error("injection did not change request_hash — a glossary change would silently hit a stale checkpoint")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderGlossaryBlock(t *testing.T) {
|
||||||
|
confirmed := pickedEntry{entry: &memoryEntry{src: "阿Q", dst: "А-кью", status: "approved"}, disp: memConfirmed}
|
||||||
|
ambiguous := pickedEntry{entry: &memoryEntry{src: "小D", dst: "Малыш Дэ", status: "auto"}, disp: memAmbiguous}
|
||||||
|
emptyDst := pickedEntry{entry: &memoryEntry{src: "鈴木", dst: "", status: "auto"}, disp: memAmbiguous}
|
||||||
|
|
||||||
|
block := renderGlossaryBlock([]pickedEntry{confirmed, ambiguous, emptyDst})
|
||||||
|
if !strings.Contains(block, "阿Q → А-кью") {
|
||||||
|
t.Errorf("confirmed line missing: %q", block)
|
||||||
|
}
|
||||||
|
if !strings.Contains(block, "小D → Малыш Дэ ⟨проверить⟩") {
|
||||||
|
t.Errorf("ambiguous line missing its unverified tag: %q", block)
|
||||||
|
}
|
||||||
|
if strings.Contains(block, "鈴木") {
|
||||||
|
t.Errorf("empty-dst candidate should be skipped (nothing to inject): %q", block)
|
||||||
|
}
|
||||||
|
// Nothing to inject → empty block (лучше ничего).
|
||||||
|
if renderGlossaryBlock(nil) != "" {
|
||||||
|
t.Error("empty selection must render an empty block")
|
||||||
|
}
|
||||||
|
if renderGlossaryBlock([]pickedEntry{emptyDst}) != "" {
|
||||||
|
t.Error("only-empty-dst selection must render an empty block")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -56,6 +56,13 @@ type Runner struct {
|
||||||
|
|
||||||
clients map[string]llm.LLMClient
|
clients map[string]llm.LLMClient
|
||||||
templates map[string]*PromptTemplate
|
templates map[string]*PromptTemplate
|
||||||
|
|
||||||
|
// memory is the book's glossary FROZEN for this job (materialized once in
|
||||||
|
// TranslateBook, after seeding, before snapshotID). Its Version() is the F1
|
||||||
|
// content-hash folded into the snapshot; its Select drives per-chunk injection.
|
||||||
|
// nil until materialized (report path / no glossary) → memoryVersion() falls back
|
||||||
|
// to the empty-materialization hash, a stable constant.
|
||||||
|
memory *MemoryBank
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRunner loads all three configs rooted at book.yaml and opens the project
|
// NewRunner loads all three configs rooted at book.yaml and opens the project
|
||||||
|
|
@ -220,19 +227,21 @@ func (r *Runner) escalationBudgetRemains() (bool, error) {
|
||||||
return spent < budget, nil
|
return spent < budget, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// memoryVersion is the content-hash of the deterministically materialized
|
// memoryVersion is the content-hash of the deterministically materialized injected
|
||||||
// injected memory (approved glossary + summaries + series-bible), the memory
|
// memory (the frozen APPROVED glossary rows + the normalization/matcher algorithm
|
||||||
// component of the snapshot (D5.2/D8). Until the memory bank v2 migration lands
|
// versions), the memory component of the snapshot (D5.2/D8, F1 CLOSED). It is the
|
||||||
// the materialization is empty, so the hash is a stable constant; once memory
|
// Version() of the bank materialized once before the loop, so a change to the approved
|
||||||
// enters msgs this changes and the resnapshot-gate fires loudly instead of a
|
// glossary — or to the deterministic machinery (trad→simp table, matcher) — fires the
|
||||||
// stale checkpoint re-paying a diverged translation. STM is excluded (rebuilt
|
// resnapshot gate LOUDLY instead of a stale checkpoint re-paying a diverged translation.
|
||||||
// from checkpoints, §3.2).
|
// nil bank (report path / a book with no glossary) → the empty-materialization hash, a
|
||||||
|
// stable constant. STM is excluded (rebuilt from checkpoints, §3.2). Auto/draft rows are
|
||||||
|
// excluded per D8 (their injected-content changes are caught at the per-chunk
|
||||||
|
// content_hash level; a future autopopulation milestone revisits this).
|
||||||
func (r *Runner) memoryVersion() string {
|
func (r *Runner) memoryVersion() string {
|
||||||
h := sha256.New()
|
if r.memory != nil {
|
||||||
// Domain tag + empty materialization; the store query lands with the v2
|
return r.memory.Version()
|
||||||
// memory schema and feeds ORDER BY-stable rows here.
|
}
|
||||||
h.Write([]byte("tm-memory-v1\x00"))
|
return computeMemoryVersion(nil, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||||||
return hex.EncodeToString(h.Sum(nil))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// snapshotID materializes the job-context snapshot (§3.2): brief_hash, chunker
|
// snapshotID materializes the job-context snapshot (§3.2): brief_hash, chunker
|
||||||
|
|
@ -324,6 +333,12 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
||||||
// хэш стабилен; когда память войдёт в msgs, поле сменится и resnapshot-
|
// хэш стабилен; когда память войдёт в msgs, поле сменится и resnapshot-
|
||||||
// гейт сработает громко. STM сюда НЕ входит (пересобирается из чекпоинтов).
|
// гейт сработает громко. STM сюда НЕ входит (пересобирается из чекпоинтов).
|
||||||
MemoryVersion string `json:"memory_version"`
|
MemoryVersion string `json:"memory_version"`
|
||||||
|
// PostcheckGate — the memory-bank post-check gate (E1). When true a post-check
|
||||||
|
// miss flips the chunk to flagged; folding its on/off state makes toggling it a
|
||||||
|
// loud --resnapshot (it changes a checkpoint's RESOLVED disposition), not a silent
|
||||||
|
// re-verdict on resume (same class as coverage/classifier). The post-check
|
||||||
|
// ALGORITHM version is already inside MemoryVersion (memoryMatchVersion).
|
||||||
|
PostcheckGate bool `json:"postcheck_gate"`
|
||||||
// Coverage — excision QA-gate config (D12 Q3). It does not touch the wire,
|
// Coverage — excision QA-gate config (D12 Q3). It does not touch the wire,
|
||||||
// but it determines a checkpoint's RESOLVED verdict, so a gate change must be
|
// but it determines a checkpoint's RESOLVED verdict, so a gate change must be
|
||||||
// a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot).
|
// a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot).
|
||||||
|
|
@ -346,6 +361,7 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
||||||
CacheTTL: r.Pipeline.Context.CacheTTL,
|
CacheTTL: r.Pipeline.Context.CacheTTL,
|
||||||
},
|
},
|
||||||
MemoryVersion: r.memoryVersion(),
|
MemoryVersion: r.memoryVersion(),
|
||||||
|
PostcheckGate: r.Pipeline.Gates.Glossary.PostcheckGate,
|
||||||
Coverage: r.coverageSnapshot(),
|
Coverage: r.coverageSnapshot(),
|
||||||
}
|
}
|
||||||
for _, st := range r.Pipeline.Stages {
|
for _, st := range r.Pipeline.Stages {
|
||||||
|
|
@ -480,8 +496,23 @@ func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Persist captured ruby readings (idempotent; consumed by seedGlossary below into
|
||||||
|
// auto glossary candidates — шаг 4). Never injected into a prompt here (§7d); a
|
||||||
|
// resume re-persists the same rows at $0.
|
||||||
|
if err := r.persistRuby(doc.Ruby); err != nil {
|
||||||
|
return nil, fmt.Errorf("pipeline: persist ruby readings: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed + materialize the glossary BEFORE the snapshot: the frozen approved rows feed
|
||||||
|
// memoryVersion() → the snapshot (F1), so the snapshot must be computed with the bank
|
||||||
|
// already materialized. Deterministic and $0 (seed file + classified ruby, no LLM).
|
||||||
|
if err := r.seedGlossary(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// Snapshot is book-level (brief + stage plan + memory), computed once and
|
// Snapshot is book-level (brief + stage plan + memory), computed once and
|
||||||
// upserted before the loop; every job pins to it.
|
// upserted before the loop; every job pins to it. memoryVersion() now reflects the
|
||||||
|
// materialized bank, so a changed approved glossary re-pins loudly (F1).
|
||||||
snapID, snapPayload, err := r.snapshotID()
|
snapID, snapPayload, err := r.snapshotID()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
@ -495,17 +526,19 @@ func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) {
|
||||||
return nil, fmt.Errorf("pipeline: source file %s produced no chunks after normalization", r.Book.SourceFile)
|
return nil, fmt.Errorf("pipeline: source file %s produced no chunks after normalization", r.Book.SourceFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Persist captured ruby readings (idempotent; consumed by memory v2 for a
|
|
||||||
// glossary name-lock — шаг 4). Never injected into a prompt here (§7d), so it
|
|
||||||
// touches neither the snapshot nor request_hash; a resume re-persists the same
|
|
||||||
// rows at $0.
|
|
||||||
if err := r.persistRuby(doc.Ruby); err != nil {
|
|
||||||
return nil, fmt.Errorf("pipeline: persist ruby readings: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
res := &BookResult{BookID: r.Book.BookID}
|
res := &BookResult{BookID: r.Book.BookID}
|
||||||
|
// Sticky scene-inertia state (A5): the exact-matched ids of the recent chunks in
|
||||||
|
// the CURRENT chapter, reset at a chapter boundary (a new chapter is a scene change).
|
||||||
|
// Rebuilt deterministically each run because translateChunk recomputes the ($0)
|
||||||
|
// selection for EVERY chunk, resumed ones included — so sticky is resume-stable.
|
||||||
|
var stickyWin []map[string]bool
|
||||||
|
prevChapter := 0
|
||||||
for _, ch := range chunks {
|
for _, ch := range chunks {
|
||||||
outcome, err := r.translateChunk(ctx, snapID, ch)
|
if ch.Chapter != prevChapter {
|
||||||
|
stickyWin = nil
|
||||||
|
prevChapter = ch.Chapter
|
||||||
|
}
|
||||||
|
outcome, activeIDs, err := r.translateChunk(ctx, snapID, ch, unionSticky(stickyWin))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Infra failure: abort. Chunks already committed are checkpointed;
|
// Infra failure: abort. Chunks already committed are checkpointed;
|
||||||
// resume continues from here at $0 for the done work.
|
// resume continues from here at $0 for the done work.
|
||||||
|
|
@ -516,10 +549,73 @@ func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) {
|
||||||
if outcome.Disposition == DispFlagged {
|
if outcome.Disposition == DispFlagged {
|
||||||
res.Flagged++
|
res.Flagged++
|
||||||
}
|
}
|
||||||
|
// Advance the sticky window (keep the last stickyDepth chunks' exact matches).
|
||||||
|
stickyWin = append(stickyWin, activeIDs)
|
||||||
|
if len(stickyWin) > stickyDepth {
|
||||||
|
stickyWin = stickyWin[len(stickyWin)-stickyDepth:]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unionSticky merges the recent chunks' exact-matched id sets into one sticky_prev.
|
||||||
|
func unionSticky(win []map[string]bool) map[string]bool {
|
||||||
|
if len(win) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := map[string]bool{}
|
||||||
|
for _, s := range win {
|
||||||
|
for id := range s {
|
||||||
|
out[id] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// seedGlossary REPLACES the book's glossary from its deterministic inputs — the manual
|
||||||
|
// seed file (curated approved/draft terms) and the captured ruby readings (classified
|
||||||
|
// into auto candidates) — then MATERIALIZES the frozen bank for this job (r.memory). Run
|
||||||
|
// once before snapshotID: the frozen APPROVED rows are hashed into memoryVersion (F1), so
|
||||||
|
// editing the seed is a loud --resnapshot. Idempotent (full replace), $0, no LLM. B2
|
||||||
|
// approved dst-collisions are logged (not fatal — some collisions are legitimate).
|
||||||
|
func (r *Runner) seedGlossary(ctx context.Context) error {
|
||||||
|
var entries []store.GlossaryEntry
|
||||||
|
if r.Book.GlossarySeed != "" {
|
||||||
|
seed, err := loadGlossarySeed(r.Book.GlossarySeed)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
entries = append(entries, seed...)
|
||||||
|
}
|
||||||
|
manualSrcs := map[string]bool{}
|
||||||
|
for _, e := range entries {
|
||||||
|
manualSrcs[e.Src] = true
|
||||||
|
}
|
||||||
|
ruby, err := r.Store.RubyReadingsForBook(r.Book.BookID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
entries = append(entries, rubyToCandidates(ruby, manualSrcs)...)
|
||||||
|
for i := range entries {
|
||||||
|
entries[i].BookID = r.Book.BookID
|
||||||
|
}
|
||||||
|
if err := r.Store.ReplaceGlossary(r.Book.BookID, entries); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows, err := r.Store.GlossaryForBook(r.Book.BookID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
r.memory = materializeMemory(rows, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||||||
|
if cols := injectivityCollisions(rows); len(cols) > 0 {
|
||||||
|
r.Log.WarnContext(ctx, "glossary approved dst-collisions (B2: two source terms share one Russian surface — the reader cannot tell them apart)",
|
||||||
|
"collisions", strings.Join(cols, "; "))
|
||||||
|
}
|
||||||
|
r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID,
|
||||||
|
"entries", len(rows), "memory_version", r.memory.Version()[:12])
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// persistRuby aggregates the ingested ruby occurrences into one row per
|
// persistRuby aggregates the ingested ruby occurrences into one row per
|
||||||
// (base, reading) — first_chapter = MIN, occurrences = full-book count — and
|
// (base, reading) — first_chapter = MIN, occurrences = full-book count — and
|
||||||
// REPLACES the book's whole ruby set (store.ReplaceRubyReadings). Idempotent: the
|
// REPLACES the book's whole ruby set (store.ReplaceRubyReadings). Idempotent: the
|
||||||
|
|
@ -569,14 +665,30 @@ func (r *Runner) persistRuby(readings []RubyReading) error {
|
||||||
|
|
||||||
// translateChunk drives ONE chunk through the stage list. Stages run in order,
|
// translateChunk drives ONE chunk through the stage list. Stages run in order,
|
||||||
// each feeding the next; the FIRST flagged stage stops the chunk — later stages
|
// each feeding the next; the FIRST flagged stage stops the chunk — later stages
|
||||||
// are recorded `skipped` (no paid edit over a garbage draft, D2). Returns an
|
// are recorded `skipped` (no paid edit over a garbage draft, D2). It also runs the
|
||||||
// error only on an infra failure.
|
// memory bank v2 hot path: it Selects the glossary injection once ($0, deterministic),
|
||||||
func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (*ChunkOutcome, error) {
|
// injects it into the TRANSLATOR stage, post-checks the translator output (E1), and
|
||||||
|
// persists the per-chunk retrieval-state (observability). Returns the chunk outcome, the
|
||||||
|
// exact-matched entity ids (the next chunk's sticky_prev, A5), and an error only on an
|
||||||
|
// infra failure. stickyPrev is the prior chunks' exact matches in this chapter.
|
||||||
|
func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, stickyPrev map[string]bool) (*ChunkOutcome, map[string]bool, error) {
|
||||||
out := &ChunkOutcome{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: DispOK}
|
out := &ChunkOutcome{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: DispOK}
|
||||||
prev := ""
|
prev := ""
|
||||||
flagged := false
|
flagged := false
|
||||||
var flagReason FlagReason
|
var flagReason FlagReason
|
||||||
|
|
||||||
|
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and
|
||||||
|
// serialize the translator injection block. Recomputed on every run (resumed chunks
|
||||||
|
// included) so the sticky window and the injected bytes are resume-stable.
|
||||||
|
var memSel memorySelection
|
||||||
|
var translatorInjection string
|
||||||
|
activeIDs := map[string]bool{}
|
||||||
|
if r.memory != nil {
|
||||||
|
memSel = r.memory.Select(ch.Text, ch.Chapter, stickyPrev, r.Pipeline.Context.GlossaryTokenBudget)
|
||||||
|
translatorInjection = renderGlossaryBlock(memSel.injected)
|
||||||
|
activeIDs = memSel.activeIDs
|
||||||
|
}
|
||||||
|
|
||||||
for stageIdx, st := range r.Pipeline.Stages {
|
for stageIdx, st := range r.Pipeline.Stages {
|
||||||
if flagged {
|
if flagged {
|
||||||
// An earlier stage flagged → this stage is not attempted or billed.
|
// An earlier stage flagged → this stage is not attempted or billed.
|
||||||
|
|
@ -586,7 +698,7 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (*
|
||||||
SnapshotID: snapID, Disposition: string(DispSkipped), FlagReason: string(flagReason),
|
SnapshotID: snapID, Disposition: string(DispSkipped), FlagReason: string(flagReason),
|
||||||
Detail: detail,
|
Detail: detail,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return out, fmt.Errorf("pipeline: record skipped chunk_status: %w", err)
|
return out, activeIDs, fmt.Errorf("pipeline: record skipped chunk_status: %w", err)
|
||||||
}
|
}
|
||||||
out.Stages = append(out.Stages, StageResult{
|
out.Stages = append(out.Stages, StageResult{
|
||||||
Stage: st.Name, Role: st.Role, Model: st.Model,
|
Stage: st.Name, Role: st.Role, Model: st.Model,
|
||||||
|
|
@ -595,9 +707,17 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (*
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
sr, err := r.runStage(ctx, st, stageIdx, snapID, ch, prev)
|
// Inject the glossary ONLY into the translator (шаг 4 scope): its src→dst records
|
||||||
|
// are matched against the source chunk. Other stages (the monolingual editor) get
|
||||||
|
// no injection in v1 (a dst-constraint injection for the editor is a fast-follow on
|
||||||
|
// this same surface). Empty injection → the plain 2-message layout.
|
||||||
|
injection := ""
|
||||||
|
if st.Role == roleTranslator {
|
||||||
|
injection = translatorInjection
|
||||||
|
}
|
||||||
|
sr, err := r.runStage(ctx, st, stageIdx, snapID, ch, prev, injection)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, err
|
return out, activeIDs, err
|
||||||
}
|
}
|
||||||
out.Stages = append(out.Stages, *sr)
|
out.Stages = append(out.Stages, *sr)
|
||||||
out.CostUSD += sr.CostUSD
|
out.CostUSD += sr.CostUSD
|
||||||
|
|
@ -609,14 +729,82 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (*
|
||||||
prev = sr.Text
|
prev = sr.Text
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Post-check the FINAL, exported output (E1): the reader sees the last stage's text
|
||||||
|
// (the editor's), not the translator draft — the monolingual editor can drift an
|
||||||
|
// approved term the translator got right, so checking the draft alone would miss it.
|
||||||
|
// Runs on the fresh OR fully-resumed chunk (both carry the final text through `prev`),
|
||||||
|
// so it is resume-reproducible. In the default FLAGGER mode a miss is recorded only in
|
||||||
|
// the retrieval-state (observability); with the opt-in gate a miss flags the chunk
|
||||||
|
// (glossary_miss). Skipped when a stage already flagged (no usable output to check).
|
||||||
|
var postMisses []postcheckMiss
|
||||||
|
outputChecked := false
|
||||||
|
if r.memory != nil && !flagged && prev != "" {
|
||||||
|
outputChecked = true
|
||||||
|
postMisses = r.memory.postcheck(memSel.injected, prev)
|
||||||
|
if r.Pipeline.Gates.Glossary.PostcheckGate && len(postMisses) > 0 {
|
||||||
|
flagged = true
|
||||||
|
flagReason = FlagGlossaryMiss
|
||||||
|
r.Log.WarnContext(ctx, "glossary post-check gate flagged the chunk",
|
||||||
|
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "misses", len(postMisses))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Persist the per-chunk retrieval-state (registry gate #4: convert silent memory
|
||||||
|
// degradation into a loud, visible record). Deterministic + idempotent, so a resume
|
||||||
|
// re-derives the identical row. Only when a glossary is materialized.
|
||||||
|
if r.memory != nil {
|
||||||
|
if err := r.persistRetrievalState(snapID, ch, memSel, postMisses, outputChecked); err != nil {
|
||||||
|
return out, activeIDs, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if flagged {
|
if flagged {
|
||||||
out.Disposition = DispFlagged
|
out.Disposition = DispFlagged
|
||||||
out.FlagReason = flagReason
|
out.FlagReason = flagReason
|
||||||
out.FinalText = "" // garbage/refusal never propagates to the next chunk or export
|
out.FinalText = "" // garbage/refusal/glossary-miss never propagates to the next chunk or export
|
||||||
} else {
|
} else {
|
||||||
out.FinalText = prev
|
out.FinalText = prev
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, activeIDs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// persistRetrievalState writes the per-chunk observability record from the deterministic
|
||||||
|
// selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED
|
||||||
|
// records (what the model saw); spoiler/eviction are the dropped-and-logged totals;
|
||||||
|
// post-check misses are recorded only when the translator actually produced text.
|
||||||
|
func (r *Runner) persistRetrievalState(snapID string, ch Chunk, sel memorySelection, misses []postcheckMiss, outputChecked bool) error {
|
||||||
|
rs := store.RetrievalState{
|
||||||
|
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, SnapshotID: snapID,
|
||||||
|
}
|
||||||
|
for _, p := range sel.injected {
|
||||||
|
if p.via == "sticky" {
|
||||||
|
rs.NSticky++
|
||||||
|
} else {
|
||||||
|
rs.NExactHits++
|
||||||
|
}
|
||||||
|
if p.disp == memAmbiguous {
|
||||||
|
rs.NAmbiguousFlagged++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rs.NSpoilerBlocked = len(sel.rejected)
|
||||||
|
rs.NEvicted = len(sel.evicted)
|
||||||
|
if outputChecked {
|
||||||
|
rs.NPostcheckMiss = len(misses)
|
||||||
|
if len(misses) > 0 {
|
||||||
|
if b, err := json.Marshal(misses); err == nil {
|
||||||
|
rs.PostcheckDetail = string(b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ids := make([]string, 0, len(sel.activeIDs))
|
||||||
|
for id := range sel.activeIDs {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
if b, err := json.Marshal(ids); err == nil {
|
||||||
|
rs.InjectedIDs = string(b)
|
||||||
|
}
|
||||||
|
return r.Store.UpsertRetrievalState(rs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// runStage runs ONE stage of ONE chunk to a terminal disposition. It first
|
// runStage runs ONE stage of ONE chunk to a terminal disposition. It first
|
||||||
|
|
@ -626,7 +814,7 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk) (*
|
||||||
// subset with a doubled budget up to the regenerate cap, then flag. Returns an
|
// subset with a doubled budget up to the regenerate cap, then flag. Returns an
|
||||||
// error ONLY on an infra failure; a bad completion is a disposition, never an
|
// error ONLY on an infra failure; a bad completion is a disposition, never an
|
||||||
// error.
|
// error.
|
||||||
func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, snapID string, ch Chunk, prev string) (*StageResult, error) {
|
func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, snapID string, ch Chunk, prev, injection string) (*StageResult, error) {
|
||||||
job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID)
|
job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
@ -649,7 +837,7 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
||||||
// fast-path be CONTENT-VERIFIED: the source bytes are not in the snapshot, so
|
// fast-path be CONTENT-VERIFIED: the source bytes are not in the snapshot, so
|
||||||
// a positional chunk_status row must be checked against the current rendered
|
// a positional chunk_status row must be checked against the current rendered
|
||||||
// content, else an edited source would serve a stale, divergent translation.
|
// content, else an edited source would serve a stale, divergent translation.
|
||||||
msgs, err := Messages(r.templates[st.Name], RenderVars{Book: r.Book, Text: ch.Text, Draft: prev})
|
msgs, err := MessagesWithInjection(r.templates[st.Name], RenderVars{Book: r.Book, Text: ch.Text, Draft: prev}, injection)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
245
backend/internal/pipeline/runner_memory_test.go
Normal file
245
backend/internal/pipeline/runner_memory_test.go
Normal file
|
|
@ -0,0 +1,245 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// runner_memory_test.go: the шаг-4 acceptance — the memory bank v2 mechanism end-to-end
|
||||||
|
// on a seeded glossary against the mock provider (injection → translate → post-check →
|
||||||
|
// retrieval-state → resume $0 → resnapshot on an approved change). Per §8, acceptance is
|
||||||
|
// "the mechanism is correct on a seeded glossary", not "≥98% on a real book".
|
||||||
|
|
||||||
|
const suzukiSeed = `
|
||||||
|
terms:
|
||||||
|
- src: 鈴木
|
||||||
|
dst: Судзуки
|
||||||
|
type: name
|
||||||
|
status: approved
|
||||||
|
decl: { invariant: true, forms: ["Судзуки"] }
|
||||||
|
`
|
||||||
|
|
||||||
|
// suzukiSource is one ja chunk that names 鈴木 (so the glossary fires).
|
||||||
|
const suzukiSource = "鈴木は静かな図書館へ行った。"
|
||||||
|
|
||||||
|
type wireMsg struct {
|
||||||
|
Role string
|
||||||
|
Content string
|
||||||
|
}
|
||||||
|
|
||||||
|
func bodyMessages(t *testing.T, body string) []wireMsg {
|
||||||
|
t.Helper()
|
||||||
|
var req struct {
|
||||||
|
Messages []wireMsg `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(body), &req); err != nil {
|
||||||
|
t.Fatalf("bad request body: %v\n%s", err, body)
|
||||||
|
}
|
||||||
|
return req.Messages
|
||||||
|
}
|
||||||
|
|
||||||
|
// translatorBody returns the recorded draft(translator) request body.
|
||||||
|
func translatorBody(t *testing.T, rec *reqRec) string {
|
||||||
|
t.Helper()
|
||||||
|
for _, b := range rec.all() {
|
||||||
|
if !isEditBody(b) {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("no translator request recorded")
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerMemoryInjectionAndPostcheck(t *testing.T) {
|
||||||
|
rec := &reqRec{}
|
||||||
|
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||||
|
if isEditBody(body) {
|
||||||
|
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД про Судзуки.", "stop"
|
||||||
|
}
|
||||||
|
return "Судзуки пошёл в тихую библиотеку.", "stop" // renders the approved dst
|
||||||
|
})
|
||||||
|
defer srv.Close()
|
||||||
|
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed})
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
r := newRunner(t, bookPath)
|
||||||
|
res, err := r.TranslateBook(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Flagged != 0 {
|
||||||
|
t.Fatalf("flagger default must not flag a correct render: flagged=%d", res.Flagged)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The TRANSLATOR request carries the injection as its own system message between the
|
||||||
|
// stable prefix and the user chunk; the dst appears only via injection (not in the ja
|
||||||
|
// source), so its presence proves the injection landed on the wire.
|
||||||
|
tb := translatorBody(t, rec)
|
||||||
|
msgs := bodyMessages(t, tb)
|
||||||
|
if len(msgs) != 3 {
|
||||||
|
t.Fatalf("translator must have 3 messages (system, injection, user), got %d: %+v", len(msgs), msgs)
|
||||||
|
}
|
||||||
|
if msgs[1].Role != "system" || !strings.Contains(msgs[1].Content, "Судзуки") || !strings.Contains(msgs[1].Content, "ГЛОССАРИЙ") {
|
||||||
|
t.Errorf("injection message wrong: %+v", msgs[1])
|
||||||
|
}
|
||||||
|
if strings.Contains(msgs[2].Content, "ГЛОССАРИЙ") {
|
||||||
|
t.Errorf("glossary leaked into the user (ch.Text) message: %q", msgs[2].Content)
|
||||||
|
}
|
||||||
|
// The EDITOR gets no injection in v1.
|
||||||
|
for _, b := range rec.all() {
|
||||||
|
if isEditBody(b) && strings.Contains(b, "ГЛОССАРИЙ") {
|
||||||
|
t.Error("editor received a glossary injection (not in v1 scope)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// retrieval-state: one exact hit, post-check ran with 0 misses.
|
||||||
|
rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
|
||||||
|
if err != nil || rs == nil {
|
||||||
|
t.Fatalf("retrieval-state missing: %v", err)
|
||||||
|
}
|
||||||
|
if rs.NExactHits != 1 || rs.NPostcheckMiss != 0 {
|
||||||
|
t.Errorf("retrieval-state wrong: exact=%d miss=%d", rs.NExactHits, rs.NPostcheckMiss)
|
||||||
|
}
|
||||||
|
callsRun1 := rec.count()
|
||||||
|
r.Close()
|
||||||
|
|
||||||
|
// Resume: no new provider calls, $0 (the deterministic memory path re-derives for free).
|
||||||
|
r2 := newRunner(t, bookPath)
|
||||||
|
defer r2.Close()
|
||||||
|
res2, err := r2.TranslateBook(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec.count() != callsRun1 || res2.TotalUSD != 0 {
|
||||||
|
t.Fatalf("resume must be $0 with no new calls: calls=%d (run1=%d) usd=%v", rec.count(), callsRun1, res2.TotalUSD)
|
||||||
|
}
|
||||||
|
// retrieval-state re-derived identically.
|
||||||
|
rs2, _ := r2.Store.GetRetrievalState("test-book", 1, 0)
|
||||||
|
if rs2 == nil || rs2.NExactHits != 1 || rs2.NPostcheckMiss != 0 {
|
||||||
|
t.Errorf("retrieval-state not stable on resume: %+v", rs2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerMemoryFlaggerRecordsMissWithoutFlagging(t *testing.T) {
|
||||||
|
rec := &reqRec{}
|
||||||
|
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||||
|
if isEditBody(body) {
|
||||||
|
return "ОТРЕДАКТИРОВАННЫЙ.", "stop"
|
||||||
|
}
|
||||||
|
return "Некто пошёл в библиотеку.", "stop" // DROPS the approved name → a post-check miss
|
||||||
|
})
|
||||||
|
defer srv.Close()
|
||||||
|
// Default flagger mode (postcheckGate: false).
|
||||||
|
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed})
|
||||||
|
|
||||||
|
r := newRunner(t, bookPath)
|
||||||
|
defer r.Close()
|
||||||
|
res, err := r.TranslateBook(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Flagger mode: the chunk is NOT flagged (translation still exported), but the miss IS
|
||||||
|
// recorded (silent degradation converted to a loud, visible record).
|
||||||
|
if res.Flagged != 0 {
|
||||||
|
t.Errorf("flagger mode must not flag the chunk: flagged=%d", res.Flagged)
|
||||||
|
}
|
||||||
|
rs, _ := r.Store.GetRetrievalState("test-book", 1, 0)
|
||||||
|
if rs == nil || rs.NPostcheckMiss != 1 {
|
||||||
|
t.Fatalf("post-check miss not recorded: %+v", rs)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rs.PostcheckDetail, "鈴木") {
|
||||||
|
t.Errorf("post-check detail should name the missed src: %q", rs.PostcheckDetail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerMemoryPostcheckGateFlags(t *testing.T) {
|
||||||
|
rec := &reqRec{}
|
||||||
|
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||||
|
if isEditBody(body) {
|
||||||
|
return "ОТРЕДАКТИРОВАННЫЙ.", "stop"
|
||||||
|
}
|
||||||
|
return "Некто пошёл в библиотеку.", "stop" // drops the name
|
||||||
|
})
|
||||||
|
defer srv.Close()
|
||||||
|
// Opt-in hard gate.
|
||||||
|
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true})
|
||||||
|
|
||||||
|
r := newRunner(t, bookPath)
|
||||||
|
defer r.Close()
|
||||||
|
res, err := r.TranslateBook(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Flagged != 1 {
|
||||||
|
t.Fatalf("gate mode must flag the missed chunk: flagged=%d", res.Flagged)
|
||||||
|
}
|
||||||
|
if res.Chunks[0].FlagReason != FlagGlossaryMiss {
|
||||||
|
t.Errorf("flag reason = %q, want glossary_miss", res.Chunks[0].FlagReason)
|
||||||
|
}
|
||||||
|
// The post-check runs on the FINAL exported output, so both stages ran (each ok in
|
||||||
|
// chunk_status), but the CHUNK is flagged and its text is not exported.
|
||||||
|
if res.Chunks[0].FinalText != "" {
|
||||||
|
t.Errorf("a glossary-flagged chunk must not export text, got %q", res.Chunks[0].FinalText)
|
||||||
|
}
|
||||||
|
var editorRan bool
|
||||||
|
for _, st := range res.Chunks[0].Stages {
|
||||||
|
if st.Role == "editor" && st.Disposition == DispOK {
|
||||||
|
editorRan = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !editorRan {
|
||||||
|
t.Error("editor stage should have run (post-check validates the final/editor output, not the draft)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerMemoryResnapshotOnApprovedChange(t *testing.T) {
|
||||||
|
rec := &reqRec{}
|
||||||
|
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||||
|
if isEditBody(body) {
|
||||||
|
return "ОТРЕДАКТИРОВАННЫЙ про Судзуки.", "stop"
|
||||||
|
}
|
||||||
|
return "Судзуки пошёл в библиотеку.", "stop"
|
||||||
|
})
|
||||||
|
defer srv.Close()
|
||||||
|
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed})
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
r1 := newRunner(t, bookPath)
|
||||||
|
if _, err := r1.TranslateBook(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r1.Close()
|
||||||
|
callsAfterRun1 := rec.count()
|
||||||
|
|
||||||
|
// Change the APPROVED dst in the seed → the frozen materialization changes → the
|
||||||
|
// snapshot changes → a resume without --resnapshot must fail loud (F1: no silent
|
||||||
|
// divergent re-pay).
|
||||||
|
seedPath := filepath.Join(filepath.Dir(bookPath), "glossary-seed.yaml")
|
||||||
|
if err := os.WriteFile(seedPath, []byte(strings.Replace(suzukiSeed, "Судзуки", "Сузуки", -1)), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r2 := newRunner(t, bookPath)
|
||||||
|
_, err := r2.TranslateBook(ctx)
|
||||||
|
r2.Close()
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||||||
|
t.Fatalf("changed approved glossary must fail loud mentioning --resnapshot, got: %v", err)
|
||||||
|
}
|
||||||
|
if rec.count() != callsAfterRun1 {
|
||||||
|
t.Fatalf("denied resume must not call the provider: calls=%d (after run1=%d)", rec.count(), callsAfterRun1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// With --resnapshot the book re-translates under the new approved glossary.
|
||||||
|
r3 := newRunner(t, bookPath)
|
||||||
|
defer r3.Close()
|
||||||
|
r3.Resnapshot = true
|
||||||
|
if _, err := r3.TranslateBook(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec.count() <= callsAfterRun1 {
|
||||||
|
t.Fatal("resnapshot run must re-call the provider for the re-translation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -108,6 +108,8 @@ type projectOpts struct {
|
||||||
minMaxTokens int
|
minMaxTokens int
|
||||||
bookUSD float64
|
bookUSD float64
|
||||||
gatesYAML string // optional gates:/... block appended to pipeline.yaml
|
gatesYAML string // optional gates:/... block appended to pipeline.yaml
|
||||||
|
glossarySeed string // optional glossary seed YAML content; "" = no glossary
|
||||||
|
postcheckGate bool // enable the memory post-check hard gate (assumes gatesYAML is empty)
|
||||||
}
|
}
|
||||||
|
|
||||||
func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
|
func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
|
||||||
|
|
@ -142,15 +144,20 @@ models:
|
||||||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||||||
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
||||||
|
|
||||||
|
gatesBlock := o.gatesYAML
|
||||||
|
if o.postcheckGate {
|
||||||
|
gatesBlock += "\ngates:\n glossary:\n postcheck_gate: true\n"
|
||||||
|
}
|
||||||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
||||||
core: C1
|
core: C1
|
||||||
version: 1
|
version: 1
|
||||||
defaults: { max_output_ratio: 2.0, min_max_tokens: %d }
|
defaults: { max_output_ratio: 2.0, min_max_tokens: %d }
|
||||||
retries: { regenerate_before_escalate: %d }
|
retries: { regenerate_before_escalate: %d }
|
||||||
|
context: { glossary_injection: selective, glossary_token_budget: 800 }
|
||||||
stages:
|
stages:
|
||||||
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
||||||
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||||||
%s`, o.minMaxTokens, o.regenerate, o.gatesYAML))
|
%s`, o.minMaxTokens, o.regenerate, gatesBlock))
|
||||||
|
|
||||||
sourceName := "source.txt"
|
sourceName := "source.txt"
|
||||||
if len(o.epub) > 0 {
|
if len(o.epub) > 0 {
|
||||||
|
|
@ -159,6 +166,11 @@ stages:
|
||||||
} else {
|
} else {
|
||||||
writeFile(t, filepath.Join(dir, "source.txt"), o.source)
|
writeFile(t, filepath.Join(dir, "source.txt"), o.source)
|
||||||
}
|
}
|
||||||
|
glossaryLine := ""
|
||||||
|
if o.glossarySeed != "" {
|
||||||
|
writeFile(t, filepath.Join(dir, "glossary-seed.yaml"), o.glossarySeed)
|
||||||
|
glossaryLine = "glossary_seed: glossary-seed.yaml"
|
||||||
|
}
|
||||||
writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(`
|
writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(`
|
||||||
book_id: test-book
|
book_id: test-book
|
||||||
title: Тест
|
title: Тест
|
||||||
|
|
@ -173,8 +185,9 @@ footnotes: minimal
|
||||||
pipeline: pipeline.yaml
|
pipeline: pipeline.yaml
|
||||||
models: models.yaml
|
models: models.yaml
|
||||||
source_file: %s
|
source_file: %s
|
||||||
|
%s
|
||||||
ceilings: { book_usd: %g, day_usd: 2.0 }
|
ceilings: { book_usd: %g, day_usd: 2.0 }
|
||||||
`, sourceName, o.bookUSD))
|
`, sourceName, glossaryLine, o.bookUSD))
|
||||||
return filepath.Join(dir, "book.yaml")
|
return filepath.Join(dir, "book.yaml")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
322
backend/internal/store/glossary.go
Normal file
322
backend/internal/store/glossary.go
Normal file
|
|
@ -0,0 +1,322 @@
|
||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
// glossary.go: durable storage for the memory bank v2 glossary (schema v5). Dumb
|
||||||
|
// storage, mirroring ruby.go/chunkstatus.go: the pipeline owns aggregation,
|
||||||
|
// classification and matching; this layer only replaces and reads back. A book's
|
||||||
|
// glossary is a deterministic function of its inputs (seed file + classified ruby),
|
||||||
|
// so ReplaceGlossary rewrites the WHOLE book set in one transaction — re-seeding the
|
||||||
|
// identical inputs rewrites identical rows, a seed edit converges every column, and a
|
||||||
|
// removed term disappears instead of lingering (the same full-replace idempotency the
|
||||||
|
// ruby step adopted after external-review #6).
|
||||||
|
|
||||||
|
// GlossaryEntry is one term row plus its aliases. Strings are stored raw; the
|
||||||
|
// pipeline normalizes for matching (§3.6). Decl is a JSON blob the post-check reads.
|
||||||
|
type GlossaryEntry struct {
|
||||||
|
ID int64
|
||||||
|
BookID string
|
||||||
|
Src string
|
||||||
|
Dst string
|
||||||
|
Type string
|
||||||
|
Sense string
|
||||||
|
Gender string
|
||||||
|
Speech string
|
||||||
|
Decl string // JSON {"invariant":bool,"forms":[...]}
|
||||||
|
TranslitPolicy string
|
||||||
|
FirstPerson string
|
||||||
|
NicknameTranslation string
|
||||||
|
SinceCh int
|
||||||
|
UntilCh int
|
||||||
|
Status string // auto|draft|approved
|
||||||
|
AllowShort bool
|
||||||
|
Source string // seed|ruby|auto
|
||||||
|
RubyReading string
|
||||||
|
RubyClass string
|
||||||
|
Confidence int
|
||||||
|
Note string
|
||||||
|
Aliases []GlossaryAlias
|
||||||
|
}
|
||||||
|
|
||||||
|
// GlossaryAlias is one alias-graph edge (an alternative source surface + its type).
|
||||||
|
type GlossaryAlias struct {
|
||||||
|
Alias string
|
||||||
|
AliasType string
|
||||||
|
}
|
||||||
|
|
||||||
|
// GlossaryRevision is one append-only editorial-time record (B1).
|
||||||
|
type GlossaryRevision struct {
|
||||||
|
Src string
|
||||||
|
Sense string
|
||||||
|
OldDst string
|
||||||
|
NewDst string
|
||||||
|
EditorialTS string
|
||||||
|
Reason string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetrievalState is one per-chunk observability record (registry gate #4).
|
||||||
|
type RetrievalState struct {
|
||||||
|
BookID string
|
||||||
|
Chapter int
|
||||||
|
ChunkIdx int
|
||||||
|
SnapshotID string
|
||||||
|
NExactHits int
|
||||||
|
NSticky int
|
||||||
|
NAmbiguousFlagged int
|
||||||
|
NSpoilerBlocked int
|
||||||
|
NEvicted int
|
||||||
|
EmbeddingTierUsed int
|
||||||
|
NPostcheckMiss int
|
||||||
|
PostcheckDetail string // JSON
|
||||||
|
InjectedIDs string // JSON
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReplaceGlossary replaces a book's ENTIRE glossary (rows + aliases) with the given
|
||||||
|
// entries in ONE transaction, and appends a B1 revision record for every term whose
|
||||||
|
// approved rendering (dst) changed since the previous set. Scoped to book_id, so other
|
||||||
|
// books are untouched. Each entry's BookID must equal bookID. term_id linkage is
|
||||||
|
// established within the tx: a glossary row is inserted, its rowid taken, then its
|
||||||
|
// aliases inserted against that id (ids are fresh each replace — never hashed).
|
||||||
|
func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error {
|
||||||
|
ctx, cancel := opContext()
|
||||||
|
defer cancel()
|
||||||
|
tx, err := s.w.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
|
||||||
|
// Snapshot the prior (src,sense,window)→dst to detect editorial-time changes (B1).
|
||||||
|
prior := map[[4]any]string{}
|
||||||
|
rows, err := tx.QueryContext(ctx, `SELECT src, sense, since_ch, until_ch, dst FROM glossary WHERE book_id = ?`, bookID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var src, sense, dst string
|
||||||
|
var since, until int
|
||||||
|
if err := rows.Scan(&src, &sense, &since, &until, &dst); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
prior[[4]any{src, sense, since, until}] = dst
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
|
||||||
|
if _, err := tx.ExecContext(ctx, `DELETE FROM glossary_aliases WHERE book_id = ?`, bookID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.ExecContext(ctx, `DELETE FROM glossary WHERE book_id = ?`, bookID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, e := range entries {
|
||||||
|
res, err := tx.ExecContext(ctx, `
|
||||||
|
INSERT INTO glossary (
|
||||||
|
book_id, src, dst, type, sense, gender, speech, decl,
|
||||||
|
translit_policy, first_person, nickname_translation,
|
||||||
|
since_ch, until_ch, status, allow_short, source,
|
||||||
|
ruby_reading, ruby_class, confidence, note
|
||||||
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||||
|
e.BookID, e.Src, e.Dst, e.Type, e.Sense, e.Gender, e.Speech, e.Decl,
|
||||||
|
e.TranslitPolicy, e.FirstPerson, e.NicknameTranslation,
|
||||||
|
e.SinceCh, e.UntilCh, e.Status, boolToInt(e.AllowShort), e.Source,
|
||||||
|
e.RubyReading, e.RubyClass, e.Confidence, e.Note)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
termID, err := res.LastInsertId()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, a := range e.Aliases {
|
||||||
|
if _, err := tx.ExecContext(ctx, `
|
||||||
|
INSERT INTO glossary_aliases (book_id, term_id, alias, alias_type)
|
||||||
|
VALUES (?, ?, ?, ?)`, e.BookID, termID, a.Alias, a.AliasType); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// B1: a term that existed with a non-empty dst and now renders differently
|
||||||
|
// (also non-empty) is an editorial change → append the revision journal row.
|
||||||
|
if old, ok := prior[[4]any{e.Src, e.Sense, e.SinceCh, e.UntilCh}]; ok && old != "" && e.Dst != "" && old != e.Dst {
|
||||||
|
if _, err := tx.ExecContext(ctx, `
|
||||||
|
INSERT INTO glossary_revisions (book_id, src, sense, old_dst, new_dst, reason)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)`, e.BookID, e.Src, e.Sense, old, e.Dst, "seed-replace"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GlossaryForBook returns every glossary entry for a book with its aliases, in a
|
||||||
|
// DETERMINISTIC order (src, sense, since_ch, until_ch, status, dst) with aliases sorted
|
||||||
|
// — the stable materialization the hot-path matcher and the F1 memoryVersion() consume.
|
||||||
|
func (s *Store) GlossaryForBook(bookID string) ([]GlossaryEntry, error) {
|
||||||
|
ctx, cancel := opContext()
|
||||||
|
defer cancel()
|
||||||
|
rows, err := s.r.QueryContext(ctx, `
|
||||||
|
SELECT id, src, dst, type, sense, gender, speech, decl,
|
||||||
|
translit_policy, first_person, nickname_translation,
|
||||||
|
since_ch, until_ch, status, allow_short, source,
|
||||||
|
ruby_reading, ruby_class, confidence, note
|
||||||
|
FROM glossary WHERE book_id = ?
|
||||||
|
ORDER BY src, sense, since_ch, until_ch, status, dst`, bookID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []GlossaryEntry
|
||||||
|
byID := map[int64]int{} // glossary.id → index in out
|
||||||
|
for rows.Next() {
|
||||||
|
e := GlossaryEntry{BookID: bookID}
|
||||||
|
var allowShort int
|
||||||
|
if err := rows.Scan(&e.ID, &e.Src, &e.Dst, &e.Type, &e.Sense, &e.Gender, &e.Speech, &e.Decl,
|
||||||
|
&e.TranslitPolicy, &e.FirstPerson, &e.NicknameTranslation,
|
||||||
|
&e.SinceCh, &e.UntilCh, &e.Status, &allowShort, &e.Source,
|
||||||
|
&e.RubyReading, &e.RubyClass, &e.Confidence, &e.Note); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
e.AllowShort = allowShort != 0
|
||||||
|
byID[e.ID] = len(out)
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Aliases in a second pass, ordered so each entry's alias list is deterministic.
|
||||||
|
arows, err := s.r.QueryContext(ctx, `
|
||||||
|
SELECT term_id, alias, alias_type FROM glossary_aliases
|
||||||
|
WHERE book_id = ? ORDER BY term_id, alias`, bookID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer arows.Close()
|
||||||
|
for arows.Next() {
|
||||||
|
var termID int64
|
||||||
|
var a GlossaryAlias
|
||||||
|
if err := arows.Scan(&termID, &a.Alias, &a.AliasType); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if idx, ok := byID[termID]; ok {
|
||||||
|
out[idx].Aliases = append(out[idx].Aliases, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, arows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GlossaryRevisionsForBook returns the editorial-time journal (B1) for a book,
|
||||||
|
// newest first, for the report / a human audit.
|
||||||
|
func (s *Store) GlossaryRevisionsForBook(bookID string) ([]GlossaryRevision, error) {
|
||||||
|
ctx, cancel := opContext()
|
||||||
|
defer cancel()
|
||||||
|
rows, err := s.r.QueryContext(ctx, `
|
||||||
|
SELECT src, sense, old_dst, new_dst, editorial_ts, reason
|
||||||
|
FROM glossary_revisions WHERE book_id = ?
|
||||||
|
ORDER BY id DESC`, bookID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []GlossaryRevision
|
||||||
|
for rows.Next() {
|
||||||
|
var r GlossaryRevision
|
||||||
|
if err := rows.Scan(&r.Src, &r.Sense, &r.OldDst, &r.NewDst, &r.EditorialTS, &r.Reason); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertRetrievalState writes (or overwrites) the per-chunk retrieval-state record.
|
||||||
|
// Overwrite is the resolve semantics: recomputing the deterministic matcher over the
|
||||||
|
// same chunk reproduces the same row (self-heals on resume).
|
||||||
|
func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
|
||||||
|
ctx, cancel := opContext()
|
||||||
|
defer cancel()
|
||||||
|
_, err := s.w.ExecContext(ctx, `
|
||||||
|
INSERT INTO retrieval_state (
|
||||||
|
book_id, chapter, chunk_idx, snapshot_id,
|
||||||
|
n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted,
|
||||||
|
embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids, updated_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
|
||||||
|
ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET
|
||||||
|
snapshot_id = excluded.snapshot_id,
|
||||||
|
n_exact_hits = excluded.n_exact_hits,
|
||||||
|
n_sticky = excluded.n_sticky,
|
||||||
|
n_ambiguous_flagged = excluded.n_ambiguous_flagged,
|
||||||
|
n_spoiler_blocked = excluded.n_spoiler_blocked,
|
||||||
|
n_evicted = excluded.n_evicted,
|
||||||
|
embedding_tier_used = excluded.embedding_tier_used,
|
||||||
|
n_postcheck_miss = excluded.n_postcheck_miss,
|
||||||
|
postcheck_detail = excluded.postcheck_detail,
|
||||||
|
injected_ids = excluded.injected_ids,
|
||||||
|
updated_at = excluded.updated_at`,
|
||||||
|
rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID,
|
||||||
|
rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted,
|
||||||
|
rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRetrievalState returns the record for a chunk, or nil if none.
|
||||||
|
func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*RetrievalState, error) {
|
||||||
|
ctx, cancel := opContext()
|
||||||
|
defer cancel()
|
||||||
|
rs := RetrievalState{BookID: bookID, Chapter: chapter, ChunkIdx: chunkIdx}
|
||||||
|
err := s.r.QueryRowContext(ctx, `
|
||||||
|
SELECT snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked,
|
||||||
|
n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids
|
||||||
|
FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`,
|
||||||
|
bookID, chapter, chunkIdx).Scan(
|
||||||
|
&rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked,
|
||||||
|
&rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &rs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetrievalStatesForBook returns every retrieval-state record for a book, ordered for
|
||||||
|
// a stable report (the memory section of tmctl report).
|
||||||
|
func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error) {
|
||||||
|
ctx, cancel := opContext()
|
||||||
|
defer cancel()
|
||||||
|
rows, err := s.r.QueryContext(ctx, `
|
||||||
|
SELECT chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged,
|
||||||
|
n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids
|
||||||
|
FROM retrieval_state WHERE book_id = ?
|
||||||
|
ORDER BY chapter, chunk_idx`, bookID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []RetrievalState
|
||||||
|
for rows.Next() {
|
||||||
|
rs := RetrievalState{BookID: bookID}
|
||||||
|
if err := rows.Scan(&rs.Chapter, &rs.ChunkIdx, &rs.SnapshotID, &rs.NExactHits, &rs.NSticky,
|
||||||
|
&rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed,
|
||||||
|
&rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, rs)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolToInt(b bool) int {
|
||||||
|
if b {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
153
backend/internal/store/glossary_test.go
Normal file
153
backend/internal/store/glossary_test.go
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
package store
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func gEntry(src, dst, status string, aliases ...string) GlossaryEntry {
|
||||||
|
e := GlossaryEntry{BookID: "book", Src: src, Dst: dst, Status: status, Source: "seed"}
|
||||||
|
for _, a := range aliases {
|
||||||
|
e.Aliases = append(e.Aliases, GlossaryAlias{Alias: a, AliasType: "прозвище"})
|
||||||
|
}
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplaceGlossaryIdempotentAndScoped(t *testing.T) {
|
||||||
|
s, _ := openTemp(t)
|
||||||
|
entries := []GlossaryEntry{
|
||||||
|
gEntry("阿Q", "А-кью", "approved"),
|
||||||
|
gEntry("四叔", "Четвёртый дядюшка", "approved", "鲁四老爷"),
|
||||||
|
}
|
||||||
|
if err := s.ReplaceGlossary("book", entries); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Another book is untouched.
|
||||||
|
if err := s.ReplaceGlossary("other", []GlossaryEntry{{BookID: "other", Src: "x", Dst: "икс", Status: "approved"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got1, err := s.GlossaryForBook("book")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got1) != 2 {
|
||||||
|
t.Fatalf("want 2 entries, got %d", len(got1))
|
||||||
|
}
|
||||||
|
// Re-replacing the identical set yields the identical content (idempotent).
|
||||||
|
if err := s.ReplaceGlossary("book", entries); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got2, err := s.GlossaryForBook("book")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got2) != 2 {
|
||||||
|
t.Fatalf("re-replace changed count: %d", len(got2))
|
||||||
|
}
|
||||||
|
// The alias survived the replace and attached to the right term.
|
||||||
|
var sishu *GlossaryEntry
|
||||||
|
for i := range got2 {
|
||||||
|
if got2[i].Src == "四叔" {
|
||||||
|
sishu = &got2[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if sishu == nil || len(sishu.Aliases) != 1 || sishu.Aliases[0].Alias != "鲁四老爷" {
|
||||||
|
t.Fatalf("alias not preserved: %+v", sishu)
|
||||||
|
}
|
||||||
|
// other book still has its one row.
|
||||||
|
other, _ := s.GlossaryForBook("other")
|
||||||
|
if len(other) != 1 {
|
||||||
|
t.Fatalf("scope leak: other book has %d rows", len(other))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplaceGlossaryRemovesDroppedRows(t *testing.T) {
|
||||||
|
s, _ := openTemp(t)
|
||||||
|
if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("阿Q", "А-кью", "approved"), gEntry("四叔", "дядюшка", "approved")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Drop one term: full replace must make it disappear (no phantom).
|
||||||
|
if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("阿Q", "А-кью", "approved")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _ := s.GlossaryForBook("book")
|
||||||
|
if len(got) != 1 || got[0].Src != "阿Q" {
|
||||||
|
t.Fatalf("dropped term still present: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGlossaryRevisionLogged(t *testing.T) {
|
||||||
|
s, _ := openTemp(t)
|
||||||
|
if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("王胡", "Бородатый Ван", "approved")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Change the approved rendering: a B1 editorial revision must be journaled.
|
||||||
|
if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("王胡", "Ван Ху", "approved")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
revs, err := s.GlossaryRevisionsForBook("book")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(revs) != 1 || revs[0].OldDst != "Бородатый Ван" || revs[0].NewDst != "Ван Ху" {
|
||||||
|
t.Fatalf("revision not logged correctly: %+v", revs)
|
||||||
|
}
|
||||||
|
// Re-replacing identical does NOT add a second revision.
|
||||||
|
if err := s.ReplaceGlossary("book", []GlossaryEntry{gEntry("王胡", "Ван Ху", "approved")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
revs, _ = s.GlossaryRevisionsForBook("book")
|
||||||
|
if len(revs) != 1 {
|
||||||
|
t.Fatalf("idempotent replace added a spurious revision: %d", len(revs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUniqueSrcSenseWindow(t *testing.T) {
|
||||||
|
s, _ := openTemp(t)
|
||||||
|
// Same src+sense+window twice → UNIQUE violation (B2).
|
||||||
|
dup := []GlossaryEntry{gEntry("道", "путь", "approved"), gEntry("道", "дао", "approved")}
|
||||||
|
if err := s.ReplaceGlossary("book", dup); err == nil {
|
||||||
|
t.Fatal("expected UNIQUE(src,sense,window) violation for duplicate src+sense+window")
|
||||||
|
}
|
||||||
|
// Same src, different SENSE → allowed (polysemy, A3).
|
||||||
|
ok := []GlossaryEntry{
|
||||||
|
{BookID: "book", Src: "道", Dst: "путь", Sense: "way", Status: "approved"},
|
||||||
|
{BookID: "book", Src: "道", Dst: "дао", Sense: "dao", Status: "approved"},
|
||||||
|
}
|
||||||
|
if err := s.ReplaceGlossary("book", ok); err != nil {
|
||||||
|
t.Fatalf("distinct senses should be allowed: %v", err)
|
||||||
|
}
|
||||||
|
got, _ := s.GlossaryForBook("book")
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("want 2 polysemy rows, got %d", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetrievalStateUpsert(t *testing.T) {
|
||||||
|
s, _ := openTemp(t)
|
||||||
|
rs := RetrievalState{BookID: "book", Chapter: 1, ChunkIdx: 0, SnapshotID: "snap",
|
||||||
|
NExactHits: 3, NAmbiguousFlagged: 1, NSpoilerBlocked: 1, NPostcheckMiss: 1,
|
||||||
|
PostcheckDetail: `[{"src":"阿Q","dst":"А-кью"}]`, InjectedIDs: `["ah_q"]`}
|
||||||
|
if err := s.UpsertRetrievalState(rs); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := s.GetRetrievalState("book", 1, 0)
|
||||||
|
if err != nil || got == nil {
|
||||||
|
t.Fatalf("get: %v %v", got, err)
|
||||||
|
}
|
||||||
|
if got.NExactHits != 3 || got.NSpoilerBlocked != 1 || got.NPostcheckMiss != 1 {
|
||||||
|
t.Fatalf("roundtrip mismatch: %+v", got)
|
||||||
|
}
|
||||||
|
// Overwrite (resolve semantics on resume): new counts win.
|
||||||
|
rs.NExactHits = 5
|
||||||
|
rs.NPostcheckMiss = 0
|
||||||
|
if err := s.UpsertRetrievalState(rs); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _ = s.GetRetrievalState("book", 1, 0)
|
||||||
|
if got.NExactHits != 5 || got.NPostcheckMiss != 0 {
|
||||||
|
t.Fatalf("overwrite failed: %+v", got)
|
||||||
|
}
|
||||||
|
all, _ := s.RetrievalStatesForBook("book")
|
||||||
|
if len(all) != 1 {
|
||||||
|
t.Fatalf("want 1 retrieval-state row, got %d", len(all))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -161,6 +161,98 @@ var migrations = []string{
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS ruby_readings_book_idx ON ruby_readings (book_id);
|
CREATE INDEX IF NOT EXISTS ruby_readings_book_idx ON ruby_readings (book_id);
|
||||||
`,
|
`,
|
||||||
|
// v5: the memory bank v2 glossary (шаг 4, registry architecture/06 §Гейты + D7).
|
||||||
|
// The DETERMINISTIC substrate the hot-path matcher, the injection surface and the
|
||||||
|
// post-check are built on. Rows are the frozen-at-job-start injectable pool whose
|
||||||
|
// approved subset the F1 memoryVersion() hashes into the snapshot. Like ruby_readings
|
||||||
|
// this is "dumb storage" (aggregation/classification live in the pipeline); the
|
||||||
|
// pipeline REPLACES a book's whole glossary from its deterministic inputs (seed file
|
||||||
|
// + classified ruby), so re-seeding is idempotent and a seed edit converges every
|
||||||
|
// column. No cross-table FK (mirrors the ruby/chunkstatus dumb-storage style): both
|
||||||
|
// child tables are scoped by book_id and replaced in the same transaction, term_id is
|
||||||
|
// a logical link (glossary.id is a fresh autoincrement each replace and is NEVER hashed
|
||||||
|
// into memoryVersion — only content columns are, ORDER BY-stable).
|
||||||
|
`
|
||||||
|
CREATE TABLE IF NOT EXISTS glossary (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
book_id TEXT NOT NULL,
|
||||||
|
src TEXT NOT NULL, -- source key (raw; normalized in Go for matching, §3.6)
|
||||||
|
dst TEXT NOT NULL DEFAULT '', -- approved Russian translation (LEMMA/base form); '' for a ruby candidate with no dst yet
|
||||||
|
type TEXT NOT NULL DEFAULT '', -- name|term|title|… alias-graph node type (B3)
|
||||||
|
sense TEXT NOT NULL DEFAULT '', -- A3 polysemy disambiguator (part of the uniqueness key)
|
||||||
|
gender TEXT NOT NULL DEFAULT '', -- male|female|hidden|'' (C2; hidden = gender-avoidant rendering until reveal)
|
||||||
|
speech TEXT NOT NULL DEFAULT '', -- ты/вы register note (series-bible-lite; the transition JOURNAL is deferred)
|
||||||
|
decl TEXT NOT NULL DEFAULT '', -- JSON {"invariant":bool,"forms":[...]}: the dst declension forms the post-check accepts (E1/E2 safety, decl = grammaticality mechanism, not a market feature)
|
||||||
|
translit_policy TEXT NOT NULL DEFAULT '', -- B5 western-name-via-katakana (field only in v1; validator = Phase 2)
|
||||||
|
first_person TEXT NOT NULL DEFAULT '', -- D7 first_person field (field only)
|
||||||
|
nickname_translation TEXT NOT NULL DEFAULT '', -- D7 (keep in v1: a column + a "translate once" lock)
|
||||||
|
since_ch INTEGER NOT NULL DEFAULT 0, -- spoiler window start; 0 = valid from the beginning (C1 hard reject-gate below since_ch)
|
||||||
|
until_ch INTEGER NOT NULL DEFAULT 0, -- spoiler window end; 0 = no end (valid forever)
|
||||||
|
status TEXT NOT NULL DEFAULT 'auto',-- auto|draft|approved (term status machine; only approved is CONFIRMED-injected)
|
||||||
|
allow_short INTEGER NOT NULL DEFAULT 0, -- rare escape hatch: permit a below-min_key_len key (A3 single-key ban override)
|
||||||
|
source TEXT NOT NULL DEFAULT '', -- provenance: seed|ruby|auto (who created the row)
|
||||||
|
ruby_reading TEXT NOT NULL DEFAULT '', -- ruby-seeded rows: the furigana reading (bridge to Polivanov dst validation, B6 Phase 2 — NOT a dst)
|
||||||
|
ruby_class TEXT NOT NULL DEFAULT '', -- name|gloss|ambiguous (ruby classifier §8: phonetic-name → name-lock candidate, unrelated reading → gloss/footnote, else flag)
|
||||||
|
confidence INTEGER NOT NULL DEFAULT 0, -- ruby occurrences count = confidence prior (NOT a direct promotion to approved)
|
||||||
|
note TEXT NOT NULL DEFAULT '',
|
||||||
|
-- B2: at most one row per (source key, sense, spoiler window). since_ch/until_ch are
|
||||||
|
-- NOT NULL with default 0, so the SQLite "NULLs are distinct in UNIQUE" gotcha cannot
|
||||||
|
-- silently admit a duplicate. A term whose approved rendering changes across a spoiler
|
||||||
|
-- boundary is two rows with different windows (legitimately distinct), not a collision.
|
||||||
|
UNIQUE (book_id, src, sense, since_ch, until_ch)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS glossary_book_idx ON glossary (book_id, status);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS glossary_aliases (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
book_id TEXT NOT NULL,
|
||||||
|
term_id INTEGER NOT NULL, -- logical link to glossary.id within the SAME replace tx (no hard FK — dumb storage, scoped+replaced by book_id)
|
||||||
|
alias TEXT NOT NULL, -- an alternative source surface (raw; normalized in Go for matching)
|
||||||
|
alias_type TEXT NOT NULL DEFAULT '', -- имя|цзы|хао|прозвище|титул (B3 alias-graph node type)
|
||||||
|
UNIQUE (book_id, term_id, alias)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS glossary_aliases_term_idx ON glossary_aliases (term_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS glossary_aliases_book_idx ON glossary_aliases (book_id);
|
||||||
|
|
||||||
|
-- B1 editorial-time axis, ORTHOGONAL to the spoiler window (since_ch/until_ch is the
|
||||||
|
-- STORY timeline; this is the EDITORIAL timeline). Append-only journal of approved-dst
|
||||||
|
-- changes so a stale first-translation-wins rendering is DETECTABLE (DelTA PNR generates
|
||||||
|
-- stale/drift; this is the substrate that catches it). Not injected → not in any hash;
|
||||||
|
-- datetime('now') here is fine (telemetry, like request_log), never feeds memoryVersion.
|
||||||
|
CREATE TABLE IF NOT EXISTS glossary_revisions (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
book_id TEXT NOT NULL,
|
||||||
|
src TEXT NOT NULL,
|
||||||
|
sense TEXT NOT NULL DEFAULT '',
|
||||||
|
old_dst TEXT NOT NULL DEFAULT '',
|
||||||
|
new_dst TEXT NOT NULL,
|
||||||
|
editorial_ts TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
reason TEXT NOT NULL DEFAULT ''
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS glossary_revisions_book_idx ON glossary_revisions (book_id, src);
|
||||||
|
|
||||||
|
-- Per-chunk retrieval-state (registry gate #4 / A1/F2/G2/E1): observability of memory
|
||||||
|
-- degradation FROM DAY ONE — the mechanism that converts silent degradation into loud.
|
||||||
|
-- Recomputed deterministically each run (the matcher is $0), so it self-heals on resume;
|
||||||
|
-- NOT wire-load-bearing (it records what the injection did, it does not change the request).
|
||||||
|
CREATE TABLE IF NOT EXISTS retrieval_state (
|
||||||
|
book_id TEXT NOT NULL,
|
||||||
|
chapter INTEGER NOT NULL,
|
||||||
|
chunk_idx INTEGER NOT NULL,
|
||||||
|
snapshot_id TEXT NOT NULL,
|
||||||
|
n_exact_hits INTEGER NOT NULL DEFAULT 0, -- entries matched by an exact key/alias in this chunk
|
||||||
|
n_sticky INTEGER NOT NULL DEFAULT 0, -- entries carried by scene-inertia (A5), not re-matched here
|
||||||
|
n_ambiguous_flagged INTEGER NOT NULL DEFAULT 0, -- injected AMBIGUOUS (auto/draft) → "unverified" + forced post-check (A2)
|
||||||
|
n_spoiler_blocked INTEGER NOT NULL DEFAULT 0, -- matched but hard-rejected by the since_ch/until_ch window (C1)
|
||||||
|
n_evicted INTEGER NOT NULL DEFAULT 0, -- dropped by the token budget (F2 — logged, never silent)
|
||||||
|
embedding_tier_used INTEGER NOT NULL DEFAULT 0, -- always 0 in v1 (no embeddings on the hot path, Р3)
|
||||||
|
n_postcheck_miss INTEGER NOT NULL DEFAULT 0, -- E1 flags: a confirmed src matched but no accepted dst form is in the output
|
||||||
|
postcheck_detail TEXT NOT NULL DEFAULT '', -- JSON [{src,dst}] of the misses, for the report / human
|
||||||
|
injected_ids TEXT NOT NULL DEFAULT '', -- JSON of exact-matched entity keys (audit + sticky reconstruction)
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
PRIMARY KEY (book_id, chapter, chunk_idx)
|
||||||
|
);
|
||||||
|
`,
|
||||||
}
|
}
|
||||||
|
|
||||||
// migrate runs all pending migrations on the write pool, one transaction per
|
// migrate runs all pending migrations on the write pool, one transaction per
|
||||||
|
|
|
||||||
|
|
@ -430,6 +430,30 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор
|
||||||
|
|
||||||
Отсеянная находка (oversize-хвост <500) осталась документированной границей §7b. Дальше — банк памяти v2 (шаг 4): реестр рисков `architecture/06` + ruby-контракт готовы (F1-materialization + горячий путь + post-check). Онбординг-промт след. сессии — `docs/BACKEND_SESSION_PROMPT_MEMORY.md`.
|
Отсеянная находка (oversize-хвост <500) осталась документированной границей §7b. Дальше — банк памяти v2 (шаг 4): реестр рисков `architecture/06` + ruby-контракт готовы (F1-materialization + горячий путь + post-check). Онбординг-промт след. сессии — `docs/BACKEND_SESSION_PROMPT_MEMORY.md`.
|
||||||
|
|
||||||
|
### 2026-07-05 — Шаг 4: банк памяти v2 (глоссарий + горячий путь + инъекция + F1 + post-check)
|
||||||
|
|
||||||
|
**Главный блокер качества (консистентность имён/терминов) реализован — веха A–E на seeded-глоссарии** (граница вехи подтверждена оркестратором §8: механизм на ручном+ruby-сиде; автопопуляция/адъюдикация/резюме/series-bible — отдельные вехи). Всё зелёное `go build/vet/test ./... -race` (125 тестов в pipeline), `tmctl report` $0. Ратифицированный порядок реестра `architecture/06` §Гейты соблюдён.
|
||||||
|
|
||||||
|
**A4 нормализация (`memnorm.go` + `data/trad2simp.txt`):** NFKC + trad→simp (вендорённая таблица, **версия = хеш содержимого таблицы** → дополнение = громкий `--resnapshot`, не тихая мини-карта) + катакана→хирагана + lower для источника; NFC + lower + ё→е + **схлопывание whitespace + унификация тире** для ru-цели. Симметрично ключ↔текст. Полный OpenCC оффлайн недоступен (нет данных на стенде) → курированный высокочастотный набор, все referenced/live-bug кейсы (爺→爷 и др.) покрыты и заассерчены; **завершение полного OpenCC — единственный отслеживаемый data-drop перед zh-приёмкой** (приёмочная книга D9 — ja→ru, где trad→simp не несущий).
|
||||||
|
|
||||||
|
**A схема (миграция v5):** `glossary` (D7 + `sense`/ось редакционного времени `glossary_revisions`/`UNIQUE(src,sense,window)`/`min_key_len`-запрет одиночных ключей/`allow_short`) + `glossary_aliases` (alias-граф) + `glossary_revisions` (B1 append-only) + `retrieval_state` (гейт #4). Idempotent full-replace (как ruby). `store/glossary.go`.
|
||||||
|
|
||||||
|
**B горячий путь (`memory.go`):** frozen `MemoryBank` из store-строк; **Aho-Corasick** по нормализованным ключам; запрет одиночных ключей (A3); longest-match containment; спойлер-hard-reject (C1, `since_ch/until_ch`, 0=безграничен); sticky scene-inertia (A5); токен-бюджет с логируемым вытеснением (F2); трёхсторонний disposition (A2 confirmed/ambiguous/reject). Детерминизм: 0 map-order в выводе, T1–T10 портированы (**не accept-регэкспы**), ловушки `retrieval_bench` 0/N, 200× determinism. **НЕ FTS5/вектор/эмбеддинги** (Р3).
|
||||||
|
|
||||||
|
**C инъекция (`render.go`):** `MessagesWithInjection` — код-собранный `system`-месседж глоссария МЕЖДУ стабильным префиксом (кэш-граница) и user-чанком (вариант «а» оркестратора, шаблон не трогаем); НЕ в `ch.Text` (coverage/`{{text}}` чисты); свёрнут в `request_hash`/`content_hash`.
|
||||||
|
|
||||||
|
**D закрыт F1 (`runner.go`):** `memoryVersion()` = content-hash замороженных **approved**-строк + версии норм/матч-алгоритмов (D8, не version-counter). Смена approved → громкий `--resnapshot` (e2e доказал `1e7b…`→`552c…`). id не хешируется; auto/draft вне хеша (их инъекция ловится `content_hash`).
|
||||||
|
|
||||||
|
**E post-check (`mempostcheck.go`) + E1-замер:** decl-осознанный (**наивный регэксп = 18–36% ложных, research/14**), whole-word матч по записанным `decl`-формам, sticky исключён, слепой post-replace запрещён (E2). **Флаггер по умолчанию** (миссы → `retrieval_state`, видны в `tmctl report`), **жёсткий гейт opt-in** (`gates.glossary.postcheck_gate`, как coverage — флип после валидации владельцем). **E1 измерен как Go-тест на реальном склонённом русском (канон Рогова): full-decl 0% ложных, base-only 67%** — эмпирическое подтверждение, что stored-decl надёжен ТОЛЬКО при полноте (ответ на развилку stored-decl-в-Go vs pymorphy-сайдкар Ф2: держим флаггером, сайдкар — валидированный фолбэк если реальные книги зашумят).
|
||||||
|
|
||||||
|
**Ruby→глоссарий (`memseed.go`):** классификатор `name_candidate`/`gloss_candidate`/`ambiguous` — **никогда не авто-лочит** (блайнд-лок = mistranslation; 強敵→とも неотличим от имени оффлайн → кандидат человеку). Ruby → auto-кандидаты (src=base, reading=мост к Поливанову B6, без dst), дедуп по base, скип manual-src.
|
||||||
|
|
||||||
|
**Интеграция + config:** `TranslateBook`: ingest→persistRuby→seedGlossary(replace+materialize)→snapshot→loop с sticky-окном (ресет на границе главы, пересобирается детерминированно на resume). `book.yaml: glossary_seed`. `tmctl report` — секция ПАМЯТЬ (агрегаты + пост-check миссы). Пост-check валидирует **финальный (редакторский) вывод** — то, что экспортируется, не только черновик.
|
||||||
|
|
||||||
|
**Агентское адверсариальное селфревью (§6, многоагентный Workflow):** 7 дименсий → 10 находок → независимая refute-by-default верификация с прогоном реального кода → **8 CONFIRMED (все воспроизведены), 2 REFUTED** (документированные размены: min-key-2 гомографы; редакторский drift — закрыт пост-check'ом финала). Все 8 исправлены + регресс-тесты **мутационно-проверены** (реверт → падает именно его тест): #1 empty-dst не матчится; #2/#8 whitespace-wrap/тире в post-check; #3 whole-word матч (дропнутый «Ван» не маскируется «караван»); #4 gate-mode `memoryVersion` сворачивает draft-decl (иначе resume молча флипал чанк); #5 дубль-alias в сиде дедупится (не крашит книгу); #6 спойлер-фильтр перед longest-match (спойлер-блокнутый длинный ключ не глотает валидное короткое имя); #7 floor-free токен-бюджет.
|
||||||
|
|
||||||
|
**Заложено/отложено (реестр §5, поля/место есть, реализация — Ф2/веха+):** Палладий/Поливанов B6 (reading как мост заложен), гейт фактичности резюме D1, эмбеддинг-слой A7/D2, морфо-гейт глагольного рода C3 (pymorphy-сайдкар), бэкап файла F4, автопопуляция/адъюдикация G1/F5, series-bible ты/вы-журнал, инъекция глоссария в редактор (dst-констрейнты) — fast-follow на той же поверхности. Ставка (селективная инъекция vs длинный контекст) гейтится in-house eval'ом пилота Ф2.5 (параллельно, не блокер). Правки только в коде `backend/` + этот журнал; `architecture/06`/`research/*` не трогались.
|
||||||
|
|
||||||
## Полигон
|
## Полигон
|
||||||
(секция параллельной сессии — записи добавлять сюда)
|
(секция параллельной сессии — записи добавлять сюда)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue