diff --git a/eval/extract_bench.py b/eval/extract_bench.py new file mode 100644 index 0000000..1aa95e2 --- /dev/null +++ b/eval/extract_bench.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Экстракция терминов — БЕНЧМАРК локальных vs облачных моделей (write-path банка памяти, реестр G1). + +Локалка касается банка памяти в 2 точках: эмбеддинги (retrieval_bench.py — bge-m3) и +ЭКСТРАКЦИЯ кандидатов в глоссарий из сырого текста (bootstrap write-path). Полигон exp03 +мерил локалки как ПЕРЕВОДЧИКОВ (не годятся); экстракция — другая, более лёгкая задача. +Здесь мерим её нормально: много моделей, много кейсов, разные языки/эдж-кейсы, и КАЖДЫЙ +ответ верифицируется на здоровость (пустой/echo/config-error → ретрай и явная пометка +FAILED, НЕ засчитывается как recall 0 — урок предыдущего обрывочного прогона). + +Запросы к провайдерам — через `refusal_bench.call_provider` + канонический `providers.json` +(квирки уже верны: deepseek thinking-ON+max_tokens 8000, kimi temp=1/24k, gpt-5-mini +max_completion_tokens+no-temp, gemini/glm thinking-off). Локаль — ollama native (think:false). +Плюс вариант deepseek-nothink (`thinking:disabled`) — дешёвая экстракция без reasoning. + +Метрики (ДЕТЕРМИНИРОВАННЫЕ, без судьи — чтобы «верифицированно и точно»): + recall — доля эталонных сущностей, найденных (спот) + render_ok — доля найденных gold, где предложенный RU совпал с каноном (Рогов/Поливанов/Палладий/устоявш.) + halluc — доля извлечённого, чего НЕТ в исходном тексте (явный FP, детерминированно) + health — ok / empty / http / refused / unparseable (ретраи; в агрегат идут только ok) + +Запуск: eval/.venv/bin/python eval/extract_bench.py [--models a,b] [--cases zh,ja,en] +Выход: eval/data/extract_bench/results.json + таблица. +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import re +import time +import unicodedata +import urllib.request +from pathlib import Path + +from dotenv import dotenv_values + +ROOT = Path(__file__).resolve().parent +OUT = ROOT / "data" / "extract_bench" +ENV = dotenv_values(ROOT / ".env") +for k, v in ENV.items(): # call_provider читает ключи из os.environ + os.environ.setdefault(k, v or "") +NOPROXY = urllib.request.build_opener(urllib.request.ProxyHandler({})) + +# reuse verified request machinery +_spec = importlib.util.spec_from_file_location("rb", ROOT / "refusal_bench.py") +rb = importlib.util.module_from_spec(_spec); _spec.loader.exec_module(rb) +PROVIDERS = {p["name"]: p for p in json.loads((ROOT / "providers.json").read_text())["providers"]} + +CLOUD = ["deepseek", "grok", "glm", "kimi", "gemini", "openai"] +CLOUD_EXTRA = { # deepseek без thinking — дешёвая экстракция (echo бьёт перевод, не JSON?) + "deepseek-nothink": dict(PROVIDERS["deepseek"], name="deepseek-nothink", + extra_body={"thinking": {"type": "disabled"}}), +} +LOCAL = ["huihui_ai/qwen3-abliterated:8b", "huihui_ai/qwen3.5-abliterated:9b", + "qwen3:8b", "qwen3-vojo:latest", "ruadapt-qwen3:8b", + "huihui_ai/qwen3-abliterated:30b-a3b"] + +SYS = "Ты — терминолог. Извлекаешь из текста имена собственные и культурные реалии для глоссария художественного перевода на русский." +USER = ("Извлеки из фрагмента ВСЕ имена собственные (персонажи, места, организации) и " + "культурно-специфичные реалии/титулы. Для каждого верни: src (ровно как в тексте), " + "type (character|place|org|realia|title), dst (перевод/транскрипция на русский по канону). " + 'Верни СТРОГО JSON без пояснений: {"terms":[{"src":"...","type":"...","dst":"..."}]}.\n\nФрагмент:\n<<>>') + +# ---- корпус: реальные PD-чанки, разные языки/эдж-кейсы, gold с каноническим RU (accept-регэксп) ---- +CASES = [ + # zh — имена + реалии + алиас + dict(id="zh-ahq", lang="zh", + text="我要给阿Q做正传,已经不止一两年了。那是赵太爷的儿子进了秀才的时候,锣声铛铛的报到村里来,阿Q正喝了两碗黄酒,因为他和赵太爷原来是本家。", + gold={"阿Q": r"А[-\s]?[Кк]ью", "赵太爷": r"Чжао", "秀才": r"сюцай"}), + dict(id="zh-zhufu", lang="zh", + text="接着一声钝响,是送灶的爆竹。我是正在这一夜回到我的故乡鲁镇的,暂寓在鲁四老爷的宅子里,应该称之曰「四叔」。家中都在准备着「祝福」,这是鲁镇年终的大典,迎接福神。", + gold={"送灶": r"очаг|Цзао", "鲁镇": r"Лучжэн", "鲁四老爷": r"дядюшк|[Лл]у\b|Лу[ -]", "祝福": r"жертвоприношен|благословени|[Мм]олени|счасть"}), + dict(id="zh-xianglin", lang="zh", + text="况且,一想到昨天遇见祥林嫂的事,也就使我不能安住。我在鲁镇所见的人们中,改变之大,可以说无过于她的了;她分明已经纯乎是一个乞丐了。", + gold={"祥林嫂": r"Сянлин", "鲁镇": r"Лучжэн"}), + dict(id="zh-wanghu", lang="zh", # алиас: 王胡 = 王癞胡 + text="他看见王胡在那里赤着膊捉虱子。这王胡,又癞又胡,别人都叫他王癞胡,阿Q却删去了一个癞字,然而非常渺视他。", + gold={"王胡": r"Ван|Бородат", "阿Q": r"А[-\s]?[Кк]ью"}), + # ja — имена + реалии + dict(id="ja-rashomon", lang="ja", + text="一人の下人が、羅生門の下で雨やみを待っていた。羅生門が、朱雀大路にある以上は、この男のほかにも、雨やみをする者がありそうなものである。", + gold={"下人": r"слуг|гэнин", "羅生門": r"Рас[её]мон", "朱雀大路": r"Судзаку"}), + dict(id="ja-kyoto", lang="ja", + text="この二三年、京都には、地震とか辻風とか火事とか饑饉とか云う災がつづいて起った。そこで洛中のさびれ方は一通りではない。", + gold={"京都": r"Киото", "洛中": r"Ракутю|Лочжун|столиц|город"}), + # ja — западные имена через катакану (translit_policy эдж-кейс) + dict(id="ja-melos", lang="ja", + text="メロスは激怒した。メロスは、村の牧人である。きょう未明メロスは村を出発し、十里はなれた此のシラクスの市にやって来た。", + gold={"メロス": r"Мелос", "シラクス": r"Сиракуз"}), + dict(id="ja-seri", lang="ja", # длинное западное имя через катакану + text="メロスには竹馬の友があった。セリヌンティウスである。今は此のシラクスの市で、石工をしている。", + gold={"セリヌンティウス": r"Селинунти", "メロス": r"Мелос", "シラクス": r"Сиракуз"}), + # ja — буддийское имя + топоним (эталон Стругацкого) + dict(id="ja-hana", lang="ja", + text="禅智内供の鼻と云えば、池尾で知らない者はない。長さは五六寸あって上唇の上から顋の下まで下っている。", + gold={"禅智内供": r"Дзэн[тч]и|Найгу", "池尾": r"Икэ"}), + # en — западные имена → русская транскрипция (render-эдж) + dict(id="en-carter", lang="en", + text="My name is John Carter; I am better known as Captain Jack Carter of Virginia. I spent nearly a year prospecting in company with another Confederate officer, Captain James K. Powell of Richmond, in an Arizona cave.", + gold={"John Carter": r"Джон\s?Картер", "Virginia": r"Виргини|Вирджини", "Powell": r"Пауэл", "Richmond": r"Ричмонд", "Arizona": r"Аризон"}), + dict(id="en-conan", lang="en", + text="'I am Conan, a Cimmerian,' he answered. 'I came into Argos seeking employment, but with no wars forward, there was nothing to which I might turn my hand.'", + gold={"Conan": r"Конан", "Cimmerian": r"[Кк]иммери", "Argos": r"Аргос"}), + dict(id="en-wells", lang="en", + text="The Time Traveller (for so it will be convenient to speak of him) was expounding a recondite matter to us. 'You must follow me carefully,' said Filby.", + gold={"Time Traveller": r"[Пп]утешественник", "Filby": r"Филби"}), +] + + +def norm(s): + return unicodedata.normalize("NFKC", str(s)).lower() + + +def parse_terms(txt): + txt = re.sub(r".*?\s*", "", txt or "", flags=re.S) + m = re.search(r"\{.*\}|\[.*\]", txt, re.S) + if not m: + return None # unparseable + try: + d = json.loads(m.group(0)) + except Exception: + return None + terms = d.get("terms", d) if isinstance(d, dict) else d + if not isinstance(terms, list): + return [] + return [{"src": str(t["src"]), "type": str(t.get("type", "")), "dst": str(t.get("dst", ""))} + for t in terms if isinstance(t, dict) and t.get("src")] + + +def call_local(model, text, max_pred=4000): + payload = dict(model=model, stream=False, think=False, format="json", + options=dict(num_ctx=8192, temperature=0.2, num_predict=max_pred), + messages=[{"role": "system", "content": SYS}, + {"role": "user", "content": USER.replace("<<>>", text)}]) + req = urllib.request.Request("http://localhost:11434/api/chat", + data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) + with NOPROXY.open(req, timeout=600) as r: + d = json.load(r) + return (d.get("message", {}) or {}).get("content", "") or "", d.get("done_reason", "") + + +def healthy_extract(kind, model, case): + """Возвращает (terms|None, health, dt). Ретраит пустой ответ с бо́льшим бюджетом.""" + text = case["text"] + for attempt in range(3): + t0 = time.time() + try: + if kind == "local": + raw, fin = call_local(model, text, max_pred=4000 * (attempt + 1)) + err = None if raw.strip() else f"empty|finish={fin}" + else: + pmap = CLOUD_EXTRA.get(model) or PROVIDERS[model] + if attempt: # ретрай пустого — поднять бюджет + pmap = dict(pmap, max_tokens=int(pmap.get("max_tokens", 8000) * 1.7)) + raw, err, _ = rb.call_provider(pmap, SYS, USER.replace("<<>>", text)) + raw = raw or "" + except Exception as e: + return None, f"transport:{type(e).__name__}", round(time.time() - t0, 1) + dt = round(time.time() - t0, 1) + if err: + kindn = err.split("|", 1)[0] + if kindn == "content_filter": + return None, "refused", dt + if kindn in ("empty",): + continue # ретрай + return None, f"http:{err[:40]}", dt + terms = parse_terms(raw) + if terms is None: + if attempt < 2: + continue # unparseable → ретрай + return None, "unparseable", dt + return terms, "ok", dt + return None, "empty", dt + + +def score(case, terms): + ntext = norm(case["text"]) + found, render = {}, {} + for g, accept in case["gold"].items(): + ng = norm(g) + hit = next((t for t in terms if ng in norm(t["src"]) or (norm(t["src"]) in ng and len(norm(t["src"])) >= 2)), None) + found[g] = hit is not None + render[g] = bool(hit and re.search(accept, hit["dst"])) + ex = [t["src"] for t in terms] + halluc = [s for s in ex if norm(s) and norm(s) not in ntext] + return dict(recall=sum(found.values()) / len(found), + render_ok=(sum(render.values()) / max(1, sum(found.values())) if any(found.values()) else 0.0), + halluc_rate=round(len(halluc) / len(ex), 2) if ex else 0.0, + n=len(terms), miss=[g for g, ok in found.items() if not ok], + bad_render=[g for g in found if found[g] and not render[g]], + halluc=halluc[:5], raw=[f"{t['src']}→{t['dst']}" for t in terms][:14]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--models", default="") + ap.add_argument("--cases", default="") + args = ap.parse_args() + OUT.mkdir(parents=True, exist_ok=True) + cases = [c for c in CASES if not args.cases or c["lang"] in args.cases.split(",")] + models = ([("local", m) for m in LOCAL] + [("cloud", m) for m in CLOUD] + + [("cloud", "deepseek-nothink")]) + if args.models: + want = set(args.models.split(",")) + models = [(k, m) for k, m in models if m in want] + + jsonl = OUT / "records.jsonl" + done = set() + if jsonl.exists(): + for ln in jsonl.read_text().splitlines(): + if ln.strip(): + r = json.loads(ln); done.add((r["model"], r["case"])) + fh = jsonl.open("a", encoding="utf-8") + records = [json.loads(ln) for ln in (jsonl.read_text().splitlines() if jsonl.exists() else []) if ln.strip()] + + for kind, model in models: + print(f"\n### {model} ({kind})") + for case in cases: + if (model, case["id"]) in done: + continue + terms, health, dt = healthy_extract(kind, model, case) + rec = dict(model=model, kind=kind, case=case["id"], lang=case["lang"], health=health, dt=dt) + if health == "ok": + rec.update(score(case, terms)) + records.append(rec) + fh.write(json.dumps(rec, ensure_ascii=False) + "\n"); fh.flush() + if health == "ok": + print(f" {case['id']:<12} recall={rec['recall']:.2f} render={rec['render_ok']:.2f} " + f"halluc={rec['halluc_rate']} n={rec['n']} {dt}s" + + (f" miss={rec['miss']}" if rec['miss'] else "") + + (f" bad_render={rec['bad_render']}" if rec['bad_render'] else "")) + else: + print(f" {case['id']:<12} HEALTH={health} {dt}s") + fh.close() + + # ---- агрегаты: только health==ok идут в recall/render/halluc; health-счётчики отдельно ---- + summary = {} + for kind, model in models: + recs = [r for r in records if r["model"] == model] + ok = [r for r in recs if r["health"] == "ok"] + health_ct = {} + for r in recs: + health_ct[r["health"]] = health_ct.get(r["health"], 0) + 1 + if ok: + summary[model] = dict( + kind=kind, n_ok=len(ok), n_total=len(recs), health=health_ct, + recall=round(sum(r["recall"] for r in ok) / len(ok), 3), + render_ok=round(sum(r["render_ok"] for r in ok) / len(ok), 3), + halluc=round(sum(r["halluc_rate"] for r in ok) / len(ok), 3), + avg_n=round(sum(r["n"] for r in ok) / len(ok), 1), + avg_s=round(sum(r["dt"] for r in ok) / len(ok), 1)) + else: + summary[model] = dict(kind=kind, n_ok=0, n_total=len(recs), health=health_ct) + + print("\n=== СВОДКА (экстракция; агрегат по health==ok) ===") + print(f" {'model':<34}{'kind':<7}{'recall':>7}{'render':>7}{'halluc':>7}{'n_ex':>5}{'sec':>6}{'ok/all':>8}") + for m, s in summary.items(): + ratio = f"{s['n_ok']}/{s['n_total']}" + if s["n_ok"]: + print(f" {m:<34}{s['kind']:<7}{s['recall']:>7}{s['render_ok']:>7}{s['halluc']:>7}" + f"{s['avg_n']:>5}{s['avg_s']:>6}{ratio:>8}") + else: + print(f" {m:<34}{s['kind']:<7}{'—':>7}{'—':>7}{'—':>7}{'—':>5}{'—':>6}{ratio:>8} health={s['health']}") + print(" (health!=ok — пустой/echo/http/unparseable — НЕ засчитан как recall 0)") + (OUT / "results.json").write_text(json.dumps(dict(summary=summary, records=records), ensure_ascii=False, indent=2), encoding="utf-8") + print(f"\nsaved → {OUT/'results.json'}") + + +if __name__ == "__main__": + main() diff --git a/eval/memory_hotpath.py b/eval/memory_hotpath.py new file mode 100644 index 0000000..e14539f --- /dev/null +++ b/eval/memory_hotpath.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Референс-реализация ГОРЯЧЕГО ПУТИ банка памяти + self-tests. + +Не продукт — исполняемая СПЕКА механизма (по architecture/06 §Контракт горячего +пути и research/13 Q1/Q3/Q4), по которой бэкенд пишет Go. Проверяет, что сам +МЕХАНИЗМ работает как задумано, без бэкенда и без LLM-вызовов: + + normalize (NFKC + trad→simp + kana-fold) A4 «тихо-пусто» из-за орфографии + → multi-pattern точный матч ключей/алиасов Q3 detерм. > dense на именах + с запретом одиночных ключей (longest-match) A3 омографы 灶/炎/气/修 + → спойлер-фильтр since_ch/until_ch (hard) C1 утечка будущего факта + → sticky scene-inertia (местоименные чанки) A5 + → токен-бюджет с приоритетом + лог вытеснения F2 + → disposition: confirmed / ambiguous / reject A2 трёхстороннее решение + → post-check: утв. dst (по decl) в выходе? E1 ловит и утечку, и отравление + +Запуск: eval/.venv/bin/python eval/memory_hotpath.py (печатает PASS/FAIL по кейсам) +Прод-замены помечены [PROD]: OpenCC вместо мини-карты, Aho-Corasick вместо substring, +pymorphy/decl вместо accept-регэкспов. +""" +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass, field + +# --- [PROD] в бою — OpenCC (opencc: t2s); здесь мини-карта под демо-термины --- +TRAD2SIMP = {"魯": "鲁", "鎮": "镇", "趙": "赵", "蕭": "萧", "煉": "炼", "門": "门", "闆": "板"} + + +def normalize(s: str) -> str: + """NFKC (полу/полноширина) + trad→simp + катакана→хирагана. Симметрично к + ключам и тексту — иначе 魯鎮 ≠ 鲁镇 и запись «тихо не совпадёт» (A4).""" + s = unicodedata.normalize("NFKC", s) + s = "".join(TRAD2SIMP.get(c, c) for c in s) + out = [] + for c in s: + o = ord(c) + out.append(chr(o - 0x60) if 0x30A1 <= o <= 0x30F6 else c) # katakana→hiragana + return "".join(out).lower() + + +def cjk_len(s: str) -> int: + """Длина ключа в «значимых» знаках (CJK + буквы), для запрета одиночных ключей.""" + return len(re.findall(r"[぀-ヿ㐀-鿿豈-﫿A-Za-zЀ-ӿ]", s)) + + +@dataclass +class Entry: + id: str + src: str + dst: str + accept: str # регэксп утв. формы в РУ-выходе ([PROD] через decl/лемматизатор) + aliases: list = field(default_factory=list) + status: str = "approved" # approved | auto | draft + sense: str = "" + since_ch: int | None = None # запись валидна с этой главы (спойлер) + until_ch: int | None = None # ... по эту главу + allow_short: bool = False # разрешить одиночный ключ (редко; только с guard) + + def keys(self): + return [self.src] + self.aliases + + +MIN_KEY = 2 # запрет одиночного Han/буквы как самостоятельного ключа (A3) + + +@dataclass +class Picked: + entry: Entry + via: str # какой ключ сработал | "sticky" + disp: str # confirmed | ambiguous | reject: + + +def select(glossary, chunk, chapter, sticky_prev=(), budget=6): + """Горячий путь: из глоссария выбрать записи для инъекции в ЭТОТ чанк. + Возвращает (injected, rejected, evicted, active_now).""" + ntext = normalize(chunk) + hits, rejected = {}, [] + for e in glossary: + matched_key = None + # longest-match: длинные ключи раньше, одиночные — под запретом + for k in sorted(e.keys(), key=len, reverse=True): + nk = normalize(k) + if cjk_len(nk) < MIN_KEY and not e.allow_short: + continue + if nk and nk in ntext: + matched_key = k + break + if matched_key is None: + continue + # спойлер-окно — ЖЁСТКИЙ reject с логом (C1: safety-гейт, не UX) + if e.since_ch is not None and chapter < e.since_ch: + rejected.append(Picked(e, matched_key, f"reject:spoiler(since_ch={e.since_ch})")) + continue + if e.until_ch is not None and chapter > e.until_ch: + rejected.append(Picked(e, matched_key, f"reject:spoiler(until_ch={e.until_ch})")) + continue + disp = "confirmed" if e.status == "approved" else "ambiguous" + hits[e.id] = Picked(e, matched_key, disp) + + # sticky scene-inertia: активные в предыдущем чанке — тянем, если не совпали + active_ids = set(hits) + for e in glossary: + if e.id in sticky_prev and e.id not in hits: + if e.since_ch is not None and chapter < e.since_ch: + continue # спойлер бьёт и sticky + hits[e.id] = Picked(e, "sticky", "confirmed" if e.status == "approved" else "ambiguous") + + # токен-бюджет с приоритетом: confirmed>ambiguous, exact>sticky, approved>auto + order = sorted(hits.values(), key=lambda p: (p.disp != "confirmed", p.via == "sticky", p.entry.status != "approved")) + injected, evicted = order[:budget], order[budget:] + return injected, rejected, evicted, active_ids + + +def render_block(injected): + """Сериализация в промпт (SakuraLLM-стиль); ambiguous помечены «проверить».""" + lines = [] + for p in injected: + tag = "" if p.disp == "confirmed" else " ⟨unverified — проверить⟩" + lines.append(f"{p.entry.src} → {p.entry.dst}{tag}") + return "\n".join(lines) + + +def postcheck(injected, output): + """E1: для каждой ИНЪЕКТИРОВАННОЙ approved-записи — есть ли её утв. форма в + выходе? нет → флаг (модель проигнорировала ИЛИ мы инъектили неверное).""" + flags = [] + for p in injected: + if p.disp != "confirmed": + continue + if not re.search(p.entry.accept, output): + flags.append((p.entry.id, p.entry.dst, "MISSING")) + return flags + + +# ============================ SELF-TESTS ============================ + +G = [ + Entry("ah_q", "阿Q", "А-кью", r"А[-\s]?[Кк]ью", sense="name"), + Entry("sishu", "四叔", "Четвёртый дядюшка", r"[Дд]ядюшк", aliases=["鲁四老爷"], sense="char"), + Entry("luzhen", "鲁镇", "Лучжэнь", r"Лучжэн"), + Entry("songzao","送灶", "проводы бога очага", r"бога очага", sense="festival"), # НЕ бывает от 灶 + Entry("xiaoyan","萧炎", "Сяо Янь", r"Сяо\s?Янь", aliases=["炎哥"], sense="name"), # НЕ от 炎 + Entry("xiuwei", "修为", "уровень совершенствования", r"совершенствован", sense="cultivation"), # НЕ от 修 + Entry("shadow", "影卫", "Тёмный страж", r"[Тт]ёмн\w+ страж", since_ch=200, sense="spoiler"), # раскрытие в гл.200 + Entry("newname","小D", "Малыш Дэ", r"Малыш\s?Дэ", status="auto"), # неподтверждённый → ambiguous +] + + +def check(name, cond, detail=""): + print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else "")) + return cond + + +def main(): + print("=== Референс горячего пути банка памяти — self-tests ===\n") + ok = [] + + # T1: имена в чанке → отбор + confirmed; алиас 鲁四老爷 тянет 四叔 + inj, rej, ev, _ = select(G, "阿Q走进鲁镇,遇见鲁四老爷。", chapter=1) + ids = {p.entry.id: p.disp for p in inj} + ok.append(check("T1 exact+alias select", ids.get("ah_q") == "confirmed" and ids.get("luzhen") == "confirmed" + and ids.get("sishu") == "confirmed", f"выбрано: {ids}")) + + # T2: 送...灶 в бытовом смысле — 送灶 НЕ инъектится (биграм 送灶 нет; 灶 одиночный запрещён) + inj, *_ = select(G, "厨房里,老妈子把饭菜送到灶间去热一热。", chapter=1) + ok.append(check("T2 wrong-sense 送灶 rejected", "songzao" not in {p.entry.id for p in inj}, + "омограф не инъектирован (A3)")) + + # T3: 炎=пламя — 萧炎 НЕ инъектится (одиночный 炎 не ключ, 萧炎 отсутствует) + inj, *_ = select(G, "山火蔓延,炎势冲天,村民连夜逃走。", chapter=1) + ok.append(check("T3 wrong-sense 炎 rejected", "xiaoyan" not in {p.entry.id for p in inj})) + + # T4: 修=чинить — 修为 НЕ инъектится + inj, *_ = select(G, "工匠把石桥修好了。", chapter=1) + ok.append(check("T4 wrong-sense 修 rejected", "xiuwei" not in {p.entry.id for p in inj})) + + # T5: нормализация trad→simp — 魯鎮 совпадает с ключом 鲁镇 + inj, *_ = select(G, "魯鎮的冬天很冷。", chapter=1) # traditional + ok.append(check("T5 trad→simp normalization", "luzhen" in {p.entry.id for p in inj}, + "魯鎮 → 鲁镇 сматчилось (A4)")) + + # T6: местоименный чанк (нет имён) — sticky тянет 四叔 из прошлого чанка + inj, *_ = select(G, "他慢慢站起来,叹了口气。", chapter=2, sticky_prev={"sishu"}) + picked = {p.entry.id: p.via for p in inj} + ok.append(check("T6 sticky scene-inertia", picked.get("sishu") == "sticky", + f"местоименный чанк: {picked}")) + + # T7: спойлер — 影卫 (since_ch=200) в главе 5 → hard reject + inj, rej, *_ = select(G, "影卫出现了。", chapter=5) + ok.append(check("T7 spoiler hard-reject", "shadow" not in {p.entry.id for p in inj} + and any(p.entry.id == "shadow" for p in rej), + f"reject: {[p.disp for p in rej]}")) + # ... а в главе 250 та же запись проходит + inj250, *_ = select(G, "影卫出现了。", chapter=250) + ok.append(check("T7b spoiler passes after reveal", "shadow" in {p.entry.id for p in inj250})) + + # T8: disposition — неподтверждённый auto-термин 小D помечен ambiguous + inj, *_ = select(G, "小D也来了。", chapter=1) + d = {p.entry.id: p.disp for p in inj} + ok.append(check("T8 auto-term → ambiguous", d.get("newname") == "ambiguous", + f"disposition: {d}; render:\n " + render_block(inj).replace("\n", "\n "))) + + # T9: post-check ловит утечку — approved 阿Q инъектирован, но в выходе имени НЕТ + inj, *_ = select(G, "阿Q走进鲁镇。", chapter=1) + flags_bad = postcheck(inj, "Некто вошёл в Лучжэнь.") # А-кью пропал + flags_good = postcheck(inj, "А-кью вошёл в Лучжэнь.") # всё на месте + ok.append(check("T9 post-check catches missing dst", + any(f[0] == "ah_q" for f in flags_bad) and not flags_good, + f"плохой выход флаги={flags_bad}; хороший={flags_good}")) + + # T10: post-check ловит ОТРАВЛЕНИЕ — инъектировали неверный dst, выход ему следует + poisoned = Picked(Entry("ah_q", "阿Q", "А-кью", r"А[-\s]?[Кк]ью"), "阿Q", "confirmed") + fl = postcheck([poisoned], "Линь Чун вошёл в деревню.") # выход использует чужое имя + ok.append(check("T10 post-check catches poisoning", any(f[2] == "MISSING" for f in fl), + "утв. форма отсутствует → флаг (страх владельца, эксп.05 C3)")) + + print(f"\n=== {sum(ok)}/{len(ok)} PASS ===") + return 0 if all(ok) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/retrieval_bench.py b/eval/retrieval_bench.py new file mode 100644 index 0000000..e00394e --- /dev/null +++ b/eval/retrieval_bench.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Retrieval benchmark for the TextMachine book-memory-bank +(session «Валидация банка памяти», docs/research/13-memory-bank-validation.md). + +Task: given a source chunk (query) → retrieve the relevant glossary/summary +entries (docs) from the memory bank. Compares three embedding models mandated by +the session (bge-m3 vs Qwen3-Embedding-0.6B vs LaBSE) against two non-neural +baselines that model the real design (deterministic substring/alias key-match = +the "hot path" of Р3; BM25 char-level = the lexical leg of a hybrid). + +Answers, empirically, on real zh/ja PD literary text + constructed wrong-sense traps: + Q1 Can a fixed cosine threshold separate "relevant" from "garbage"? (ROC-AUC, + precision-0.9 threshold, recall there, behaviour on a fully-unrelated query.) + Q3 Dense vs deterministic-substring vs BM25 — who retrieves what, and the + exact-term vs semantic/crosslingual split. + Q4 Wrong-sense / homonym false-fire: 送灶 (festival vs literal), 修 (cultivate + vs repair), 炎 (name vs flame), 气 (qi vs air) — does dense inject the wrong + entry that exact-substring correctly rejects? + Q2 Brute-force KNN latency vs N (model-agnostic, per embedding dim) → the + migration trigger to a real vector layer. + +Runs on CPU by default (GPU on the stand is held by the polygon's resident model). +Usage: eval/.venv/bin/python eval/retrieval_bench.py [--models bge-m3,qwen3-0.6b,labse] [--skip-scale] +Output: stdout tables + eval/data/retrieval_bench/results.json +""" +from __future__ import annotations + +import argparse +import json +import re +import time +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parent +CORPUS = ROOT / "data" / "retrieval_bench" / "corpus.json" +OUT = ROOT / "data" / "retrieval_bench" / "results.json" + +# HF ids + how queries must be prompted. Qwen3-Embedding uses a query instruction +# (asymmetric); bge-m3 and LaBSE are symmetric (plain encode both sides). +MODELS = { + "bge-m3": {"hf": "BAAI/bge-m3", "query_prompt": None}, + "qwen3-0.6b": {"hf": "Qwen/Qwen3-Embedding-0.6B", + "query_prompt": "Instruct: Given a passage of a novel, retrieve the glossary term or summary it refers to\nQuery: "}, + "labse": {"hf": "sentence-transformers/LaBSE", "query_prompt": None}, +} + +GENUINE_KINDS = {"real", "semantic", "crosslingual"} # queries that have gold + + +# ---------- corpus ---------- + +def load_corpus(): + data = json.loads(CORPUS.read_text(encoding="utf-8")) + entries = data["entries"] + for e in entries: + note = e.get("note", "") + e["embed_text"] = f"{e['src']} — {e['dst']}." + (f" {note}" if note else "") + return data["queries"], entries + + +# ---------- metrics ---------- + +def rank_metrics(scores_row, entry_ids, gold, ks=(1, 3, 5)): + order = np.argsort(-scores_row) + ranked = [entry_ids[i] for i in order] + goldset = set(gold) + out = {} + for k in ks: + topk = set(ranked[:k]) + out[f"R@{k}"] = len(topk & goldset) / len(goldset) + # MRR (first relevant) + mrr = 0.0 + for r, eid in enumerate(ranked, 1): + if eid in goldset: + mrr = 1.0 / r + break + out["MRR"] = mrr + # nDCG@5 binary + dcg = 0.0 + for r, eid in enumerate(ranked[:5], 1): + if eid in goldset: + dcg += 1.0 / np.log2(r + 1) + idcg = sum(1.0 / np.log2(r + 1) for r in range(1, min(len(goldset), 5) + 1)) + out["nDCG@5"] = dcg / idcg if idcg else 0.0 + return out, ranked + + +def threshold_analysis(sim, queries, entries): + """Pool (score,label) over genuine queries × all entries; find the cosine + threshold giving precision≥0.9 and the recall there; ROC-AUC.""" + from sklearn.metrics import roc_auc_score + eid = [e["id"] for e in entries] + scores, labels = [], [] + rel_scores, irr_scores = [], [] + for qi, q in enumerate(queries): + if q["kind"] not in GENUINE_KINDS: + continue + gold = set(q["gold"]) + for ei, e in enumerate(entries): + s = float(sim[qi, ei]) + lab = 1 if e["id"] in gold else 0 + scores.append(s); labels.append(lab) + (rel_scores if lab else irr_scores).append(s) + scores = np.array(scores); labels = np.array(labels) + auc = float(roc_auc_score(labels, scores)) if labels.min() != labels.max() else float("nan") + # sweep thresholds for precision≥0.9 with max recall + best = {"t": None, "precision": 0.0, "recall": 0.0} + for t in np.unique(scores): + pred = scores >= t + tp = int((pred & (labels == 1)).sum()) + fp = int((pred & (labels == 0)).sum()) + fn = int((~pred & (labels == 1)).sum()) + prec = tp / (tp + fp) if (tp + fp) else 1.0 + rec = tp / (tp + fn) if (tp + fn) else 0.0 + if prec >= 0.9 and rec > best["recall"]: + best = {"t": float(t), "precision": prec, "recall": rec} + return { + "roc_auc": auc, + "rel_mean": float(np.mean(rel_scores)), "rel_min": float(np.min(rel_scores)), + "irr_mean": float(np.mean(irr_scores)), "irr_p95": float(np.percentile(irr_scores, 95)), + "irr_max": float(np.max(irr_scores)), + "prec90_threshold": best["t"], "recall_at_prec90": best["recall"], + } + + +def trap_analysis(sim, queries, entries, t_star): + """At the precision-0.9 threshold t*, do wrong-sense trap targets get injected? + Also the fully-unrelated negative query's top-1 score (abstention).""" + eidx = {e["id"]: i for i, e in enumerate(entries)} + traps = [] + for qi, q in enumerate(queries): + if q["kind"] != "trap": + continue + for tgt in q["traps"]: + s = float(sim[qi, eidx[tgt]]) + traps.append({"q": q["id"], "target": tgt, "cos": round(s, 3), + "injected_at_t*": (t_star is not None and s >= t_star)}) + neg = None + for qi, q in enumerate(queries): + if q["kind"] == "negative": + top = float(np.max(sim[qi])) + neg = {"q": q["id"], "top1_cos": round(top, 3), + "abstains_at_t*": (t_star is None or top < t_star)} + inj = sum(1 for t in traps if t["injected_at_t*"]) + return {"traps": traps, "n_injected_at_t*": inj, "negative": neg} + + +# ---------- neural ---------- + +def run_embed_model(name, spec, queries, entries): + from sentence_transformers import SentenceTransformer + t0 = time.time() + model = SentenceTransformer(spec["hf"], device="cpu") + load_s = time.time() - t0 + doc_texts = [e["embed_text"] for e in entries] + q_texts = [q["text"] for q in queries] + enc = dict(normalize_embeddings=True, batch_size=16, show_progress_bar=False, convert_to_numpy=True) + doc_emb = model.encode(doc_texts, **enc) + t1 = time.time() + if spec["query_prompt"]: + q_emb = model.encode(q_texts, prompt=spec["query_prompt"], **enc) + else: + q_emb = model.encode(q_texts, **enc) + enc_s = time.time() - t1 + dim = int(doc_emb.shape[1]) + sim = q_emb @ doc_emb.T # cosine (normalized) + + entry_ids = [e["id"] for e in entries] + per_q, agg = [], {} + for qi, q in enumerate(queries): + if q["kind"] not in GENUINE_KINDS: + continue + m, ranked = rank_metrics(sim[qi], entry_ids, q["gold"]) + per_q.append({"q": q["id"], "kind": q["kind"], **{k: round(v, 3) for k, v in m.items()}, + "top5": ranked[:5]}) + for k, v in m.items(): + agg.setdefault(k, []).append(v) + agg = {k: round(float(np.mean(v)), 3) for k, v in agg.items()} + thr = threshold_analysis(sim, queries, entries) + trp = trap_analysis(sim, queries, entries, thr["prec90_threshold"]) + return {"model": name, "hf": spec["hf"], "dim": dim, + "load_s": round(load_s, 1), "encode_s": round(enc_s, 2), + "aggregate": agg, "threshold": thr, "trap": trp, "per_query": per_q} + + +# ---------- baselines ---------- + +def tok(s): + """char-level for CJK, lowercase word-level for latin/cyrillic.""" + out = [] + for m in re.finditer(r"[぀-ヿ㐀-鿿]|[A-Za-zА-Яа-яЁё0-9]+", s): + out.append(m.group(0).lower()) + return out + + +def run_substring(queries, entries): + """Deterministic exact substring/alias key-match = the Р3 hot path.""" + def keys(e): + return [e["src"]] + e.get("aliases", []) if e["type"] != "summary" else [] + tp = fp = fn = 0 + per_q, trap_fires = [], 0 + for q in queries: + hit = [e["id"] for e in entries if any(k and k in q["text"] for k in keys(e))] + if q["kind"] in GENUINE_KINDS: + gold = set(q["gold"]); h = set(hit) + tp += len(h & gold); fp += len(h - gold); fn += len(gold - h) + per_q.append({"q": q["id"], "kind": q["kind"], "hits": hit, + "recall": round(len(h & gold) / len(gold), 3), + "missed": sorted(gold - h)}) + if q["kind"] == "trap": + fired = [e for e in hit if e in q["traps"]] + trap_fires += len(fired) + per_q.append({"q": q["id"], "kind": "trap", "hits": hit, "false_fire": fired}) + prec = tp / (tp + fp) if (tp + fp) else 1.0 + rec = tp / (tp + fn) if (tp + fn) else 0.0 + return {"method": "substring-exact", "precision": round(prec, 3), "recall": round(rec, 3), + "trap_false_fires": trap_fires, "per_query": per_q} + + +def run_bm25(queries, entries): + from rank_bm25 import BM25Okapi + corpus = [tok(e["embed_text"]) for e in entries] + bm = BM25Okapi(corpus) + entry_ids = [e["id"] for e in entries] + agg, per_q, trap_top5 = {}, [], 0 + for q in queries: + sc = np.array(bm.get_scores(tok(q["text"]))) + if q["kind"] in GENUINE_KINDS: + m, ranked = rank_metrics(sc, entry_ids, q["gold"]) + per_q.append({"q": q["id"], "kind": q["kind"], **{k: round(v, 3) for k, v in m.items()}}) + for k, v in m.items(): + agg.setdefault(k, []).append(v) + if q["kind"] == "trap": + order = np.argsort(-sc) + top5 = {entry_ids[i] for i in order[:5]} + if set(q["traps"]) & top5: + trap_top5 += 1 + agg = {k: round(float(np.mean(v)), 3) for k, v in agg.items()} + return {"method": "bm25-char", "aggregate": agg, "trap_in_top5": trap_top5, "per_query": per_q} + + +# ---------- scale ---------- + +def scale_bench(dims=(768, 1024), Ns=(1000, 10000, 50000, 100000, 200000), reps=20): + """Model-agnostic brute-force cosine KNN latency vs N (single query, top-10).""" + rng = np.random.default_rng(42) + res = {} + for d in dims: + row = {} + for N in Ns: + mat = rng.standard_normal((N, d)).astype(np.float32) + mat /= np.linalg.norm(mat, axis=1, keepdims=True) + q = rng.standard_normal(d).astype(np.float32); q /= np.linalg.norm(q) + ts = [] + for _ in range(reps): + t = time.perf_counter() + s = mat @ q + np.argpartition(-s, 10)[:10] + ts.append((time.perf_counter() - t) * 1000) + row[N] = round(float(np.median(ts)), 2) + res[d] = row + return res + + +# ---------- main ---------- + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--models", default="bge-m3,qwen3-0.6b,labse") + ap.add_argument("--skip-scale", action="store_true") + args = ap.parse_args() + + queries, entries = load_corpus() + n_gold = sum(1 for q in queries if q["kind"] in GENUINE_KINDS) + print(f"corpus: {len(entries)} entries, {len(queries)} queries " + f"({n_gold} with gold, {sum(1 for q in queries if q['kind']=='trap')} traps)\n") + + results = {"embedding": [], "baselines": {}, "scale": None} + + print("=== baselines ===") + sub = run_substring(queries, entries) + results["baselines"]["substring"] = sub + print(f"substring-exact: precision={sub['precision']} recall={sub['recall']} " + f"trap_false_fires={sub['trap_false_fires']}") + bm = run_bm25(queries, entries) + results["baselines"]["bm25"] = bm + print(f"bm25-char: {bm['aggregate']} trap_in_top5={bm['trap_in_top5']}\n") + + print("=== embedding models (CPU) ===") + for name in [m.strip() for m in args.models.split(",") if m.strip()]: + spec = MODELS[name] + try: + r = run_embed_model(name, spec, queries, entries) + except Exception as e: + print(f"{name}: FAILED — {type(e).__name__}: {e}") + results["embedding"].append({"model": name, "error": f"{type(e).__name__}: {e}"}) + continue + results["embedding"].append(r) + t = r["threshold"] + print(f"\n[{name}] dim={r['dim']} load={r['load_s']}s encode={r['encode_s']}s") + print(f" retrieval: {r['aggregate']}") + print(f" threshold: ROC-AUC={t['roc_auc']:.3f} rel_mean={t['rel_mean']:.3f} " + f"rel_min={t['rel_min']:.3f} irr_mean={t['irr_mean']:.3f} irr_p95={t['irr_p95']:.3f}") + print(f" prec≥0.9 @ cos≥{t['prec90_threshold']} → recall={t['recall_at_prec90']:.3f}") + tr = r["trap"] + print(f" wrong-sense traps injected at t*: {tr['n_injected_at_t*']}/{len(tr['traps'])} " + f"| {[(x['q'], x['target'], x['cos']) for x in tr['traps']]}") + if tr["negative"]: + print(f" unrelated-query top1 cos: {tr['negative']['top1_cos']} " + f"(abstains at t*: {tr['negative']['abstains_at_t*']})") + + if not args.skip_scale: + print("\n=== brute-force KNN latency (ms, single query, top-10, median) ===") + sc = scale_bench() + results["scale"] = sc + for d, row in sc.items(): + print(f" dim={d}: " + " ".join(f"N={N}:{ms}ms" for N, ms in row.items())) + + OUT.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") + print(f"\nsaved → {OUT}") + + +if __name__ == "__main__": + main()