Track the closed memory-validation and extraction sessions' eval artifacts that committed docs already reference
This commit is contained in:
Claude (backend session)
parent
0b31222b5c
commit
1878c3ea95
3 changed files with 822 additions and 0 deletions
272
eval/extract_bench.py
Normal file
272
eval/extract_bench.py
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Экстракция терминов — БЕНЧМАРК локальных vs облачных моделей (write-path банка памяти, реестр G1).
|
||||
|
||||
Локалка касается банка памяти в 2 точках: эмбеддинги (retrieval_bench.py — bge-m3) и
|
||||
ЭКСТРАКЦИЯ кандидатов в глоссарий из сырого текста (bootstrap write-path). Полигон exp03
|
||||
мерил локалки как ПЕРЕВОДЧИКОВ (не годятся); экстракция — другая, более лёгкая задача.
|
||||
Здесь мерим её нормально: много моделей, много кейсов, разные языки/эдж-кейсы, и КАЖДЫЙ
|
||||
ответ верифицируется на здоровость (пустой/echo/config-error → ретрай и явная пометка
|
||||
FAILED, НЕ засчитывается как recall 0 — урок предыдущего обрывочного прогона).
|
||||
|
||||
Запросы к провайдерам — через `refusal_bench.call_provider` + канонический `providers.json`
|
||||
(квирки уже верны: deepseek thinking-ON+max_tokens 8000, kimi temp=1/24k, gpt-5-mini
|
||||
max_completion_tokens+no-temp, gemini/glm thinking-off). Локаль — ollama native (think:false).
|
||||
Плюс вариант deepseek-nothink (`thinking:disabled`) — дешёвая экстракция без reasoning.
|
||||
|
||||
Метрики (ДЕТЕРМИНИРОВАННЫЕ, без судьи — чтобы «верифицированно и точно»):
|
||||
recall — доля эталонных сущностей, найденных (спот)
|
||||
render_ok — доля найденных gold, где предложенный RU совпал с каноном (Рогов/Поливанов/Палладий/устоявш.)
|
||||
halluc — доля извлечённого, чего НЕТ в исходном тексте (явный FP, детерминированно)
|
||||
health — ok / empty / http / refused / unparseable (ретраи; в агрегат идут только ok)
|
||||
|
||||
Запуск: eval/.venv/bin/python eval/extract_bench.py [--models a,b] [--cases zh,ja,en]
|
||||
Выход: eval/data/extract_bench/results.json + таблица.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import unicodedata
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import dotenv_values
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
OUT = ROOT / "data" / "extract_bench"
|
||||
ENV = dotenv_values(ROOT / ".env")
|
||||
for k, v in ENV.items(): # call_provider читает ключи из os.environ
|
||||
os.environ.setdefault(k, v or "")
|
||||
NOPROXY = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
# reuse verified request machinery
|
||||
_spec = importlib.util.spec_from_file_location("rb", ROOT / "refusal_bench.py")
|
||||
rb = importlib.util.module_from_spec(_spec); _spec.loader.exec_module(rb)
|
||||
PROVIDERS = {p["name"]: p for p in json.loads((ROOT / "providers.json").read_text())["providers"]}
|
||||
|
||||
CLOUD = ["deepseek", "grok", "glm", "kimi", "gemini", "openai"]
|
||||
CLOUD_EXTRA = { # deepseek без thinking — дешёвая экстракция (echo бьёт перевод, не JSON?)
|
||||
"deepseek-nothink": dict(PROVIDERS["deepseek"], name="deepseek-nothink",
|
||||
extra_body={"thinking": {"type": "disabled"}}),
|
||||
}
|
||||
LOCAL = ["huihui_ai/qwen3-abliterated:8b", "huihui_ai/qwen3.5-abliterated:9b",
|
||||
"qwen3:8b", "qwen3-vojo:latest", "ruadapt-qwen3:8b",
|
||||
"huihui_ai/qwen3-abliterated:30b-a3b"]
|
||||
|
||||
SYS = "Ты — терминолог. Извлекаешь из текста имена собственные и культурные реалии для глоссария художественного перевода на русский."
|
||||
USER = ("Извлеки из фрагмента ВСЕ имена собственные (персонажи, места, организации) и "
|
||||
"культурно-специфичные реалии/титулы. Для каждого верни: src (ровно как в тексте), "
|
||||
"type (character|place|org|realia|title), dst (перевод/транскрипция на русский по канону). "
|
||||
'Верни СТРОГО JSON без пояснений: {"terms":[{"src":"...","type":"...","dst":"..."}]}.\n\nФрагмент:\n<<<TEXT>>>')
|
||||
|
||||
# ---- корпус: реальные PD-чанки, разные языки/эдж-кейсы, gold с каноническим RU (accept-регэксп) ----
|
||||
CASES = [
|
||||
# zh — имена + реалии + алиас
|
||||
dict(id="zh-ahq", lang="zh",
|
||||
text="我要给阿Q做正传，已经不止一两年了。那是赵太爷的儿子进了秀才的时候，锣声铛铛的报到村里来，阿Q正喝了两碗黄酒，因为他和赵太爷原来是本家。",
|
||||
gold={"阿Q": r"А[-\s]?[Кк]ью", "赵太爷": r"Чжао", "秀才": r"сюцай"}),
|
||||
dict(id="zh-zhufu", lang="zh",
|
||||
text="接着一声钝响，是送灶的爆竹。我是正在这一夜回到我的故乡鲁镇的，暂寓在鲁四老爷的宅子里，应该称之曰「四叔」。家中都在准备着「祝福」，这是鲁镇年终的大典，迎接福神。",
|
||||
gold={"送灶": r"очаг|Цзао", "鲁镇": r"Лучжэн", "鲁四老爷": r"дядюшк|[Лл]у\b|Лу[ -]", "祝福": r"жертвоприношен|благословени|[Мм]олени|счасть"}),
|
||||
dict(id="zh-xianglin", lang="zh",
|
||||
text="况且，一想到昨天遇见祥林嫂的事，也就使我不能安住。我在鲁镇所见的人们中，改变之大，可以说无过于她的了；她分明已经纯乎是一个乞丐了。",
|
||||
gold={"祥林嫂": r"Сянлин", "鲁镇": r"Лучжэн"}),
|
||||
dict(id="zh-wanghu", lang="zh", # алиас: 王胡 = 王癞胡
|
||||
text="他看见王胡在那里赤着膊捉虱子。这王胡，又癞又胡，别人都叫他王癞胡，阿Q却删去了一个癞字，然而非常渺视他。",
|
||||
gold={"王胡": r"Ван|Бородат", "阿Q": r"А[-\s]?[Кк]ью"}),
|
||||
# ja — имена + реалии
|
||||
dict(id="ja-rashomon", lang="ja",
|
||||
text="一人の下人が、羅生門の下で雨やみを待っていた。羅生門が、朱雀大路にある以上は、この男のほかにも、雨やみをする者がありそうなものである。",
|
||||
gold={"下人": r"слуг|гэнин", "羅生門": r"Рас[её]мон", "朱雀大路": r"Судзаку"}),
|
||||
dict(id="ja-kyoto", lang="ja",
|
||||
text="この二三年、京都には、地震とか辻風とか火事とか饑饉とか云う災がつづいて起った。そこで洛中のさびれ方は一通りではない。",
|
||||
gold={"京都": r"Киото", "洛中": r"Ракутю|Лочжун|столиц|город"}),
|
||||
# ja — западные имена через катакану (translit_policy эдж-кейс)
|
||||
dict(id="ja-melos", lang="ja",
|
||||
text="メロスは激怒した。メロスは、村の牧人である。きょう未明メロスは村を出発し、十里はなれた此のシラクスの市にやって来た。",
|
||||
gold={"メロス": r"Мелос", "シラクス": r"Сиракуз"}),
|
||||
dict(id="ja-seri", lang="ja", # длинное западное имя через катакану
|
||||
text="メロスには竹馬の友があった。セリヌンティウスである。今は此のシラクスの市で、石工をしている。",
|
||||
gold={"セリヌンティウス": r"Селинунти", "メロス": r"Мелос", "シラクス": r"Сиракуз"}),
|
||||
# ja — буддийское имя + топоним (эталон Стругацкого)
|
||||
dict(id="ja-hana", lang="ja",
|
||||
text="禅智内供の鼻と云えば、池尾で知らない者はない。長さは五六寸あって上唇の上から顋の下まで下っている。",
|
||||
gold={"禅智内供": r"Дзэн[тч]и|Найгу", "池尾": r"Икэ"}),
|
||||
# en — западные имена → русская транскрипция (render-эдж)
|
||||
dict(id="en-carter", lang="en",
|
||||
text="My name is John Carter; I am better known as Captain Jack Carter of Virginia. I spent nearly a year prospecting in company with another Confederate officer, Captain James K. Powell of Richmond, in an Arizona cave.",
|
||||
gold={"John Carter": r"Джон\s?Картер", "Virginia": r"Виргини|Вирджини", "Powell": r"Пауэл", "Richmond": r"Ричмонд", "Arizona": r"Аризон"}),
|
||||
dict(id="en-conan", lang="en",
|
||||
text="'I am Conan, a Cimmerian,' he answered. 'I came into Argos seeking employment, but with no wars forward, there was nothing to which I might turn my hand.'",
|
||||
gold={"Conan": r"Конан", "Cimmerian": r"[Кк]иммери", "Argos": r"Аргос"}),
|
||||
dict(id="en-wells", lang="en",
|
||||
text="The Time Traveller (for so it will be convenient to speak of him) was expounding a recondite matter to us. 'You must follow me carefully,' said Filby.",
|
||||
gold={"Time Traveller": r"[Пп]утешественник", "Filby": r"Филби"}),
|
||||
]
|
||||
|
||||
|
||||
def norm(s):
|
||||
return unicodedata.normalize("NFKC", str(s)).lower()
|
||||
|
||||
|
||||
def parse_terms(txt):
|
||||
txt = re.sub(r"<think>.*?</think>\s*", "", txt or "", flags=re.S)
|
||||
m = re.search(r"\{.*\}|\[.*\]", txt, re.S)
|
||||
if not m:
|
||||
return None # unparseable
|
||||
try:
|
||||
d = json.loads(m.group(0))
|
||||
except Exception:
|
||||
return None
|
||||
terms = d.get("terms", d) if isinstance(d, dict) else d
|
||||
if not isinstance(terms, list):
|
||||
return []
|
||||
return [{"src": str(t["src"]), "type": str(t.get("type", "")), "dst": str(t.get("dst", ""))}
|
||||
for t in terms if isinstance(t, dict) and t.get("src")]
|
||||
|
||||
|
||||
def call_local(model, text, max_pred=4000):
|
||||
payload = dict(model=model, stream=False, think=False, format="json",
|
||||
options=dict(num_ctx=8192, temperature=0.2, num_predict=max_pred),
|
||||
messages=[{"role": "system", "content": SYS},
|
||||
{"role": "user", "content": USER.replace("<<<TEXT>>>", text)}])
|
||||
req = urllib.request.Request("http://localhost:11434/api/chat",
|
||||
data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"})
|
||||
with NOPROXY.open(req, timeout=600) as r:
|
||||
d = json.load(r)
|
||||
return (d.get("message", {}) or {}).get("content", "") or "", d.get("done_reason", "")
|
||||
|
||||
|
||||
def healthy_extract(kind, model, case):
|
||||
"""Возвращает (terms|None, health, dt). Ретраит пустой ответ с бо́льшим бюджетом."""
|
||||
text = case["text"]
|
||||
for attempt in range(3):
|
||||
t0 = time.time()
|
||||
try:
|
||||
if kind == "local":
|
||||
raw, fin = call_local(model, text, max_pred=4000 * (attempt + 1))
|
||||
err = None if raw.strip() else f"empty|finish={fin}"
|
||||
else:
|
||||
pmap = CLOUD_EXTRA.get(model) or PROVIDERS[model]
|
||||
if attempt: # ретрай пустого — поднять бюджет
|
||||
pmap = dict(pmap, max_tokens=int(pmap.get("max_tokens", 8000) * 1.7))
|
||||
raw, err, _ = rb.call_provider(pmap, SYS, USER.replace("<<<TEXT>>>", text))
|
||||
raw = raw or ""
|
||||
except Exception as e:
|
||||
return None, f"transport:{type(e).__name__}", round(time.time() - t0, 1)
|
||||
dt = round(time.time() - t0, 1)
|
||||
if err:
|
||||
kindn = err.split("|", 1)[0]
|
||||
if kindn == "content_filter":
|
||||
return None, "refused", dt
|
||||
if kindn in ("empty",):
|
||||
continue # ретрай
|
||||
return None, f"http:{err[:40]}", dt
|
||||
terms = parse_terms(raw)
|
||||
if terms is None:
|
||||
if attempt < 2:
|
||||
continue # unparseable → ретрай
|
||||
return None, "unparseable", dt
|
||||
return terms, "ok", dt
|
||||
return None, "empty", dt
|
||||
|
||||
|
||||
def score(case, terms):
|
||||
ntext = norm(case["text"])
|
||||
found, render = {}, {}
|
||||
for g, accept in case["gold"].items():
|
||||
ng = norm(g)
|
||||
hit = next((t for t in terms if ng in norm(t["src"]) or (norm(t["src"]) in ng and len(norm(t["src"])) >= 2)), None)
|
||||
found[g] = hit is not None
|
||||
render[g] = bool(hit and re.search(accept, hit["dst"]))
|
||||
ex = [t["src"] for t in terms]
|
||||
halluc = [s for s in ex if norm(s) and norm(s) not in ntext]
|
||||
return dict(recall=sum(found.values()) / len(found),
|
||||
render_ok=(sum(render.values()) / max(1, sum(found.values())) if any(found.values()) else 0.0),
|
||||
halluc_rate=round(len(halluc) / len(ex), 2) if ex else 0.0,
|
||||
n=len(terms), miss=[g for g, ok in found.items() if not ok],
|
||||
bad_render=[g for g in found if found[g] and not render[g]],
|
||||
halluc=halluc[:5], raw=[f"{t['src']}→{t['dst']}" for t in terms][:14])
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--models", default="")
|
||||
ap.add_argument("--cases", default="")
|
||||
args = ap.parse_args()
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
cases = [c for c in CASES if not args.cases or c["lang"] in args.cases.split(",")]
|
||||
models = ([("local", m) for m in LOCAL] + [("cloud", m) for m in CLOUD]
|
||||
+ [("cloud", "deepseek-nothink")])
|
||||
if args.models:
|
||||
want = set(args.models.split(","))
|
||||
models = [(k, m) for k, m in models if m in want]
|
||||
|
||||
jsonl = OUT / "records.jsonl"
|
||||
done = set()
|
||||
if jsonl.exists():
|
||||
for ln in jsonl.read_text().splitlines():
|
||||
if ln.strip():
|
||||
r = json.loads(ln); done.add((r["model"], r["case"]))
|
||||
fh = jsonl.open("a", encoding="utf-8")
|
||||
records = [json.loads(ln) for ln in (jsonl.read_text().splitlines() if jsonl.exists() else []) if ln.strip()]
|
||||
|
||||
for kind, model in models:
|
||||
print(f"\n### {model} ({kind})")
|
||||
for case in cases:
|
||||
if (model, case["id"]) in done:
|
||||
continue
|
||||
terms, health, dt = healthy_extract(kind, model, case)
|
||||
rec = dict(model=model, kind=kind, case=case["id"], lang=case["lang"], health=health, dt=dt)
|
||||
if health == "ok":
|
||||
rec.update(score(case, terms))
|
||||
records.append(rec)
|
||||
fh.write(json.dumps(rec, ensure_ascii=False) + "\n"); fh.flush()
|
||||
if health == "ok":
|
||||
print(f" {case['id']:<12} recall={rec['recall']:.2f} render={rec['render_ok']:.2f} "
|
||||
f"halluc={rec['halluc_rate']} n={rec['n']} {dt}s"
|
||||
+ (f" miss={rec['miss']}" if rec['miss'] else "")
|
||||
+ (f" bad_render={rec['bad_render']}" if rec['bad_render'] else ""))
|
||||
else:
|
||||
print(f" {case['id']:<12} HEALTH={health} {dt}s")
|
||||
fh.close()
|
||||
|
||||
# ---- агрегаты: только health==ok идут в recall/render/halluc; health-счётчики отдельно ----
|
||||
summary = {}
|
||||
for kind, model in models:
|
||||
recs = [r for r in records if r["model"] == model]
|
||||
ok = [r for r in recs if r["health"] == "ok"]
|
||||
health_ct = {}
|
||||
for r in recs:
|
||||
health_ct[r["health"]] = health_ct.get(r["health"], 0) + 1
|
||||
if ok:
|
||||
summary[model] = dict(
|
||||
kind=kind, n_ok=len(ok), n_total=len(recs), health=health_ct,
|
||||
recall=round(sum(r["recall"] for r in ok) / len(ok), 3),
|
||||
render_ok=round(sum(r["render_ok"] for r in ok) / len(ok), 3),
|
||||
halluc=round(sum(r["halluc_rate"] for r in ok) / len(ok), 3),
|
||||
avg_n=round(sum(r["n"] for r in ok) / len(ok), 1),
|
||||
avg_s=round(sum(r["dt"] for r in ok) / len(ok), 1))
|
||||
else:
|
||||
summary[model] = dict(kind=kind, n_ok=0, n_total=len(recs), health=health_ct)
|
||||
|
||||
print("\n=== СВОДКА (экстракция; агрегат по health==ok) ===")
|
||||
print(f" {'model':<34}{'kind':<7}{'recall':>7}{'render':>7}{'halluc':>7}{'n_ex':>5}{'sec':>6}{'ok/all':>8}")
|
||||
for m, s in summary.items():
|
||||
ratio = f"{s['n_ok']}/{s['n_total']}"
|
||||
if s["n_ok"]:
|
||||
print(f" {m:<34}{s['kind']:<7}{s['recall']:>7}{s['render_ok']:>7}{s['halluc']:>7}"
|
||||
f"{s['avg_n']:>5}{s['avg_s']:>6}{ratio:>8}")
|
||||
else:
|
||||
print(f" {m:<34}{s['kind']:<7}{'—':>7}{'—':>7}{'—':>7}{'—':>5}{'—':>6}{ratio:>8} health={s['health']}")
|
||||
print(" (health!=ok — пустой/echo/http/unparseable — НЕ засчитан как recall 0)")
|
||||
(OUT / "results.json").write_text(json.dumps(dict(summary=summary, records=records), ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"\nsaved → {OUT/'results.json'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
224
eval/memory_hotpath.py
Normal file
224
eval/memory_hotpath.py
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Референс-реализация ГОРЯЧЕГО ПУТИ банка памяти + self-tests.
|
||||
|
||||
Не продукт — исполняемая СПЕКА механизма (по architecture/06 §Контракт горячего
|
||||
пути и research/13 Q1/Q3/Q4), по которой бэкенд пишет Go. Проверяет, что сам
|
||||
МЕХАНИЗМ работает как задумано, без бэкенда и без LLM-вызовов:
|
||||
|
||||
normalize (NFKC + trad→simp + kana-fold) A4 «тихо-пусто» из-за орфографии
|
||||
→ multi-pattern точный матч ключей/алиасов Q3 detерм. > dense на именах
|
||||
с запретом одиночных ключей (longest-match) A3 омографы 灶/炎/气/修
|
||||
→ спойлер-фильтр since_ch/until_ch (hard) C1 утечка будущего факта
|
||||
→ sticky scene-inertia (местоименные чанки) A5
|
||||
→ токен-бюджет с приоритетом + лог вытеснения F2
|
||||
→ disposition: confirmed / ambiguous / reject A2 трёхстороннее решение
|
||||
→ post-check: утв. dst (по decl) в выходе? E1 ловит и утечку, и отравление
|
||||
|
||||
Запуск: eval/.venv/bin/python eval/memory_hotpath.py (печатает PASS/FAIL по кейсам)
|
||||
Прод-замены помечены [PROD]: OpenCC вместо мини-карты, Aho-Corasick вместо substring,
|
||||
pymorphy/decl вместо accept-регэкспов.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# --- [PROD] в бою — OpenCC (opencc: t2s); здесь мини-карта под демо-термины ---
|
||||
TRAD2SIMP = {"魯": "鲁", "鎮": "镇", "趙": "赵", "蕭": "萧", "煉": "炼", "門": "门", "闆": "板"}
|
||||
|
||||
|
||||
def normalize(s: str) -> str:
|
||||
"""NFKC (полу/полноширина) + trad→simp + катакана→хирагана. Симметрично к
|
||||
ключам и тексту — иначе 魯鎮 ≠ 鲁镇 и запись «тихо не совпадёт» (A4)."""
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
s = "".join(TRAD2SIMP.get(c, c) for c in s)
|
||||
out = []
|
||||
for c in s:
|
||||
o = ord(c)
|
||||
out.append(chr(o - 0x60) if 0x30A1 <= o <= 0x30F6 else c) # katakana→hiragana
|
||||
return "".join(out).lower()
|
||||
|
||||
|
||||
def cjk_len(s: str) -> int:
|
||||
"""Длина ключа в «значимых» знаках (CJK + буквы), для запрета одиночных ключей."""
|
||||
return len(re.findall(r"[-ヿ㐀-鿿豈-A-Za-zЀ-ӿ]", s))
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entry:
|
||||
id: str
|
||||
src: str
|
||||
dst: str
|
||||
accept: str # регэксп утв. формы в РУ-выходе ([PROD] через decl/лемматизатор)
|
||||
aliases: list = field(default_factory=list)
|
||||
status: str = "approved" # approved | auto | draft
|
||||
sense: str = ""
|
||||
since_ch: int | None = None # запись валидна с этой главы (спойлер)
|
||||
until_ch: int | None = None # ... по эту главу
|
||||
allow_short: bool = False # разрешить одиночный ключ (редко; только с guard)
|
||||
|
||||
def keys(self):
|
||||
return [self.src] + self.aliases
|
||||
|
||||
|
||||
MIN_KEY = 2 # запрет одиночного Han/буквы как самостоятельного ключа (A3)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Picked:
|
||||
entry: Entry
|
||||
via: str # какой ключ сработал | "sticky"
|
||||
disp: str # confirmed | ambiguous | reject:<reason>
|
||||
|
||||
|
||||
def select(glossary, chunk, chapter, sticky_prev=(), budget=6):
|
||||
"""Горячий путь: из глоссария выбрать записи для инъекции в ЭТОТ чанк.
|
||||
Возвращает (injected, rejected, evicted, active_now)."""
|
||||
ntext = normalize(chunk)
|
||||
hits, rejected = {}, []
|
||||
for e in glossary:
|
||||
matched_key = None
|
||||
# longest-match: длинные ключи раньше, одиночные — под запретом
|
||||
for k in sorted(e.keys(), key=len, reverse=True):
|
||||
nk = normalize(k)
|
||||
if cjk_len(nk) < MIN_KEY and not e.allow_short:
|
||||
continue
|
||||
if nk and nk in ntext:
|
||||
matched_key = k
|
||||
break
|
||||
if matched_key is None:
|
||||
continue
|
||||
# спойлер-окно — ЖЁСТКИЙ reject с логом (C1: safety-гейт, не UX)
|
||||
if e.since_ch is not None and chapter < e.since_ch:
|
||||
rejected.append(Picked(e, matched_key, f"reject:spoiler(since_ch={e.since_ch})"))
|
||||
continue
|
||||
if e.until_ch is not None and chapter > e.until_ch:
|
||||
rejected.append(Picked(e, matched_key, f"reject:spoiler(until_ch={e.until_ch})"))
|
||||
continue
|
||||
disp = "confirmed" if e.status == "approved" else "ambiguous"
|
||||
hits[e.id] = Picked(e, matched_key, disp)
|
||||
|
||||
# sticky scene-inertia: активные в предыдущем чанке — тянем, если не совпали
|
||||
active_ids = set(hits)
|
||||
for e in glossary:
|
||||
if e.id in sticky_prev and e.id not in hits:
|
||||
if e.since_ch is not None and chapter < e.since_ch:
|
||||
continue # спойлер бьёт и sticky
|
||||
hits[e.id] = Picked(e, "sticky", "confirmed" if e.status == "approved" else "ambiguous")
|
||||
|
||||
# токен-бюджет с приоритетом: confirmed>ambiguous, exact>sticky, approved>auto
|
||||
order = sorted(hits.values(), key=lambda p: (p.disp != "confirmed", p.via == "sticky", p.entry.status != "approved"))
|
||||
injected, evicted = order[:budget], order[budget:]
|
||||
return injected, rejected, evicted, active_ids
|
||||
|
||||
|
||||
def render_block(injected):
|
||||
"""Сериализация в промпт (SakuraLLM-стиль); ambiguous помечены «проверить»."""
|
||||
lines = []
|
||||
for p in injected:
|
||||
tag = "" if p.disp == "confirmed" else " ⟨unverified — проверить⟩"
|
||||
lines.append(f"{p.entry.src} → {p.entry.dst}{tag}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def postcheck(injected, output):
|
||||
"""E1: для каждой ИНЪЕКТИРОВАННОЙ approved-записи — есть ли её утв. форма в
|
||||
выходе? нет → флаг (модель проигнорировала ИЛИ мы инъектили неверное)."""
|
||||
flags = []
|
||||
for p in injected:
|
||||
if p.disp != "confirmed":
|
||||
continue
|
||||
if not re.search(p.entry.accept, output):
|
||||
flags.append((p.entry.id, p.entry.dst, "MISSING"))
|
||||
return flags
|
||||
|
||||
|
||||
# ============================ SELF-TESTS ============================
|
||||
|
||||
G = [
|
||||
Entry("ah_q", "阿Q", "А-кью", r"А[-\s]?[Кк]ью", sense="name"),
|
||||
Entry("sishu", "四叔", "Четвёртый дядюшка", r"[Дд]ядюшк", aliases=["鲁四老爷"], sense="char"),
|
||||
Entry("luzhen", "鲁镇", "Лучжэнь", r"Лучжэн"),
|
||||
Entry("songzao","送灶", "проводы бога очага", r"бога очага", sense="festival"), # НЕ бывает от 灶
|
||||
Entry("xiaoyan","萧炎", "Сяо Янь", r"Сяо\s?Янь", aliases=["炎哥"], sense="name"), # НЕ от 炎
|
||||
Entry("xiuwei", "修为", "уровень совершенствования", r"совершенствован", sense="cultivation"), # НЕ от 修
|
||||
Entry("shadow", "影卫", "Тёмный страж", r"[Тт]ёмн\w+ страж", since_ch=200, sense="spoiler"), # раскрытие в гл.200
|
||||
Entry("newname","小D", "Малыш Дэ", r"Малыш\s?Дэ", status="auto"), # неподтверждённый → ambiguous
|
||||
]
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else ""))
|
||||
return cond
|
||||
|
||||
|
||||
def main():
|
||||
print("=== Референс горячего пути банка памяти — self-tests ===\n")
|
||||
ok = []
|
||||
|
||||
# T1: имена в чанке → отбор + confirmed; алиас 鲁四老爷 тянет 四叔
|
||||
inj, rej, ev, _ = select(G, "阿Q走进鲁镇，遇见鲁四老爷。", chapter=1)
|
||||
ids = {p.entry.id: p.disp for p in inj}
|
||||
ok.append(check("T1 exact+alias select", ids.get("ah_q") == "confirmed" and ids.get("luzhen") == "confirmed"
|
||||
and ids.get("sishu") == "confirmed", f"выбрано: {ids}"))
|
||||
|
||||
# T2: 送...灶 в бытовом смысле — 送灶 НЕ инъектится (биграм 送灶 нет; 灶 одиночный запрещён)
|
||||
inj, *_ = select(G, "厨房里，老妈子把饭菜送到灶间去热一热。", chapter=1)
|
||||
ok.append(check("T2 wrong-sense 送灶 rejected", "songzao" not in {p.entry.id for p in inj},
|
||||
"омограф не инъектирован (A3)"))
|
||||
|
||||
# T3: 炎=пламя — 萧炎 НЕ инъектится (одиночный 炎 не ключ, 萧炎 отсутствует)
|
||||
inj, *_ = select(G, "山火蔓延，炎势冲天，村民连夜逃走。", chapter=1)
|
||||
ok.append(check("T3 wrong-sense 炎 rejected", "xiaoyan" not in {p.entry.id for p in inj}))
|
||||
|
||||
# T4: 修=чинить — 修为 НЕ инъектится
|
||||
inj, *_ = select(G, "工匠把石桥修好了。", chapter=1)
|
||||
ok.append(check("T4 wrong-sense 修 rejected", "xiuwei" not in {p.entry.id for p in inj}))
|
||||
|
||||
# T5: нормализация trad→simp — 魯鎮 совпадает с ключом 鲁镇
|
||||
inj, *_ = select(G, "魯鎮的冬天很冷。", chapter=1) # traditional
|
||||
ok.append(check("T5 trad→simp normalization", "luzhen" in {p.entry.id for p in inj},
|
||||
"魯鎮 → 鲁镇 сматчилось (A4)"))
|
||||
|
||||
# T6: местоименный чанк (нет имён) — sticky тянет 四叔 из прошлого чанка
|
||||
inj, *_ = select(G, "他慢慢站起来，叹了口气。", chapter=2, sticky_prev={"sishu"})
|
||||
picked = {p.entry.id: p.via for p in inj}
|
||||
ok.append(check("T6 sticky scene-inertia", picked.get("sishu") == "sticky",
|
||||
f"местоименный чанк: {picked}"))
|
||||
|
||||
# T7: спойлер — 影卫 (since_ch=200) в главе 5 → hard reject
|
||||
inj, rej, *_ = select(G, "影卫出现了。", chapter=5)
|
||||
ok.append(check("T7 spoiler hard-reject", "shadow" not in {p.entry.id for p in inj}
|
||||
and any(p.entry.id == "shadow" for p in rej),
|
||||
f"reject: {[p.disp for p in rej]}"))
|
||||
# ... а в главе 250 та же запись проходит
|
||||
inj250, *_ = select(G, "影卫出现了。", chapter=250)
|
||||
ok.append(check("T7b spoiler passes after reveal", "shadow" in {p.entry.id for p in inj250}))
|
||||
|
||||
# T8: disposition — неподтверждённый auto-термин 小D помечен ambiguous
|
||||
inj, *_ = select(G, "小D也来了。", chapter=1)
|
||||
d = {p.entry.id: p.disp for p in inj}
|
||||
ok.append(check("T8 auto-term → ambiguous", d.get("newname") == "ambiguous",
|
||||
f"disposition: {d}; render:\n " + render_block(inj).replace("\n", "\n ")))
|
||||
|
||||
# T9: post-check ловит утечку — approved 阿Q инъектирован, но в выходе имени НЕТ
|
||||
inj, *_ = select(G, "阿Q走进鲁镇。", chapter=1)
|
||||
flags_bad = postcheck(inj, "Некто вошёл в Лучжэнь.") # А-кью пропал
|
||||
flags_good = postcheck(inj, "А-кью вошёл в Лучжэнь.") # всё на месте
|
||||
ok.append(check("T9 post-check catches missing dst",
|
||||
any(f[0] == "ah_q" for f in flags_bad) and not flags_good,
|
||||
f"плохой выход флаги={flags_bad}; хороший={flags_good}"))
|
||||
|
||||
# T10: post-check ловит ОТРАВЛЕНИЕ — инъектировали неверный dst, выход ему следует
|
||||
poisoned = Picked(Entry("ah_q", "阿Q", "А-кью", r"А[-\s]?[Кк]ью"), "阿Q", "confirmed")
|
||||
fl = postcheck([poisoned], "Линь Чун вошёл в деревню.") # выход использует чужое имя
|
||||
ok.append(check("T10 post-check catches poisoning", any(f[2] == "MISSING" for f in fl),
|
||||
"утв. форма отсутствует → флаг (страх владельца, эксп.05 C3)"))
|
||||
|
||||
print(f"\n=== {sum(ok)}/{len(ok)} PASS ===")
|
||||
return 0 if all(ok) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
326
eval/retrieval_bench.py
Normal file
326
eval/retrieval_bench.py
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Retrieval benchmark for the TextMachine book-memory-bank
|
||||
(session «Валидация банка памяти», docs/research/13-memory-bank-validation.md).
|
||||
|
||||
Task: given a source chunk (query) → retrieve the relevant glossary/summary
|
||||
entries (docs) from the memory bank. Compares three embedding models mandated by
|
||||
the session (bge-m3 vs Qwen3-Embedding-0.6B vs LaBSE) against two non-neural
|
||||
baselines that model the real design (deterministic substring/alias key-match =
|
||||
the "hot path" of Р3; BM25 char-level = the lexical leg of a hybrid).
|
||||
|
||||
Answers, empirically, on real zh/ja PD literary text + constructed wrong-sense traps:
|
||||
Q1 Can a fixed cosine threshold separate "relevant" from "garbage"? (ROC-AUC,
|
||||
precision-0.9 threshold, recall there, behaviour on a fully-unrelated query.)
|
||||
Q3 Dense vs deterministic-substring vs BM25 — who retrieves what, and the
|
||||
exact-term vs semantic/crosslingual split.
|
||||
Q4 Wrong-sense / homonym false-fire: 送灶 (festival vs literal), 修 (cultivate
|
||||
vs repair), 炎 (name vs flame), 气 (qi vs air) — does dense inject the wrong
|
||||
entry that exact-substring correctly rejects?
|
||||
Q2 Brute-force KNN latency vs N (model-agnostic, per embedding dim) → the
|
||||
migration trigger to a real vector layer.
|
||||
|
||||
Runs on CPU by default (GPU on the stand is held by the polygon's resident model).
|
||||
Usage: eval/.venv/bin/python eval/retrieval_bench.py [--models bge-m3,qwen3-0.6b,labse] [--skip-scale]
|
||||
Output: stdout tables + eval/data/retrieval_bench/results.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
CORPUS = ROOT / "data" / "retrieval_bench" / "corpus.json"
|
||||
OUT = ROOT / "data" / "retrieval_bench" / "results.json"
|
||||
|
||||
# HF ids + how queries must be prompted. Qwen3-Embedding uses a query instruction
|
||||
# (asymmetric); bge-m3 and LaBSE are symmetric (plain encode both sides).
|
||||
MODELS = {
|
||||
"bge-m3": {"hf": "BAAI/bge-m3", "query_prompt": None},
|
||||
"qwen3-0.6b": {"hf": "Qwen/Qwen3-Embedding-0.6B",
|
||||
"query_prompt": "Instruct: Given a passage of a novel, retrieve the glossary term or summary it refers to\nQuery: "},
|
||||
"labse": {"hf": "sentence-transformers/LaBSE", "query_prompt": None},
|
||||
}
|
||||
|
||||
GENUINE_KINDS = {"real", "semantic", "crosslingual"} # queries that have gold
|
||||
|
||||
|
||||
# ---------- corpus ----------
|
||||
|
||||
def load_corpus():
|
||||
data = json.loads(CORPUS.read_text(encoding="utf-8"))
|
||||
entries = data["entries"]
|
||||
for e in entries:
|
||||
note = e.get("note", "")
|
||||
e["embed_text"] = f"{e['src']} — {e['dst']}." + (f" {note}" if note else "")
|
||||
return data["queries"], entries
|
||||
|
||||
|
||||
# ---------- metrics ----------
|
||||
|
||||
def rank_metrics(scores_row, entry_ids, gold, ks=(1, 3, 5)):
|
||||
order = np.argsort(-scores_row)
|
||||
ranked = [entry_ids[i] for i in order]
|
||||
goldset = set(gold)
|
||||
out = {}
|
||||
for k in ks:
|
||||
topk = set(ranked[:k])
|
||||
out[f"R@{k}"] = len(topk & goldset) / len(goldset)
|
||||
# MRR (first relevant)
|
||||
mrr = 0.0
|
||||
for r, eid in enumerate(ranked, 1):
|
||||
if eid in goldset:
|
||||
mrr = 1.0 / r
|
||||
break
|
||||
out["MRR"] = mrr
|
||||
# nDCG@5 binary
|
||||
dcg = 0.0
|
||||
for r, eid in enumerate(ranked[:5], 1):
|
||||
if eid in goldset:
|
||||
dcg += 1.0 / np.log2(r + 1)
|
||||
idcg = sum(1.0 / np.log2(r + 1) for r in range(1, min(len(goldset), 5) + 1))
|
||||
out["nDCG@5"] = dcg / idcg if idcg else 0.0
|
||||
return out, ranked
|
||||
|
||||
|
||||
def threshold_analysis(sim, queries, entries):
|
||||
"""Pool (score,label) over genuine queries × all entries; find the cosine
|
||||
threshold giving precision≥0.9 and the recall there; ROC-AUC."""
|
||||
from sklearn.metrics import roc_auc_score
|
||||
eid = [e["id"] for e in entries]
|
||||
scores, labels = [], []
|
||||
rel_scores, irr_scores = [], []
|
||||
for qi, q in enumerate(queries):
|
||||
if q["kind"] not in GENUINE_KINDS:
|
||||
continue
|
||||
gold = set(q["gold"])
|
||||
for ei, e in enumerate(entries):
|
||||
s = float(sim[qi, ei])
|
||||
lab = 1 if e["id"] in gold else 0
|
||||
scores.append(s); labels.append(lab)
|
||||
(rel_scores if lab else irr_scores).append(s)
|
||||
scores = np.array(scores); labels = np.array(labels)
|
||||
auc = float(roc_auc_score(labels, scores)) if labels.min() != labels.max() else float("nan")
|
||||
# sweep thresholds for precision≥0.9 with max recall
|
||||
best = {"t": None, "precision": 0.0, "recall": 0.0}
|
||||
for t in np.unique(scores):
|
||||
pred = scores >= t
|
||||
tp = int((pred & (labels == 1)).sum())
|
||||
fp = int((pred & (labels == 0)).sum())
|
||||
fn = int((~pred & (labels == 1)).sum())
|
||||
prec = tp / (tp + fp) if (tp + fp) else 1.0
|
||||
rec = tp / (tp + fn) if (tp + fn) else 0.0
|
||||
if prec >= 0.9 and rec > best["recall"]:
|
||||
best = {"t": float(t), "precision": prec, "recall": rec}
|
||||
return {
|
||||
"roc_auc": auc,
|
||||
"rel_mean": float(np.mean(rel_scores)), "rel_min": float(np.min(rel_scores)),
|
||||
"irr_mean": float(np.mean(irr_scores)), "irr_p95": float(np.percentile(irr_scores, 95)),
|
||||
"irr_max": float(np.max(irr_scores)),
|
||||
"prec90_threshold": best["t"], "recall_at_prec90": best["recall"],
|
||||
}
|
||||
|
||||
|
||||
def trap_analysis(sim, queries, entries, t_star):
|
||||
"""At the precision-0.9 threshold t*, do wrong-sense trap targets get injected?
|
||||
Also the fully-unrelated negative query's top-1 score (abstention)."""
|
||||
eidx = {e["id"]: i for i, e in enumerate(entries)}
|
||||
traps = []
|
||||
for qi, q in enumerate(queries):
|
||||
if q["kind"] != "trap":
|
||||
continue
|
||||
for tgt in q["traps"]:
|
||||
s = float(sim[qi, eidx[tgt]])
|
||||
traps.append({"q": q["id"], "target": tgt, "cos": round(s, 3),
|
||||
"injected_at_t*": (t_star is not None and s >= t_star)})
|
||||
neg = None
|
||||
for qi, q in enumerate(queries):
|
||||
if q["kind"] == "negative":
|
||||
top = float(np.max(sim[qi]))
|
||||
neg = {"q": q["id"], "top1_cos": round(top, 3),
|
||||
"abstains_at_t*": (t_star is None or top < t_star)}
|
||||
inj = sum(1 for t in traps if t["injected_at_t*"])
|
||||
return {"traps": traps, "n_injected_at_t*": inj, "negative": neg}
|
||||
|
||||
|
||||
# ---------- neural ----------
|
||||
|
||||
def run_embed_model(name, spec, queries, entries):
|
||||
from sentence_transformers import SentenceTransformer
|
||||
t0 = time.time()
|
||||
model = SentenceTransformer(spec["hf"], device="cpu")
|
||||
load_s = time.time() - t0
|
||||
doc_texts = [e["embed_text"] for e in entries]
|
||||
q_texts = [q["text"] for q in queries]
|
||||
enc = dict(normalize_embeddings=True, batch_size=16, show_progress_bar=False, convert_to_numpy=True)
|
||||
doc_emb = model.encode(doc_texts, **enc)
|
||||
t1 = time.time()
|
||||
if spec["query_prompt"]:
|
||||
q_emb = model.encode(q_texts, prompt=spec["query_prompt"], **enc)
|
||||
else:
|
||||
q_emb = model.encode(q_texts, **enc)
|
||||
enc_s = time.time() - t1
|
||||
dim = int(doc_emb.shape[1])
|
||||
sim = q_emb @ doc_emb.T # cosine (normalized)
|
||||
|
||||
entry_ids = [e["id"] for e in entries]
|
||||
per_q, agg = [], {}
|
||||
for qi, q in enumerate(queries):
|
||||
if q["kind"] not in GENUINE_KINDS:
|
||||
continue
|
||||
m, ranked = rank_metrics(sim[qi], entry_ids, q["gold"])
|
||||
per_q.append({"q": q["id"], "kind": q["kind"], **{k: round(v, 3) for k, v in m.items()},
|
||||
"top5": ranked[:5]})
|
||||
for k, v in m.items():
|
||||
agg.setdefault(k, []).append(v)
|
||||
agg = {k: round(float(np.mean(v)), 3) for k, v in agg.items()}
|
||||
thr = threshold_analysis(sim, queries, entries)
|
||||
trp = trap_analysis(sim, queries, entries, thr["prec90_threshold"])
|
||||
return {"model": name, "hf": spec["hf"], "dim": dim,
|
||||
"load_s": round(load_s, 1), "encode_s": round(enc_s, 2),
|
||||
"aggregate": agg, "threshold": thr, "trap": trp, "per_query": per_q}
|
||||
|
||||
|
||||
# ---------- baselines ----------
|
||||
|
||||
def tok(s):
|
||||
"""char-level for CJK, lowercase word-level for latin/cyrillic."""
|
||||
out = []
|
||||
for m in re.finditer(r"[-ヿ㐀-鿿]|[A-Za-zА-Яа-яЁё0-9]+", s):
|
||||
out.append(m.group(0).lower())
|
||||
return out
|
||||
|
||||
|
||||
def run_substring(queries, entries):
|
||||
"""Deterministic exact substring/alias key-match = the Р3 hot path."""
|
||||
def keys(e):
|
||||
return [e["src"]] + e.get("aliases", []) if e["type"] != "summary" else []
|
||||
tp = fp = fn = 0
|
||||
per_q, trap_fires = [], 0
|
||||
for q in queries:
|
||||
hit = [e["id"] for e in entries if any(k and k in q["text"] for k in keys(e))]
|
||||
if q["kind"] in GENUINE_KINDS:
|
||||
gold = set(q["gold"]); h = set(hit)
|
||||
tp += len(h & gold); fp += len(h - gold); fn += len(gold - h)
|
||||
per_q.append({"q": q["id"], "kind": q["kind"], "hits": hit,
|
||||
"recall": round(len(h & gold) / len(gold), 3),
|
||||
"missed": sorted(gold - h)})
|
||||
if q["kind"] == "trap":
|
||||
fired = [e for e in hit if e in q["traps"]]
|
||||
trap_fires += len(fired)
|
||||
per_q.append({"q": q["id"], "kind": "trap", "hits": hit, "false_fire": fired})
|
||||
prec = tp / (tp + fp) if (tp + fp) else 1.0
|
||||
rec = tp / (tp + fn) if (tp + fn) else 0.0
|
||||
return {"method": "substring-exact", "precision": round(prec, 3), "recall": round(rec, 3),
|
||||
"trap_false_fires": trap_fires, "per_query": per_q}
|
||||
|
||||
|
||||
def run_bm25(queries, entries):
|
||||
from rank_bm25 import BM25Okapi
|
||||
corpus = [tok(e["embed_text"]) for e in entries]
|
||||
bm = BM25Okapi(corpus)
|
||||
entry_ids = [e["id"] for e in entries]
|
||||
agg, per_q, trap_top5 = {}, [], 0
|
||||
for q in queries:
|
||||
sc = np.array(bm.get_scores(tok(q["text"])))
|
||||
if q["kind"] in GENUINE_KINDS:
|
||||
m, ranked = rank_metrics(sc, entry_ids, q["gold"])
|
||||
per_q.append({"q": q["id"], "kind": q["kind"], **{k: round(v, 3) for k, v in m.items()}})
|
||||
for k, v in m.items():
|
||||
agg.setdefault(k, []).append(v)
|
||||
if q["kind"] == "trap":
|
||||
order = np.argsort(-sc)
|
||||
top5 = {entry_ids[i] for i in order[:5]}
|
||||
if set(q["traps"]) & top5:
|
||||
trap_top5 += 1
|
||||
agg = {k: round(float(np.mean(v)), 3) for k, v in agg.items()}
|
||||
return {"method": "bm25-char", "aggregate": agg, "trap_in_top5": trap_top5, "per_query": per_q}
|
||||
|
||||
|
||||
# ---------- scale ----------
|
||||
|
||||
def scale_bench(dims=(768, 1024), Ns=(1000, 10000, 50000, 100000, 200000), reps=20):
|
||||
"""Model-agnostic brute-force cosine KNN latency vs N (single query, top-10)."""
|
||||
rng = np.random.default_rng(42)
|
||||
res = {}
|
||||
for d in dims:
|
||||
row = {}
|
||||
for N in Ns:
|
||||
mat = rng.standard_normal((N, d)).astype(np.float32)
|
||||
mat /= np.linalg.norm(mat, axis=1, keepdims=True)
|
||||
q = rng.standard_normal(d).astype(np.float32); q /= np.linalg.norm(q)
|
||||
ts = []
|
||||
for _ in range(reps):
|
||||
t = time.perf_counter()
|
||||
s = mat @ q
|
||||
np.argpartition(-s, 10)[:10]
|
||||
ts.append((time.perf_counter() - t) * 1000)
|
||||
row[N] = round(float(np.median(ts)), 2)
|
||||
res[d] = row
|
||||
return res
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--models", default="bge-m3,qwen3-0.6b,labse")
|
||||
ap.add_argument("--skip-scale", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
queries, entries = load_corpus()
|
||||
n_gold = sum(1 for q in queries if q["kind"] in GENUINE_KINDS)
|
||||
print(f"corpus: {len(entries)} entries, {len(queries)} queries "
|
||||
f"({n_gold} with gold, {sum(1 for q in queries if q['kind']=='trap')} traps)\n")
|
||||
|
||||
results = {"embedding": [], "baselines": {}, "scale": None}
|
||||
|
||||
print("=== baselines ===")
|
||||
sub = run_substring(queries, entries)
|
||||
results["baselines"]["substring"] = sub
|
||||
print(f"substring-exact: precision={sub['precision']} recall={sub['recall']} "
|
||||
f"trap_false_fires={sub['trap_false_fires']}")
|
||||
bm = run_bm25(queries, entries)
|
||||
results["baselines"]["bm25"] = bm
|
||||
print(f"bm25-char: {bm['aggregate']} trap_in_top5={bm['trap_in_top5']}\n")
|
||||
|
||||
print("=== embedding models (CPU) ===")
|
||||
for name in [m.strip() for m in args.models.split(",") if m.strip()]:
|
||||
spec = MODELS[name]
|
||||
try:
|
||||
r = run_embed_model(name, spec, queries, entries)
|
||||
except Exception as e:
|
||||
print(f"{name}: FAILED — {type(e).__name__}: {e}")
|
||||
results["embedding"].append({"model": name, "error": f"{type(e).__name__}: {e}"})
|
||||
continue
|
||||
results["embedding"].append(r)
|
||||
t = r["threshold"]
|
||||
print(f"\n[{name}] dim={r['dim']} load={r['load_s']}s encode={r['encode_s']}s")
|
||||
print(f" retrieval: {r['aggregate']}")
|
||||
print(f" threshold: ROC-AUC={t['roc_auc']:.3f} rel_mean={t['rel_mean']:.3f} "
|
||||
f"rel_min={t['rel_min']:.3f} irr_mean={t['irr_mean']:.3f} irr_p95={t['irr_p95']:.3f}")
|
||||
print(f" prec≥0.9 @ cos≥{t['prec90_threshold']} → recall={t['recall_at_prec90']:.3f}")
|
||||
tr = r["trap"]
|
||||
print(f" wrong-sense traps injected at t*: {tr['n_injected_at_t*']}/{len(tr['traps'])} "
|
||||
f"| {[(x['q'], x['target'], x['cos']) for x in tr['traps']]}")
|
||||
if tr["negative"]:
|
||||
print(f" unrelated-query top1 cos: {tr['negative']['top1_cos']} "
|
||||
f"(abstains at t*: {tr['negative']['abstains_at_t*']})")
|
||||
|
||||
if not args.skip_scale:
|
||||
print("\n=== brute-force KNN latency (ms, single query, top-10, median) ===")
|
||||
sc = scale_bench()
|
||||
results["scale"] = sc
|
||||
for d, row in sc.items():
|
||||
print(f" dim={d}: " + " ".join(f"N={N}:{ms}ms" for N, ms in row.items()))
|
||||
|
||||
OUT.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"\nsaved → {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue