129 lines
6.7 KiB
Python
129 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
||
"""exp16 — A6b: local 9B as a Z1 VERIFIER of translation pairs (research/20 §D2 A6b, §B4 Z1). $0 (stand).
|
||
|
||
§B4 routes SPOT to code (A6 showed code>=local 9B) and reserves the local 9B for VERIFICATION of the
|
||
stably-wrong class Z1 (蛊→«гусеница»: high termhood, zero spread, dst is a common word — invisible to the
|
||
spread signal). This mini-bench asks: can a local 9B judge whether a ru rendering of a zh term IN CONTEXT
|
||
is faithful? Gold is AUTOMATIC: CORRECT pairs use the owner-signed seed dst; WRONG pairs inject the Z1
|
||
failure class (plausible common-word mistranslations + cross-term swaps). Accuracy + wrong-catch rate.
|
||
|
||
30 pairs (15 correct + 15 wrong). Deterministic (temp 0, think off). Health-checked.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
import exp16_common as X
|
||
|
||
MODEL = "huihui_ai/qwen3.5-abliterated:9b"
|
||
OLLAMA = "http://localhost:11434/api/chat"
|
||
OUT = Path("/home/ubuntu/books/gu-zhenren/exp16")
|
||
_opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
|
||
# Z1 verification bench: (src_term, correct_dst, wrong_dst[Z1 stably-wrong class]). Wrong = a plausible
|
||
# common Russian word or a cross-term swap that the spread signal would NOT catch.
|
||
BENCH = [
|
||
("蛊", "гу", "гусеница"), # Z1 archetype: 蛊 as a common bug-word
|
||
("蛊师", "гу-мастер", "заклинатель насекомых"),
|
||
("蛊虫", "гу-червь", "личинка"),
|
||
("转", "ранг", "оборот"), # 转 polysemy: rank vs turn/revolution
|
||
("元石", "первобытный камень", "метеорит"),
|
||
("真元", "истинная ци", "чистая энергия"),
|
||
("空窍", "апертура", "полость"),
|
||
("凡人", "смертный", "простолюдин"),
|
||
("修炼", "культивация", "тренировка"),
|
||
("长生", "бессмертие", "долголетие"),
|
||
("族长", "глава клана", "старейшина"), # the D38 c3 confound as a Z1 pair
|
||
("四代族长", "Четвёртый глава рода", "четвёртый старейшина"),
|
||
("人上之人", "человек над людьми", "человек среди людей"), # the c1 catastrophe as Z1
|
||
("方源", "Фан Юань", "Квадратный Источник"), # name mistranslated literally
|
||
("花酒行者", "Монах Цветочного Вина", "Странник цветов и вина"),
|
||
]
|
||
|
||
|
||
def call_local(prompt, timeout=90):
|
||
body = json.dumps({"model": MODEL, "think": False,
|
||
"messages": [{"role": "user", "content": prompt + "\n/no_think"}],
|
||
"stream": False, "options": {"temperature": 0.0, "num_predict": 200}}).encode()
|
||
req = urllib.request.Request(OLLAMA, data=body, headers={"Content-Type": "application/json"})
|
||
try:
|
||
with _opener.open(req, timeout=timeout) as r:
|
||
d = json.loads(r.read())
|
||
return (d.get("message", {}) or {}).get("content", ""), None
|
||
except Exception as e:
|
||
return "", f"{type(e).__name__}: {str(e)[:120]}"
|
||
|
||
|
||
def find_sentence(term, chunks):
|
||
tn = X.norm(term)
|
||
for c in chunks:
|
||
for sent in re.split(r"(?<=[。！？])", c.source):
|
||
if term in sent and 6 <= len(sent) <= 80:
|
||
return sent.strip()
|
||
return term
|
||
|
||
|
||
def verdict_yes_no(text):
|
||
text = re.sub(r"<think>.*?</think>", "", text, flags=re.S).lower()
|
||
# first explicit yes/да or no/нет
|
||
m = re.search(r"\b(да|верно|correct|yes|правильн)\b", text)
|
||
n = re.search(r"\b(нет|неверно|incorrect|no|ошибочн|неправильн)\b", text)
|
||
if n and (not m or n.start() < m.start()):
|
||
return "no"
|
||
if m:
|
||
return "yes"
|
||
return "?"
|
||
|
||
|
||
def main():
|
||
chunks = X.load_chunks()
|
||
pairs = []
|
||
for term, ok_dst, bad_dst in BENCH:
|
||
sent = find_sentence(term, chunks)
|
||
pairs.append((term, sent, ok_dst, "correct"))
|
||
pairs.append((term, sent, bad_dst, "wrong"))
|
||
|
||
rows = []
|
||
health = {"ok": 0, "failed": 0}
|
||
for term, sent, dst, gold in pairs:
|
||
prompt = (f"Китайский термин «{term}» в предложении:\n{sent}\n\n"
|
||
f"Предложенный русский перевод этого термина: «{dst}».\n\n"
|
||
f"Верен ли этот перевод термина по смыслу в данном контексте? "
|
||
f"Ответь одним словом: ДА или НЕТ, затем короткое обоснование.")
|
||
out, err = call_local(prompt)
|
||
if err:
|
||
health["failed"] += 1
|
||
rows.append(dict(term=term, dst=dst, gold=gold, pred="failed", err=err)); continue
|
||
health["ok"] += 1
|
||
pred = verdict_yes_no(out)
|
||
model_says = "correct" if pred == "yes" else ("wrong" if pred == "no" else "?")
|
||
rows.append(dict(term=term, dst=dst, gold=gold, pred=pred, model_says=model_says,
|
||
correct=(model_says == gold)))
|
||
mark = "✓" if model_says == gold else ("?" if pred == "?" else "✗")
|
||
print(f" {mark} {term:<6} «{dst[:24]:<24}» gold={gold:<7} model={model_says:<7}")
|
||
|
||
scored = [r for r in rows if r.get("pred") not in ("failed", "?")]
|
||
acc = sum(r["correct"] for r in scored) / len(scored) if scored else 0
|
||
# wrong-catch (recall on the Z1 wrong class) + correct-accept (specificity)
|
||
wrong = [r for r in scored if r["gold"] == "wrong"]
|
||
correct = [r for r in scored if r["gold"] == "correct"]
|
||
wrong_catch = sum(r["model_says"] == "wrong" for r in wrong) / len(wrong) if wrong else 0
|
||
correct_accept = sum(r["model_says"] == "correct" for r in correct) / len(correct) if correct else 0
|
||
print(f"\n=== A6b Z1 verifier ({MODEL}) — {len(pairs)} pairs ===")
|
||
print(f"health: {health}; scored: {len(scored)}")
|
||
print(f"accuracy: {acc:.3f}")
|
||
print(f"wrong-catch (recall on Z1 mistranslations): {sum(r['model_says']=='wrong' for r in wrong)}/{len(wrong)} = {wrong_catch:.3f}")
|
||
print(f"correct-accept (specificity): {sum(r['model_says']=='correct' for r in correct)}/{len(correct)} = {correct_accept:.3f}")
|
||
missed = [r["term"] for r in wrong if r["model_says"] != "wrong"]
|
||
print(f"Z1 mistranslations the local 9B FAILED to catch: {missed}")
|
||
json.dump({"model": MODEL, "health": health, "accuracy": acc, "wrong_catch": wrong_catch,
|
||
"correct_accept": correct_accept, "rows": rows},
|
||
open(OUT / "a6b_verify.json", "w"), ensure_ascii=False, indent=1)
|
||
print(f"[written] {OUT/'a6b_verify.json'}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|