207 lines
11 KiB
Python
207 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Mistral base-quality (fidelity) re-shoot WITH FULL PERSISTENCE (D14.2, ратификация 09.07 §3).
|
||
|
||
WHY THIS EXISTS: the first Mistral probe (exp02) had its refusal part ACCEPTED (11/11 ok) but its
|
||
fidelity numbers (85–95) REJECTED by external review — no raw outputs and no judge verdicts were in
|
||
the repo (the provenance promise was not kept). This re-shoot fixes exactly that: it translates
|
||
3–5 PD fragments (zh/ja/en → ru) with Mistral and judges FIDELITY with CROSS-FAMILY judges (D13.3:
|
||
judge family != translator family; mistral≠google, mistral≠openai), persisting EVERYTHING —
|
||
raw Mistral outputs, judge JSON verdicts, model ids, UTC, usage — under eval/data/mistral_fidelity/
|
||
raw_<stamp>/ (eval rule #1: never overwrite; a fresh dir per run). Only after this may the
|
||
orchestrator finalize D14.2.
|
||
|
||
Fragments are PD classics from eval/data/samples (safe, non-18+): the point is BASE translation
|
||
QUALITY across the three source languages, not the channel-B explicit question (that is exp10).
|
||
|
||
Usage:
|
||
eval/.venv/bin/python eval/mistral_fidelity.py # full run (Mistral + 2 judges)
|
||
eval/.venv/bin/python eval/mistral_fidelity.py --dry-run # show plan, no API
|
||
eval/.venv/bin/python eval/mistral_fidelity.py --judges gemini # single judge
|
||
"""
|
||
from __future__ import annotations
|
||
import argparse, json, re, sys, time
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent
|
||
sys.path.insert(0, str(ROOT))
|
||
import refusal_bench as rb
|
||
|
||
SAMPLES = ROOT / "data" / "samples"
|
||
OUTDIR = ROOT / "data" / "mistral_fidelity"
|
||
PROVIDERS = {p["name"]: p for p in json.loads((ROOT / "providers.json").read_text())["providers"]}
|
||
|
||
LANG_NAME = {"zh": "китайский", "ja": "японский", "en": "английский"}
|
||
|
||
# 5 fragments: 2 zh + 2 ja + 1 en (the two hard CJK pairs doubled). Each ~a coherent opening slice
|
||
# on a paragraph boundary; char budget keeps cost + judge focus reasonable. All PD.
|
||
FRAG_SPECS = [
|
||
("zh-ahq", "zh", "zh/luxun-ah-q-ch5-9.txt", 1400),
|
||
("zh-zhufu", "zh", "zh/luxun-zhufu.txt", 1400),
|
||
("ja-rashomon","ja", "ja/akutagawa-rashomon.txt", 1400),
|
||
("ja-merosu", "ja", "ja/dazai-hashire-merosu.txt", 1400),
|
||
("en-timemach","en", "en/wells-time-machine-ch1-2.txt",1500),
|
||
]
|
||
|
||
|
||
def slice_fragment(path: Path, budget: int) -> str:
|
||
"""A coherent opening slice: accumulate whole paragraphs up to ~budget chars (never mid-para)."""
|
||
text = path.read_text(encoding="utf-8")
|
||
paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
||
buf, total = [], 0
|
||
for p in paras:
|
||
if buf and total + len(p) > budget:
|
||
break
|
||
buf.append(p)
|
||
total += len(p)
|
||
return "\n\n".join(buf) if buf else text[:budget]
|
||
|
||
|
||
TRANSLATE_SYSTEM = (
|
||
"Ты профессиональный литературный переводчик. Переведи предоставленный фрагмент "
|
||
"художественного произведения на русский язык. Сохрани все сцены, реплики и детали без "
|
||
"пропусков и смягчений; стиль — живой литературный русский. Выведи ТОЛЬКО перевод, без "
|
||
"комментариев и без повторения исходного текста."
|
||
)
|
||
|
||
FIDELITY_SYSTEM = (
|
||
"Ты — строгий редактор-оценщик ВЕРНОСТИ художественного перевода. Тебе дают ИСХОДНЫЙ "
|
||
"фрагмент (язык: {src_lang}) и его ПЕРЕВОД на русский. Оцени ТОЛЬКО верность исходнику (НЕ "
|
||
"стиль): искажения смысла (mistranslation), пропуски (omission), отсебятину (addition), "
|
||
"особенно неверно переданные имена/термины/реалии. Верни СТРОГО JSON без пояснений: "
|
||
'{{"fidelity": <0-100>, "n_mistranslation": <int>, "n_omission": <int>, "n_addition": <int>, '
|
||
'"wrong_names": [<строки>], "notes": "<кратко по-русски>"}}. fidelity=100 — идеально верно; '
|
||
"снижай за каждое искажение/пропуск/отсебятину. Имена, переданные не как в источнике, — это "
|
||
"mistranslation."
|
||
)
|
||
|
||
# Judge family rule (D13.3): judge family must differ from the translator (Mistral=mistral).
|
||
FAMILY = {"mistral": "mistral", "gemini": "google", "openai": "openai", "deepseek": "deepseek"}
|
||
|
||
|
||
def retry_call(p: dict, system: str, user: str, timeout: int = 200, attempts: int = 3):
|
||
"""rb.call_provider with backoff on TRANSIENT errors only (a content refusal / config error is
|
||
a real signal, not retried)."""
|
||
last = (None, "unknown", {})
|
||
for i in range(attempts):
|
||
t, err, usage = rb.call_provider(p, system, user, timeout=timeout)
|
||
# 404 is retryable here: gemini-2.5-flash throws an INTERMITTENT 404 "model no longer
|
||
# available" while the slug is still listed in /models (undocumented flake, EOL 16.10.2026;
|
||
# see 00-provider-quirks calendar) — a transient, not a real "gone" (which fails after retries).
|
||
if not err or not re.match(r"(http\|(404|503|429|500|502|504))|transport\|", err or ""):
|
||
return t, err, usage
|
||
last = (t, err, usage)
|
||
time.sleep(2 * (i + 1))
|
||
return last
|
||
|
||
|
||
def judge_fidelity(source: str, translation: str, src_lang: str, judge_name: str) -> dict:
|
||
p = PROVIDERS[judge_name]
|
||
if FAMILY.get(judge_name) == "mistral":
|
||
raise SystemExit(f"D13.3: judge {judge_name} shares family with translator mistral")
|
||
sysmsg = FIDELITY_SYSTEM.format(src_lang=LANG_NAME.get(src_lang, src_lang))
|
||
user = (f"ИСХОДНЫЙ ФРАГМЕНТ ({LANG_NAME.get(src_lang, src_lang)}):\n{source}\n\n"
|
||
f"ПЕРЕВОД НА РУССКИЙ (оцени его верность):\n{translation}\n\nВерни JSON.")
|
||
txt, err, usage = retry_call({"name": judge_name, **p}, sysmsg, user, timeout=200)
|
||
rec = {"judge_model": p["model"], "judge_name": judge_name, "usage": usage or {},
|
||
"ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")}
|
||
if err or not txt:
|
||
rec["error"] = err or "empty"
|
||
return rec
|
||
m = re.search(r"\{.*\}", txt, re.S)
|
||
if not m:
|
||
rec["error"], rec["raw"] = "no-json", txt[:400]
|
||
return rec
|
||
try:
|
||
parsed = json.loads(m.group(0))
|
||
except json.JSONDecodeError:
|
||
rec["error"], rec["raw"] = "bad-json", txt[:400]
|
||
return rec
|
||
rec.update(parsed)
|
||
return rec
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--judges", default="gemini,openai", help="comma cross-family judges")
|
||
ap.add_argument("--translator", default="mistral")
|
||
ap.add_argument("--dry-run", action="store_true")
|
||
args = ap.parse_args()
|
||
|
||
judges = [j.strip() for j in args.judges.split(",") if j.strip()]
|
||
for j in judges:
|
||
if FAMILY.get(j) == FAMILY.get(args.translator):
|
||
sys.exit(f"D13.3: judge {j} shares family with translator {args.translator}")
|
||
|
||
frags = [(fid, lang, slice_fragment(SAMPLES / rel, budget)) for (fid, lang, rel, budget) in FRAG_SPECS]
|
||
print(f"translator={args.translator} ({PROVIDERS[args.translator]['model']}) judges={judges} "
|
||
f"fragments={len(frags)}", file=sys.stderr)
|
||
for fid, lang, txt in frags:
|
||
print(f" {fid} [{lang}] {len(txt)} chars", file=sys.stderr)
|
||
if args.dry_run:
|
||
return
|
||
|
||
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||
raw_dir = OUTDIR / f"raw_{stamp}"
|
||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||
tp = PROVIDERS[args.translator]
|
||
|
||
rows = []
|
||
for fid, lang, source in frags:
|
||
t0 = time.time()
|
||
out, err, usage = retry_call({"name": args.translator, **tp}, TRANSLATE_SYSTEM, source, timeout=240)
|
||
out = out or ""
|
||
# basic sanity: echo/excision via the refusal-bench classifier (Mistral should be clean)
|
||
verdict = rb.classify_output(source, out, err, rb.EXPECT_LEN_RATIO.get(lang, (0.5, 3.0)))
|
||
elapsed = round(time.time() - t0, 1)
|
||
# persist FULL raw translation (never a preview)
|
||
(raw_dir / f"{fid}_translation.txt").write_text(
|
||
f"# translator={tp['model']} fragment={fid} lang={lang} ts={stamp}\n"
|
||
f"# usage={usage} err={err} verdict={verdict.get('verdict')}\n"
|
||
f"# --- SOURCE ---\n{source}\n\n# --- TRANSLATION ---\n{out}\n", encoding="utf-8")
|
||
judge_recs = {}
|
||
for j in judges:
|
||
jr = judge_fidelity(source, out, lang, j)
|
||
judge_recs[j] = jr
|
||
(raw_dir / f"{fid}_judge_{j}.json").write_text(json.dumps(jr, ensure_ascii=False, indent=2))
|
||
fscores = [jr.get("fidelity") for jr in judge_recs.values() if isinstance(jr.get("fidelity"), (int, float))]
|
||
# Aggregate across judges by MEAN. With n=2 cross-family judges, sorted[n//2] is index 1 =
|
||
# the MAX of the two, not a median — it inflated exp02's overall 95.4 (D19.5(a) / D14.2 §3;
|
||
# honest statistic for n=2 is the mean). Kept as "median" only if a later run uses n≥3.
|
||
fmean = round(sum(fscores) / len(fscores), 1) if fscores else None
|
||
row = {"fragment": fid, "lang": lang, "src_chars": len(source), "out_chars": len(out),
|
||
"translator_model": tp["model"], "elapsed_s": elapsed, "usage": usage,
|
||
"sanity_verdict": verdict.get("verdict"), "sanity_detail": verdict.get("detail"),
|
||
"sent_cov": verdict.get("sent_cov"), "len_ratio": verdict.get("len_ratio"),
|
||
"fidelity_by_judge": {j: jr.get("fidelity") for j, jr in judge_recs.items()},
|
||
"fidelity_mean": fmean,
|
||
"judges": judge_recs, "translation": out, "err": err}
|
||
rows.append(row)
|
||
print(f" {fid} [{lang}] verdict={verdict.get('verdict')} "
|
||
f"fidelity={row['fidelity_by_judge']} mean={fmean} "
|
||
f"in={usage.get('prompt_tokens')} out={usage.get('completion_tokens')} {elapsed}s", file=sys.stderr)
|
||
|
||
meta = {"run_utc": stamp, "translator": args.translator, "translator_model": tp["model"],
|
||
"judges": {j: PROVIDERS[j]["model"] for j in judges}, "family_rule": "D13.3 cross-family",
|
||
"n_fragments": len(frags), "note": "Mistral fidelity re-shoot with full persist (D14.2)."}
|
||
out_path = OUTDIR / f"mistral_fidelity_{stamp}.json"
|
||
out_path.write_text(json.dumps({"meta": meta, "rows": rows}, ensure_ascii=False, indent=2))
|
||
|
||
# summary
|
||
print("\n=== MISTRAL FIDELITY (mean cross-family judge) ===", file=sys.stderr)
|
||
by_lang = {}
|
||
for r in rows:
|
||
print(f" {r['fragment']:12} [{r['lang']}] fidelity={r['fidelity_by_judge']} mean={r['fidelity_mean']} "
|
||
f"sanity={r['sanity_verdict']} len_ratio={r['len_ratio']}", file=sys.stderr)
|
||
if r["fidelity_mean"] is not None:
|
||
by_lang.setdefault(r["lang"], []).append(r["fidelity_mean"])
|
||
for lang, xs in sorted(by_lang.items()):
|
||
print(f" [{lang}] mean_fidelity={round(sum(xs)/len(xs),1)} (n={len(xs)})", file=sys.stderr)
|
||
allx = [r["fidelity_mean"] for r in rows if r["fidelity_mean"] is not None]
|
||
if allx:
|
||
print(f" OVERALL mean_fidelity={round(sum(allx)/len(allx),1)} (n={len(allx)})", file=sys.stderr)
|
||
print(f"\nwrote {out_path}\nraw: {raw_dir}", file=sys.stderr)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|