242 lines
13 KiB
Python
242 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""exp15 — pairwise span-citing trap judge (design §D3 / §1.6). Built from scratch: the exp14 harness
|
||
has NEITHER both-orders NOR 6->10 escalation.
|
||
|
||
Per trap-cell (one trap × one A0-vs-arm contrast), each of >=2 cross-family judges (grok-4.3 reasoning-ON
|
||
+ mistral-large-latest) is asked, BLIND (V1/V2), in BOTH orders, REP times, whether each translation
|
||
correctly resolves the SPECIFIC cohesion phenomenon (cite a <=15-word span). We record per-arm verdicts
|
||
(correct/partial/wrong -> accuracy) AND the pairwise winner AND catastrophe flags. Aggregation:
|
||
base 6 votes/judge (3 reps × 2 orders); if a judge's decisive split is <=4:2 -> +4 to 10 (pre-reg rule,
|
||
budget-relaxed from §A.7 "~10", fixed explicitly). Leave-one-judge-out. Catastrophe screen to the winner too.
|
||
Primary Q1/Q3 metric = per-arm ACCURACY (correct=1/partial=0.5/wrong=0), paired diff arm-A0. Pairwise
|
||
"better" is a robustness cross-check.
|
||
|
||
temp 0 (grok reasoning-ON is mildly stochastic across reps; mistral deterministic -> reps agree = strong
|
||
consensus, not wasted signal). Every call persists usage+cost via exp15_llm.Spender; per-call gate on.
|
||
"""
|
||
from __future__ import annotations
|
||
import json
|
||
import re
|
||
import sys
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
import exp15_llm as L
|
||
|
||
JUDGES = ["grok-4.3", "mistral-large-latest"]
|
||
BASE_REPS_PER_ORDER = 3 # 3 reps × 2 orders = 6 base votes/judge
|
||
ESCALATE_REPS_PER_ORDER = 2 # +2 reps × 2 orders = +4 -> 10 on split
|
||
SCORE = {"correct": 1.0, "partial": 0.5, "wrong": 0.0}
|
||
|
||
JUDGE_SYS = (
|
||
"Ты — придирчивый билингвальный (китайский↔русский) литературный рецензент. Тебе дают ОДИН конкретный "
|
||
"переводческий феномен связности (анафора, время/род глагола, катафора, эллипсис, перенос сущности через "
|
||
"границу и т.п.), фрагмент КИТАЙСКОГО исходника и ДВА перевода этого места — V1 и V2. Оцени СТРОГО и ТОЛЬКО "
|
||
"заданный феномен (не общий стиль). Для каждого перевода реши: correct (феномен передан верно), partial "
|
||
"(частично/неоднозначно), wrong (искажён/потерян/неверный род-время-референт). Затем реши, какой перевод "
|
||
"лучше справился ИМЕННО с этим феноменом: V1, V2 или tie. Отметь catastrophe=true, если перевод даёт грубую "
|
||
"смысловую ошибку по этому феномену (неверный род/референт/полярность/потеря). Приведи доказательство — "
|
||
"цитату <=15 слов из каждого перевода. Выведи СТРОГО JSON без пояснений:\n"
|
||
'{"v1":"correct|partial|wrong","v2":"correct|partial|wrong","better":"V1|V2|tie",'
|
||
'"cat_v1":true|false,"cat_v2":true|false,"ev_v1":"...","ev_v2":"..."}'
|
||
)
|
||
|
||
|
||
def _user_prompt(trap, v1_text, v2_text):
|
||
return (
|
||
f"ФЕНОМЕН СВЯЗНОСТИ ({trap['type']}): {trap['phenomenon']}\n"
|
||
f"ЧТО ТРЕБУЕТСЯ ДЛЯ ВЕРНОЙ ПЕРЕДАЧИ: {trap['expected']}\n\n"
|
||
f"КИТАЙСКИЙ ИСХОДНИК (место феномена):\n{trap['src_zone']}\n\n"
|
||
f"V1 (перевод этого места):\n{v1_text}\n\n"
|
||
f"V2 (перевод этого места):\n{v2_text}\n\n"
|
||
"Оцени ТОЛЬКО указанный феномен. Ответ — строго JSON по схеме."
|
||
)
|
||
|
||
|
||
_JSON_RE = re.compile(r"\{.*\}", re.S)
|
||
|
||
|
||
def _as_bool(x):
|
||
"""Robust truthiness for the catastrophe flags. A judge that emits the boolean as a JSON STRING
|
||
("cat_v1":"false") would otherwise become bool("false")==True -> a spurious catastrophe (review I4)."""
|
||
return x is True or (isinstance(x, str) and x.strip().lower() in ("true", "1", "yes"))
|
||
|
||
|
||
def parse_verdict(text):
|
||
m = _JSON_RE.search(text or "")
|
||
if not m:
|
||
return None
|
||
try:
|
||
d = json.loads(m.group(0))
|
||
except Exception:
|
||
# tolerant: pull ALL fields by regex (incl cat flags + evidence — review I4: the old fallback
|
||
# dropped cat/ev, silently zeroing a flagged catastrophe on a parse-degraded but non-empty vote)
|
||
d = {}
|
||
for k in ("v1", "v2", "better"):
|
||
mm = re.search(rf'"{k}"\s*:\s*"(\w+)"', text)
|
||
if mm:
|
||
d[k] = mm.group(1)
|
||
for k in ("cat_v1", "cat_v2"):
|
||
mm = re.search(rf'"{k}"\s*:\s*(true|false|"true"|"false")', text, re.I)
|
||
if mm:
|
||
d[k] = mm.group(1)
|
||
for k in ("ev_v1", "ev_v2"):
|
||
mm = re.search(rf'"{k}"\s*:\s*"([^"]*)"', text)
|
||
if mm:
|
||
d[k] = mm.group(1)
|
||
for k in ("v1", "v2"):
|
||
if d.get(k) not in SCORE:
|
||
return None
|
||
if d.get("better") not in ("V1", "V2", "tie"):
|
||
d["better"] = "tie"
|
||
d["cat_v1"] = _as_bool(d.get("cat_v1", False))
|
||
d["cat_v2"] = _as_bool(d.get("cat_v2", False))
|
||
return d
|
||
|
||
|
||
@dataclass
|
||
class Vote:
|
||
judge: str
|
||
order: str # "AB" (a0=V1, arm=V2) or "BA" (arm=V1, a0=V2)
|
||
a0_verdict: str # correct/partial/wrong
|
||
arm_verdict: str
|
||
winner: str # "a0" | "arm" | "tie"
|
||
a0_cat: bool
|
||
arm_cat: bool
|
||
ev_a0: str = "" # judge's <=15-word evidence span for the a0 side (order-normalised)
|
||
ev_arm: str = "" # judge's evidence span for the arm side
|
||
raw: dict = field(default_factory=dict)
|
||
|
||
|
||
# a judge verdict is a tiny JSON (~60-150 completion tokens); grok reasoning is separate/uncapped by
|
||
# max_tokens. Cap max_tokens LOW so predict_cost's (max_out + reasoning_ceil) gate estimate stays under
|
||
# the judge per-call cap — otherwise grok's 8000-default made predict ~$0.033 > $0.03 and grok was
|
||
# silently gate-blocked (0 grok votes; the bug this selftest caught).
|
||
JUDGE_MAX_TOKENS = 1500
|
||
|
||
|
||
def _one_call(sp: L.Spender, judge, trap, v1_text, v2_text):
|
||
"""Return (verdict|None, rec, status). status in {ok, gate_block, err, parse_fail} so the caller can
|
||
tell an out-of-budget refusal (gate_block) from a model parse failure (review I8)."""
|
||
msgs = [{"role": "system", "content": JUDGE_SYS},
|
||
{"role": "user", "content": _user_prompt(trap, v1_text, v2_text)}]
|
||
in_est = int((len(JUDGE_SYS) + len(msgs[1]["content"])) / 1.9) # conservative chars/token (review NIT)
|
||
ok, pred, why = sp.gate(judge, in_est, JUDGE_MAX_TOKENS)
|
||
if not ok:
|
||
return None, {"judge": judge, "trap": trap["id"], "err": why, "cost": 0.0}, "gate_block"
|
||
r = L.call(judge, msgs, temp=0.0, max_tokens=JUDGE_MAX_TOKENS)
|
||
rec = {"judge": judge, "trap": trap["id"], "usage": r.get("usage"), "cost": r.get("cost", 0.0),
|
||
"err": r.get("err"), "finish": r.get("finish"), "latency_s": r.get("latency_s"),
|
||
"retry_fails": r.get("retry_fails")}
|
||
sp.record(rec)
|
||
import os
|
||
if r.get("err"):
|
||
if os.environ.get("JUDGE_DEBUG"):
|
||
print(f" [DBG {judge}] ERR {r['err'][:70]}")
|
||
return None, rec, "err"
|
||
v = parse_verdict(r["text"])
|
||
if v is None:
|
||
if os.environ.get("JUDGE_DEBUG"):
|
||
print(f" [DBG {judge}] NOPARSE content[{len(r.get('text') or '')}]={repr((r.get('text') or '')[:80])} "
|
||
f"finish={r.get('finish')} reasoning[{len(r.get('reasoning') or '')}]")
|
||
return None, rec, "parse_fail"
|
||
return v, rec, "ok"
|
||
|
||
|
||
def judge_cell(sp: L.Spender, trap, a0_text, arm_text, judges=JUDGES,
|
||
base_reps=BASE_REPS_PER_ORDER, esc_reps=ESCALATE_REPS_PER_ORDER):
|
||
"""Return per-cell aggregate with per-arm accuracy, pairwise, catastrophe, per-judge (for LOO)."""
|
||
votes: list[Vote] = []
|
||
parse_fail = 0
|
||
budget_truncated = False
|
||
for judge in judges:
|
||
if budget_truncated:
|
||
break
|
||
jvotes = []
|
||
def do_reps(reps):
|
||
nonlocal parse_fail, budget_truncated
|
||
for _ in range(reps):
|
||
if budget_truncated:
|
||
return
|
||
for order in ("AB", "BA"):
|
||
if order == "AB":
|
||
v1, v2 = a0_text, arm_text
|
||
else:
|
||
v1, v2 = arm_text, a0_text
|
||
d, rec, status = _one_call(sp, judge, trap, v1, v2)
|
||
if status == "gate_block":
|
||
budget_truncated = True # out of budget -> stop this cell, do NOT score partial
|
||
return
|
||
if d is None:
|
||
parse_fail += 1
|
||
continue
|
||
if order == "AB":
|
||
a0v, armv = d["v1"], d["v2"]
|
||
win = {"V1": "a0", "V2": "arm", "tie": "tie"}[d["better"]]
|
||
a0c, armc = bool(d["cat_v1"]), bool(d["cat_v2"])
|
||
eva0, evarm = d.get("ev_v1", ""), d.get("ev_v2", "")
|
||
else:
|
||
a0v, armv = d["v2"], d["v1"]
|
||
win = {"V1": "arm", "V2": "a0", "tie": "tie"}[d["better"]]
|
||
a0c, armc = bool(d["cat_v2"]), bool(d["cat_v1"])
|
||
eva0, evarm = d.get("ev_v2", ""), d.get("ev_v1", "")
|
||
jvotes.append(Vote(judge, order, a0v, armv, win, a0c, armc, eva0, evarm, d))
|
||
do_reps(base_reps)
|
||
# escalation: if this judge's decisive split is <=4:2 (winner has <=4 of 6 decisive), add reps
|
||
dec = [v for v in jvotes if v.winner in ("a0", "arm")]
|
||
arm_w = sum(1 for v in dec if v.winner == "arm")
|
||
a0_w = sum(1 for v in dec if v.winner == "a0")
|
||
if not budget_truncated and max(arm_w, a0_w) <= 4: # split <=4:2 (or ties dominate) -> escalate
|
||
do_reps(esc_reps)
|
||
votes.extend(jvotes)
|
||
return _aggregate(trap, votes, parse_fail, judges, budget_truncated)
|
||
|
||
|
||
def _acc(verdicts):
|
||
return sum(SCORE[v] for v in verdicts) / len(verdicts) if verdicts else None
|
||
|
||
|
||
def _aggregate(trap, votes, parse_fail, judges, budget_truncated=False):
|
||
a0_acc = _acc([v.a0_verdict for v in votes])
|
||
arm_acc = _acc([v.arm_verdict for v in votes])
|
||
arm_w = sum(1 for v in votes if v.winner == "arm")
|
||
a0_w = sum(1 for v in votes if v.winner == "a0")
|
||
tie = sum(1 for v in votes if v.winner == "tie")
|
||
# leave-one-judge-out on the paired accuracy diff
|
||
loo = {}
|
||
for jx in judges:
|
||
sub = [v for v in votes if v.judge != jx]
|
||
if sub:
|
||
loo[f"without_{jx}"] = round((_acc([v.arm_verdict for v in sub]) or 0)
|
||
- (_acc([v.a0_verdict for v in sub]) or 0), 3)
|
||
# order-bias check: winner distribution per order
|
||
order_bias = {}
|
||
for o in ("AB", "BA"):
|
||
ov = [v for v in votes if v.order == o]
|
||
order_bias[o] = {"arm": sum(1 for v in ov if v.winner == "arm"),
|
||
"a0": sum(1 for v in ov if v.winner == "a0"),
|
||
"tie": sum(1 for v in ov if v.winner == "tie")}
|
||
return {
|
||
"trap": trap["id"], "type": trap["type"],
|
||
"n_votes": len(votes), "parse_fail": parse_fail, "budget_truncated": budget_truncated,
|
||
"a0_accuracy": round(a0_acc, 3) if a0_acc is not None else None,
|
||
"arm_accuracy": round(arm_acc, 3) if arm_acc is not None else None,
|
||
"paired_diff_arm_minus_a0": round((arm_acc or 0) - (a0_acc or 0), 3) if a0_acc is not None else None,
|
||
"pairwise": {"arm_wins": arm_w, "a0_wins": a0_w, "tie": tie},
|
||
"catastrophe": {"a0": sum(1 for v in votes if v.a0_cat), "arm": sum(1 for v in votes if v.arm_cat)},
|
||
"leave_one_judge_out": loo,
|
||
"order_bias": order_bias,
|
||
"per_judge": {j: {"arm_acc": round(_acc([v.arm_verdict for v in votes if v.judge == j]) or 0, 3),
|
||
"a0_acc": round(_acc([v.a0_verdict for v in votes if v.judge == j]) or 0, 3),
|
||
"n": sum(1 for v in votes if v.judge == j)} for j in judges},
|
||
# per-vote raw verdicts (a0/arm-normalised, order recorded) — persisted so catastrophe/LOO/order
|
||
# audits run on the actual votes, not just the aggregate counts (orchestrator D30.10 fix).
|
||
"votes": [{"judge": v.judge, "order": v.order, "a0": v.a0_verdict, "arm": v.arm_verdict,
|
||
"winner": v.winner, "a0_cat": v.a0_cat, "arm_cat": v.arm_cat,
|
||
"ev_a0": v.ev_a0, "ev_arm": v.ev_arm} for v in votes],
|
||
}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
print("judge harness module — import judge_cell(). Run selftest via judge_selftest.py.")
|