120 lines
6.1 KiB
Python
120 lines
6.1 KiB
Python
#!/usr/bin/env python3
|
||
"""exp14 Ступень II — ранжирующая панель финалистов (P0/A-2pass/F-disc) на главах 19/17/18.
|
||
РАЗДЕЛЬНО стиль/верность, span-цитаты, ≥N повторов со свапом меток, leave-one-judge-out.
|
||
Судьи gpt-5-mini + kimi + gemini(точечно). Self-family (F-disc=openai) снимается leave-one-out.
|
||
Guard мемо eval-aggregation: НЕ собирать ложную сходимость.
|
||
|
||
Использование: exp14_rank.py [--repeats 3] [--dry]
|
||
"""
|
||
from __future__ import annotations
|
||
import argparse, json, random, re, sys
|
||
from pathlib import Path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
import exp14_common as C
|
||
|
||
ARMS = C.DIAG / "arms"
|
||
MAT = json.load(open(C.DIAG / "material.json"))
|
||
FINALISTS = {"Fin-A": "P0", "Fin-B": "A-2pass", "Fin-C": "F-disc"}
|
||
CHAPTERS = [19, 17, 18]
|
||
AXES = {"style": "русская художественность/абзацы/конвенции (естественность прозы, красная строка, тире-диалог)",
|
||
"fidelity": "верность смыслу (нет инверсий полярности/участника, нет потери атомов; можно ценой лёгкой шероховатости)"}
|
||
JUDGES = ["gpt-5-mini", "kimi-k2.6", "gemini-3.1-pro-preview"]
|
||
CAPS = {"gpt-5-mini": 0.10, "kimi-k2.6": 0.15, "gemini-3.1-pro-preview": 0.30}
|
||
|
||
|
||
def chapter_text(arm, ch):
|
||
chunks = sorted(u["chunk_idx"] for u in MAT["stage2"][str(ch)])
|
||
return "\n\n".join((ARMS / arm / f"{ch}.{ck}.txt").read_text(encoding="utf-8") for ck in chunks)
|
||
|
||
|
||
def run():
|
||
ap = argparse.ArgumentParser(); ap.add_argument("--repeats", type=int, default=3)
|
||
ap.add_argument("--dry", action="store_true"); a = ap.parse_args()
|
||
sp = C.Spender(C.DIAG / "rank_costs.jsonl", CAPS)
|
||
vpath = C.DIAG / "rank_verdicts.jsonl"; out = []; done = set()
|
||
if vpath.exists():
|
||
for l in vpath.read_text(encoding="utf-8").splitlines():
|
||
if l.strip():
|
||
v = json.loads(l); out.append(v); done.add((v["ch"], v["axis"], v["judge"], v["rep"]))
|
||
tasks = []
|
||
for ch in CHAPTERS:
|
||
for axis in AXES:
|
||
for rep in range(a.repeats):
|
||
order = list(FINALISTS); random.Random(f"exp14-rank-{ch}-{axis}-{rep}").shuffle(order)
|
||
labels = {f"V{i+1}": order[i] for i in range(len(order))}
|
||
for jm in JUDGES:
|
||
if jm == "gemini-3.1-pro-preview" and rep > 0:
|
||
continue # gemini точечно: 1 повтор (бюджет €2.6)
|
||
if (ch, axis, jm, rep) in done:
|
||
continue
|
||
tasks.append((ch, axis, rep, jm, labels))
|
||
print(f"rank tasks: {len(tasks)} new ({len(done)} done)")
|
||
if a.dry:
|
||
from collections import Counter
|
||
print(Counter(t[3] for t in tasks)); return
|
||
for ch, axis, rep, jm, labels in tasks:
|
||
blocks = "\n\n".join(f"=== {lab} ===\n{chapter_text(FINALISTS[fin], ch)}" for lab, fin in labels.items())
|
||
sysmsg = ("Ты — литературный судья zh→ru. Оцени ТРИ перевода одной главы по ОДНОЙ оси. "
|
||
"Ранжируй от лучшего к худшему. Приведи 2-4 span-цитаты-обоснования (≤15 слов каждая). "
|
||
'Ответь СТРОГО JSON без markdown: {"ranking":["V?","V?","V?"],'
|
||
'"evidence":[{"label":"V?","quote":"≤15 слов","note":"почему"}]}.')
|
||
user = f"ОСЬ ОЦЕНКИ: {AXES[axis]}\n\nТРИ ВАРИАНТА:\n\n{blocks}\n\nРанжируй по оси. JSON."
|
||
s = C.spec(jm, max_tokens=(16000 if jm == "kimi-k2.6" else 6000))
|
||
in_est = C.count_tokens(sysmsg + user, "glm")
|
||
ok, pred = sp.guard(jm, in_est, s["max_tokens"])
|
||
if not ok:
|
||
print(f" OVER-CAP {ch} {axis} {jm} pred=${pred:.3f}"); continue
|
||
text, err, usage = C.call(s, [{"role": "system", "content": sysmsg},
|
||
{"role": "user", "content": user}], timeout=360)
|
||
cst = sp.record({"model": jm, "judge": jm, "keyenv": s["keyenv"], "ch": ch,
|
||
"axis": axis, "rep": rep, "err": err, "usage": usage})
|
||
rk = parse_rank(text) if not err else None
|
||
v = {"ch": ch, "axis": axis, "rep": rep, "judge": jm, "labels": labels,
|
||
"ranking_labels": rk, "err": err}
|
||
# map labels→finalist
|
||
if rk:
|
||
v["ranking_fin"] = [labels.get(x) for x in rk]
|
||
out.append(v)
|
||
with open(vpath, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(v, ensure_ascii=False) + "\n")
|
||
print(f" ch{ch} {axis:<8} rep{rep} {jm:<18} → {v.get('ranking_fin')} ${cst:.4f}")
|
||
summarize(out)
|
||
|
||
|
||
def parse_rank(text):
|
||
m = re.search(r'"ranking"\s*:\s*(\[[^\]]*\])', text)
|
||
if not m:
|
||
return None
|
||
try:
|
||
arr = json.loads(m.group(1))
|
||
return [str(x) for x in arr]
|
||
except Exception:
|
||
return re.findall(r'V\d', m.group(1))
|
||
|
||
|
||
def summarize(out):
|
||
from collections import defaultdict
|
||
import statistics
|
||
# Borda: 1st=2pts,2nd=1,3rd=0; per axis; per judge for leave-one-out
|
||
def borda(rows):
|
||
pts = defaultdict(list)
|
||
for v in rows:
|
||
rf = v.get("ranking_fin")
|
||
if not rf or len(rf) != 3 or None in rf:
|
||
continue
|
||
for i, fin in enumerate(rf):
|
||
pts[fin].append(2 - i)
|
||
return {f: round(statistics.mean(p), 2) for f, p in pts.items()}
|
||
print("\n=== РАНЖИРОВАНИЕ (Borda mean, 2=лучший) ===")
|
||
for axis in AXES:
|
||
rows = [v for v in out if v["axis"] == axis]
|
||
print(f"\n{axis}: overall {borda(rows)}")
|
||
judges = sorted(set(v["judge"] for v in rows))
|
||
for j in judges:
|
||
print(f" only {j}: {borda([v for v in rows if v['judge']==j])}")
|
||
for j in judges: # leave-one-out
|
||
print(f" leave-out {j}: {borda([v for v in rows if v['judge']!=j])}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run()
|