108 lines
4.6 KiB
Python
108 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
||
"""exp13 — агрегация судейских вердиктов + гейтов + стоимости в сводку (§2). $0.
|
||
Средняя позиция (1=лучшая) по осям и суммарно; разброс (leak-устойчивость); согласие судей;
|
||
эхо-rate/объём из gates.json; суммарная стоимость из costs.jsonl + judge_costs.jsonl.
|
||
Запуск: eval/.venv/bin/python eval/exp13_aggregate.py
|
||
Выход: печать + diag/arms13/summary.json
|
||
"""
|
||
from __future__ import annotations
|
||
import json, statistics as st
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
ARMS = Path("/home/ubuntu/books/gu-zhenren/diag/arms13")
|
||
LEGEND = {"T1": "deepseek-v4-flash", "T2": "mistral-large-2512", "T3": "glm-5",
|
||
"T4": "deepseek-v4-pro", "T5": "grok-4.3 reasoning-ON", "T6": "qwen-flash"}
|
||
|
||
|
||
def load_jsonl(p):
|
||
if not p.exists():
|
||
return []
|
||
return [json.loads(l) for l in p.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||
|
||
|
||
def positions(verdicts, axis=None, judge=None):
|
||
"""arm -> list of positions (1=best)."""
|
||
pos = defaultdict(list)
|
||
for v in verdicts:
|
||
if axis and v["axis"] != axis:
|
||
continue
|
||
if judge and v["judge"] != judge:
|
||
continue
|
||
r = v["arm_ranking"]
|
||
for i, arm in enumerate(r):
|
||
pos[arm].append(i + 1)
|
||
return pos
|
||
|
||
|
||
def summarize(pos):
|
||
out = {}
|
||
for arm, ps in pos.items():
|
||
out[arm] = {"mean": round(st.mean(ps), 3), "n": len(ps),
|
||
"std": round(st.pstdev(ps), 3) if len(ps) > 1 else 0.0}
|
||
return out
|
||
|
||
|
||
def table(title, pos):
|
||
print(f"\n== {title} ==")
|
||
rows = sorted(summarize(pos).items(), key=lambda kv: kv[1]["mean"])
|
||
print(f"{'arm':<5} {'model':<24} {'mean_pos':>8} {'std':>6} {'n':>4}")
|
||
for arm, s in rows:
|
||
print(f"{arm:<5} {LEGEND.get(arm, arm):<24} {s['mean']:>8} {s['std']:>6} {s['n']:>4}")
|
||
return {arm: s for arm, s in rows}
|
||
|
||
|
||
def main():
|
||
verdicts = load_jsonl(ARMS / "judges" / "verdicts.jsonl")
|
||
print(f"валидных вердиктов: {len(verdicts)}")
|
||
judges = sorted({v["judge"] for v in verdicts})
|
||
axes = sorted({v["axis"] for v in verdicts})
|
||
|
||
summary = {"n_verdicts": len(verdicts), "judges": judges, "axes": axes}
|
||
summary["overall"] = table("ОБЩЕЕ (обе оси, все судьи)", positions(verdicts))
|
||
for axis in axes:
|
||
summary[f"axis_{axis}"] = table(f"ось: {axis}", positions(verdicts, axis=axis))
|
||
# согласие судей (по каждой оси, средняя позиция у каждого судьи)
|
||
summary["per_judge"] = {}
|
||
for judge in judges:
|
||
summary["per_judge"][judge] = {}
|
||
for axis in axes:
|
||
summary["per_judge"][judge][axis] = summarize(positions(verdicts, axis=axis, judge=judge))
|
||
|
||
# гейты
|
||
gates = {}
|
||
gp = ARMS / "gates.json"
|
||
if gp.exists():
|
||
g = json.loads(gp.read_text(encoding="utf-8"))["per_arm"]
|
||
print("\n== ГЕЙТЫ ==")
|
||
print(f"{'arm':<5} {'echo_rate':>9} {'refusal':>7} {'volBad':>6} {'empty':>5} {'han_avg':>8} {'vol_avg':>7}")
|
||
for arm in sorted(g):
|
||
a = g[arm]
|
||
print(f"{arm:<5} {a['echo_rate']:>9} {a['refusal']:>7} {a['vol_bad']:>6} {a['empty']:>5} {a['han_avg']:>8} {a['vol_avg']:>7}")
|
||
gates[arm] = {k: a[k] for k in ("echo_rate", "refusal", "vol_bad", "empty", "han_avg", "vol_avg")}
|
||
summary["gates"] = gates
|
||
|
||
# стоимость
|
||
arm_costs = load_jsonl(ARMS / "costs.jsonl")
|
||
judge_costs = load_jsonl(ARMS / "judge_costs.jsonl")
|
||
per_arm_cost = defaultdict(float)
|
||
for r in arm_costs:
|
||
per_arm_cost[r["arm"]] += r.get("cost", 0) or 0
|
||
arm_total = sum(per_arm_cost.values())
|
||
judge_total = sum((r.get("cost", 0) or 0) for r in judge_costs)
|
||
print("\n== СТОИМОСТЬ ==")
|
||
print(f"{'arm':<5} {'model':<24} {'$/6chunks':>10} {'$/chunk':>9}")
|
||
for arm in sorted(per_arm_cost):
|
||
c = per_arm_cost[arm]
|
||
print(f"{arm:<5} {LEGEND.get(arm, arm):<24} {c:>10.4f} {c/6:>9.5f}")
|
||
print(f"\nармы: ${arm_total:.4f} судьи: ${judge_total:.4f} ИТОГО: ${arm_total+judge_total:.4f} (кап $5)")
|
||
summary["cost"] = {"per_arm": {k: round(v, 5) for k, v in per_arm_cost.items()},
|
||
"arm_total": round(arm_total, 4), "judge_total": round(judge_total, 4),
|
||
"grand_total": round(arm_total + judge_total, 4)}
|
||
|
||
(ARMS / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print("\nsummary → diag/arms13/summary.json")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|