#!/usr/bin/env python3
"""exp13 — агрегация судейских вердиктов + гейтов + стоимости в сводку (§2). $0.
Средняя позиция (1=лучшая) по осям и суммарно; разброс (leak-устойчивость); согласие судей;
эхо-rate/объём из gates.json; суммарная стоимость из costs.jsonl + judge_costs.jsonl.
Запуск: eval/.venv/bin/python eval/exp13_aggregate.py
Выход: печать + diag/arms13/summary.json
"""
from __future__ import annotations
import json, statistics as st
from collections import defaultdict
from pathlib import Path

ARMS = Path("/home/ubuntu/books/gu-zhenren/diag/arms13")
LEGEND = {"T1": "deepseek-v4-flash", "T2": "mistral-large-2512", "T3": "glm-5",
          "T4": "deepseek-v4-pro", "T5": "grok-4.3 reasoning-ON", "T6": "qwen-flash"}


def load_jsonl(p):
    if not p.exists():
        return []
    return [json.loads(l) for l in p.read_text(encoding="utf-8").splitlines() if l.strip()]


def positions(verdicts, axis=None, judge=None):
    """arm -> list of positions (1=best)."""
    pos = defaultdict(list)
    for v in verdicts:
        if axis and v["axis"] != axis:
            continue
        if judge and v["judge"] != judge:
            continue
        r = v["arm_ranking"]
        for i, arm in enumerate(r):
            pos[arm].append(i + 1)
    return pos


def summarize(pos):
    out = {}
    for arm, ps in pos.items():
        out[arm] = {"mean": round(st.mean(ps), 3), "n": len(ps),
                    "std": round(st.pstdev(ps), 3) if len(ps) > 1 else 0.0}
    return out


def table(title, pos):
    print(f"\n== {title} ==")
    rows = sorted(summarize(pos).items(), key=lambda kv: kv[1]["mean"])
    print(f"{'arm':<5} {'model':<24} {'mean_pos':>8} {'std':>6} {'n':>4}")
    for arm, s in rows:
        print(f"{arm:<5} {LEGEND.get(arm, arm):<24} {s['mean']:>8} {s['std']:>6} {s['n']:>4}")
    return {arm: s for arm, s in rows}


def main():
    verdicts = load_jsonl(ARMS / "judges" / "verdicts.jsonl")
    print(f"валидных вердиктов: {len(verdicts)}")
    judges = sorted({v["judge"] for v in verdicts})
    axes = sorted({v["axis"] for v in verdicts})

    summary = {"n_verdicts": len(verdicts), "judges": judges, "axes": axes}
    summary["overall"] = table("ОБЩЕЕ (обе оси, все судьи)", positions(verdicts))
    for axis in axes:
        summary[f"axis_{axis}"] = table(f"ось: {axis}", positions(verdicts, axis=axis))
    # согласие судей (по каждой оси, средняя позиция у каждого судьи)
    summary["per_judge"] = {}
    for judge in judges:
        summary["per_judge"][judge] = {}
        for axis in axes:
            summary["per_judge"][judge][axis] = summarize(positions(verdicts, axis=axis, judge=judge))

    # гейты
    gates = {}
    gp = ARMS / "gates.json"
    if gp.exists():
        g = json.loads(gp.read_text(encoding="utf-8"))["per_arm"]
        print("\n== ГЕЙТЫ ==")
        print(f"{'arm':<5} {'echo_rate':>9} {'refusal':>7} {'volBad':>6} {'empty':>5} {'han_avg':>8} {'vol_avg':>7}")
        for arm in sorted(g):
            a = g[arm]
            print(f"{arm:<5} {a['echo_rate']:>9} {a['refusal']:>7} {a['vol_bad']:>6} {a['empty']:>5} {a['han_avg']:>8} {a['vol_avg']:>7}")
            gates[arm] = {k: a[k] for k in ("echo_rate", "refusal", "vol_bad", "empty", "han_avg", "vol_avg")}
    summary["gates"] = gates

    # стоимость
    arm_costs = load_jsonl(ARMS / "costs.jsonl")
    judge_costs = load_jsonl(ARMS / "judge_costs.jsonl")
    per_arm_cost = defaultdict(float)
    for r in arm_costs:
        per_arm_cost[r["arm"]] += r.get("cost", 0) or 0
    arm_total = sum(per_arm_cost.values())
    judge_total = sum((r.get("cost", 0) or 0) for r in judge_costs)
    print("\n== СТОИМОСТЬ ==")
    print(f"{'arm':<5} {'model':<24} {'$/6chunks':>10} {'$/chunk':>9}")
    for arm in sorted(per_arm_cost):
        c = per_arm_cost[arm]
        print(f"{arm:<5} {LEGEND.get(arm, arm):<24} {c:>10.4f} {c/6:>9.5f}")
    print(f"\nармы: ${arm_total:.4f}  судьи: ${judge_total:.4f}  ИТОГО: ${arm_total+judge_total:.4f} (кап $5)")
    summary["cost"] = {"per_arm": {k: round(v, 5) for k, v in per_arm_cost.items()},
                       "arm_total": round(arm_total, 4), "judge_total": round(judge_total, 4),
                       "grand_total": round(arm_total + judge_total, 4)}

    (ARMS / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding="utf-8")
    print("\nsummary → diag/arms13/summary.json")


if __name__ == "__main__":
    main()