91 lines
3.9 KiB
Python
91 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""exp15 — judge-harness SELFTEST on 2-3 pilot traps (checkpoint item d + preliminary FLOOR).
|
|
|
|
Judges A0 vs A0' (SAME chunking = the noise floor: a well-behaved harness should return ~tie, low
|
|
|paired_diff|, no systematic order bias). Exercises: JSON parse-rate, both-orders, the 6->10 split
|
|
escalation branch, leave-one-judge-out, catastrophe screen. Cheap (cents).
|
|
|
|
Pilot traps are hand-specified cohesion phenomena located in specific chunks; each is judged on that
|
|
chunk's A0-final vs A0'-final. Run AFTER anchors_analyze.py (needs *.chunks.json).
|
|
Run: python judge_selftest.py
|
|
"""
|
|
from __future__ import annotations
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import exp15_llm as L
|
|
from judges import judge_cell
|
|
|
|
AH = Path("/home/ubuntu/books/gu-zhenren/exp15/anchors")
|
|
TRAP_LEDGER = Path("/home/ubuntu/books/gu-zhenren/exp15/trap_ledger.json")
|
|
LEDGER = Path("/home/ubuntu/books/gu-zhenren/exp15/judge_selftest_costs.jsonl")
|
|
OUT = Path("/home/ubuntu/books/gu-zhenren/exp15/judge_selftest.json")
|
|
A0_TAG = "flat_a0" # blind-greedy anchor (regenerated on flat S2')
|
|
A0P_TAG = "flat_a0prime"
|
|
|
|
|
|
def chunks(tag):
|
|
return {c["chunk"]: c for c in json.load(open(AH / f"{tag}.chunks.json"))}
|
|
|
|
|
|
def locate_chunk(chunk_map, src_probe):
|
|
"""Return the chunk_idx whose source contains src_probe (first ~20 chars)."""
|
|
probe = src_probe.strip()[:20]
|
|
for idx, c in chunk_map.items():
|
|
if c["source"] and probe in c["source"]:
|
|
return idx
|
|
return None
|
|
|
|
|
|
def pick_pilots(n=3):
|
|
"""2-3 diverse-type traps from the curated ledger (prefer distinct types crossing an A0 cut)."""
|
|
led = json.load(open(TRAP_LEDGER))["ledger"]
|
|
cross = [t for t in led if t.get("crosses_A0_cut")]
|
|
picks, seen_types = [], set()
|
|
for t in cross:
|
|
if t["type"] not in seen_types:
|
|
picks.append(t); seen_types.add(t["type"])
|
|
if len(picks) >= n:
|
|
break
|
|
return picks or led[:n]
|
|
|
|
|
|
def main():
|
|
a0 = chunks(A0_TAG)
|
|
a0p = chunks(A0P_TAG)
|
|
pilots = [{"id": t["id"], "type": t["type"], "chunk_idx": locate_chunk(a0, t["anchor_span"]),
|
|
"src_zone": (t["anchor_span"] + " … " + t["dependent_span"]),
|
|
"phenomenon": t["phenomenon"], "expected": t["expected_ru"]}
|
|
for t in pick_pilots(3)]
|
|
pilots = [p for p in pilots if p["chunk_idx"] is not None]
|
|
sp = L.Spender(str(LEDGER), per_call_cap=0.05, hard_cap=1.0)
|
|
results = []
|
|
for p in pilots:
|
|
ck = p["chunk_idx"]
|
|
a0_final = a0[ck]["final"]
|
|
armp_final = a0p[ck]["final"]
|
|
trap = {"id": p["id"], "type": p["type"], "src_zone": p["src_zone"],
|
|
"phenomenon": p["phenomenon"], "expected": p["expected"]}
|
|
print(f"\n=== pilot {p['id']} ({p['type']}) chunk {ck} — A0 vs A0' (floor) ===")
|
|
agg = judge_cell(sp, trap, a0_final, armp_final)
|
|
results.append(agg)
|
|
print(json.dumps({k: agg[k] for k in ("n_votes", "parse_fail", "a0_accuracy", "arm_accuracy",
|
|
"paired_diff_arm_minus_a0", "pairwise", "order_bias", "catastrophe")}, ensure_ascii=False, indent=2))
|
|
OUT.write_text(json.dumps({"pilots": results, "total_cost": round(sp.total, 5)}, ensure_ascii=False, indent=2),
|
|
encoding="utf-8")
|
|
# selftest checks
|
|
tot_parse_fail = sum(r["parse_fail"] for r in results)
|
|
tot_votes = sum(r["n_votes"] for r in results)
|
|
escalated = any(r["n_votes"] > 2 * 6 for r in results) # >12 -> at least one judge escalated
|
|
print(f"\n=== SELFTEST SUMMARY ===")
|
|
print(f"total votes {tot_votes} parse_fail {tot_parse_fail} (want 0) spend ${sp.total:.4f}")
|
|
print(f"escalation branch exercised (some cell >12 votes): {escalated}")
|
|
print(f"floor |paired_diff| per pilot: {[abs(r['paired_diff_arm_minus_a0'] or 0) for r in results]} "
|
|
f"(want small — A0 vs A0' is noise)")
|
|
print(f"-> {OUT}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|