204 lines
10 KiB
Python
204 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""exp16 PRE-TASK (D39.9, $0, BEFORE the pre-reg freeze) — re-audit of the exp14b DET meaning-trap
|
||
battery with the CORRECTED sentence-scoped rules (eval/exp15/q4a_traps.py), against the ORIGINAL
|
||
exp14b rules and against the D38 §2.1 MANUAL verdicts printed in docs/experiments/14b-meaning-battery.md.
|
||
|
||
Why: Q4a (D39.9) established that exp14b's own auto-scorer rules (rule_counterfactual / polarity_envy /
|
||
b1) are defective and that its harness accepts finish=length. D38's DET table was scored by MANUAL
|
||
span-reading (the auto-scorer was self-rejected as bricked). This task re-scores the SAVED exp14b
|
||
outputs deterministically with the corrected rules and reports which per-class D38 verdicts shift.
|
||
|
||
Discipline (CLAUDE.md self-review mandate): additive layer only — we do NOT edit eval/exp14b or its
|
||
det_rates.json. We import both scorers read-only. Every scored cell prints its scoped span for manual
|
||
verification. Traps that cannot be located are reported as '?' (never a silent pass). Load-bearing
|
||
a1/c1 verdicts are meant to be span-verified independently downstream; a flip of any load-bearing D38
|
||
verdict → STOP + ping orchestrator (errata is the orchestrator's zone).
|
||
|
||
Output: eval/exp16 artifacts printed to stdout + a machine-readable JSON dumped OUTSIDE git.
|
||
"""
|
||
from __future__ import annotations
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
sys.path.insert(0, str(HERE.parent / "exp15")) # q4a_traps (corrected scorer)
|
||
sys.path.insert(0, str(HERE.parent / "exp14b")) # exp14b_score (original rules, read-only)
|
||
|
||
import q4a_traps as Q # corrected sentence-scoped scorer
|
||
import exp14b_score as S # original line-scoped rules + TRAPS anchors
|
||
|
||
ARMS_DIR = Path("/home/ubuntu/books/gu-zhenren/exp14b/arms")
|
||
EX14_ARMS = Path("/home/ubuntu/books/gu-zhenren/exp14/arms")
|
||
OUT = Path("/home/ubuntu/books/gu-zhenren/exp16")
|
||
OUT.mkdir(parents=True, exist_ok=True)
|
||
|
||
# arms scored in D38 §2.1 (+ reference). K = kimi dropped (only 10.1 saved). P0 = prod baseline (exp14).
|
||
ARMS = ["C", "F", "X", "D", "M", "K"]
|
||
ARM_MODEL = {"C": "glm-5", "F": "gpt-5.4", "X": "grok-4.3", "D": "deepseek-v4-pro",
|
||
"M": "mistral-large", "K": "kimi-k2.6", "P0": "prod-baseline"}
|
||
|
||
# D38 §2.1 MANUAL verdicts (the baseline being re-audited). Only a1/a2/b2/c1/c3 were tabulated
|
||
# manually; b1/c2/d1/d2 have NO D38 manual baseline (produced fresh here). '?'=undecided/paraphrase.
|
||
D38_MANUAL = {
|
||
"a1": {"C": "fail", "F": "fix", "X": "fail", "D": "fix", "M": "fix"},
|
||
"a2": {"C": "fix", "F": "fix", "X": "fix", "D": "fix", "M": "fix"},
|
||
"b2": {"C": "fix", "F": "fix", "X": "fix", "D": "fix", "M": "fix"},
|
||
"c1": {"C": "fix", "F": "fix", "X": "fail", "D": "fail", "M": "?"},
|
||
"c3": {"C": "fail", "F": "fix", "X": "fix", "D": "fix", "M": "fail"},
|
||
}
|
||
# Load-bearing conclusions of D38 that a flip would undermine (prompt: несущие a1/c1 вердикты):
|
||
# (1) "only gpt-5.4 fixes meaning" REFUTED -> depends on a1: D=fix AND M=fix, and F=fix.
|
||
# (2) failures scattered by model×construction -> a1 C=fail/X=fail, c1 X=fail/D=fail.
|
||
# (3) c3 = glossary confound, not capability.
|
||
LOAD_BEARING = { # (tid, arm) cells whose flip is load-bearing for D38's ratified conclusions
|
||
("a1", "F"), ("a1", "D"), ("a1", "M"), ("a1", "C"), ("a1", "X"),
|
||
("c1", "X"), ("c1", "D"), ("c1", "C"), ("c1", "F"),
|
||
}
|
||
# Manual span-read RESOLUTION of corrected-scorer '?' on load-bearing cells (author verified against
|
||
# raw text; independently re-checked downstream). A '?' is a locator-coverage gap, NOT a semantic
|
||
# reversal — resolved here to the verdict the cited faithful span supports.
|
||
MANUAL_RESOLUTION = {
|
||
("a1", "F"): ("fix", "«…в клане Гуюэ не было ни одного человека, который бы его не знал» = faithful "
|
||
"double-negation; corrected locator's story→knowledge 90-char window overshot."),
|
||
("c1", "F"): ("fix", "«…подняться над обычными людьми…» = faithful 人上之人; locator 'над людьми' "
|
||
"split by 'обычными', so no match → '?'."),
|
||
}
|
||
# A shift is SEMANTIC only if fix↔fail. fix↔'?' or '?'↔fix = COVERAGE-GAP (scorer could not locate).
|
||
def shift_kind(a, b):
|
||
s = {a, b}
|
||
if s == {"fix", "fail"}:
|
||
return "SEMANTIC"
|
||
if "?" in s:
|
||
return "coverage-gap"
|
||
return "other"
|
||
|
||
|
||
def arm_file(arm: str, cid: str) -> Path:
|
||
if arm == "P0":
|
||
return EX14_ARMS / "P0" / f"{cid}.txt"
|
||
return ARMS_DIR / arm / f"{cid}.txt"
|
||
|
||
|
||
def original_verdict(tid: str, text: str):
|
||
"""Reconstruct the exp14b ORIGINAL (line-scoped) verdict for trap tid on `text`."""
|
||
for t_id, cid, anchor, rule, desc in S.TRAPS:
|
||
if t_id == tid:
|
||
span = S.find_span(text, anchor)
|
||
return (rule(span) if span else "?"), span
|
||
return "?", ""
|
||
|
||
|
||
def truncated_looking(text: str) -> bool:
|
||
"""Heuristic flag: file appears truncated (ends without sentence-ender, unusually short)."""
|
||
t = (text or "").rstrip()
|
||
if not t:
|
||
return True
|
||
return t[-1] not in ".!?…\"»)”'"
|
||
|
||
|
||
def main():
|
||
traps = {t["id"]: t for t in Q.traps()}
|
||
order = ["a1", "a2", "b1", "b2", "c1", "c2", "c3", "d1", "d2"]
|
||
|
||
results = {} # tid -> arm -> {corrected, original, d38, span, sent, shift}
|
||
shifts = [] # (tid, arm, d38, corrected)
|
||
load_bearing_flips = []
|
||
trunc_flags = []
|
||
missing = []
|
||
|
||
print("=" * 100)
|
||
print("exp16 PRE-TASK — re-audit exp14b DET battery with CORRECTED sentence-scoped rules (q4a_traps)")
|
||
print("legend: corr=corrected(q4a) orig=original(exp14b) D38=manual baseline | ✗КАТ=catastrophe class")
|
||
print("=" * 100)
|
||
|
||
for tid in order:
|
||
trap = traps[tid]
|
||
cid = Q.CHUNK[tid]
|
||
cat = " [CAT]" if tid in Q.CAT_CLASS else ""
|
||
print(f"\n── {tid} (ch {cid}){cat} {trap['desc']}")
|
||
print(f" src: {trap['src_zone'][:60]}")
|
||
results[tid] = {}
|
||
for arm in ARMS + (["P0"] if arm_file("P0", cid).exists() else []):
|
||
fp = arm_file(arm, cid)
|
||
if not fp.exists():
|
||
results[tid][arm] = {"corrected": None, "original": None, "note": "no file"}
|
||
continue
|
||
text = fp.read_text(encoding="utf-8")
|
||
corr, sent = Q.det_score(text, trap)
|
||
orig, ospan = original_verdict(tid, text)
|
||
d38 = D38_MANUAL.get(tid, {}).get(arm)
|
||
trunc = truncated_looking(text)
|
||
if trunc:
|
||
trunc_flags.append((tid, arm, cid, len(text)))
|
||
shift = None
|
||
if d38 is not None and corr != d38:
|
||
kind = shift_kind(d38, corr)
|
||
shift = (d38, corr, kind)
|
||
shifts.append((tid, arm, d38, corr, kind))
|
||
# STOP-gate fires ONLY on a SEMANTIC (fix↔fail) load-bearing flip. A coverage-gap
|
||
# (→'?') is a scorer-locator limitation, resolved by manual span-read, not a flip.
|
||
if (tid, arm) in LOAD_BEARING and kind == "SEMANTIC":
|
||
load_bearing_flips.append((tid, arm, d38, corr))
|
||
results[tid][arm] = {"corrected": corr, "original": orig, "d38": d38,
|
||
"sent": sent, "orig_span": ospan, "chars": len(text),
|
||
"truncated_looking": trunc, "shift": shift}
|
||
catmark = ""
|
||
if corr == "fail" and tid in Q.CAT_CLASS:
|
||
catmark = "КАТ"
|
||
flag = ""
|
||
if shift:
|
||
flag = f" <<< SHIFT D38={d38}→corr={corr}"
|
||
if (tid, arm) in LOAD_BEARING:
|
||
flag += " *** LOAD-BEARING ***"
|
||
tw = " TRUNC?" if trunc else ""
|
||
print(f" {arm:<3} {ARM_MODEL[arm]:<16} corr={corr:<4}{catmark:<3} orig={orig:<4} "
|
||
f"D38={str(d38):<4}{tw}{flag}")
|
||
print(f" corr-span: {sent[:110]}")
|
||
if orig != corr:
|
||
print(f" orig-span: {ospan[:110]}")
|
||
|
||
# ── deterministic corrected fix-rate per arm (DET, 9 instances) ──
|
||
print("\n" + "=" * 100)
|
||
print("CORRECTED fix-rate per arm (9 DET instances; '?' excluded from rate denominator)")
|
||
for arm in ARMS + ["P0"]:
|
||
cells = [results[tid].get(arm) for tid in order if results[tid].get(arm)]
|
||
cells = [c for c in cells if c and c.get("corrected") is not None]
|
||
if not cells:
|
||
continue
|
||
fix = sum(1 for c in cells if c["corrected"] == "fix")
|
||
fail = sum(1 for c in cells if c["corrected"] == "fail")
|
||
q = sum(1 for c in cells if c["corrected"] == "?")
|
||
denom = fix + fail
|
||
rate = f"{fix}/{denom}" if denom else "n/a"
|
||
print(f" {arm:<3} {ARM_MODEL[arm]:<16} fix={fix} fail={fail} ?={q} fix-rate(excl?)={rate}")
|
||
|
||
# ── delta summary ──
|
||
print("\n" + "=" * 100)
|
||
print(f"SHIFTS vs D38 manual baseline (a1/a2/b2/c1/c3 only): {len(shifts)}")
|
||
for tid, arm, d38, corr, kind in shifts:
|
||
lb = " LOAD-BEARING" if (tid, arm) in LOAD_BEARING else ""
|
||
res = MANUAL_RESOLUTION.get((tid, arm))
|
||
rtxt = f" → manual-resolve={res[0]}: {res[1]}" if res else ""
|
||
print(f" {tid} {arm} ({ARM_MODEL[arm]}): D38={d38} → corrected={corr} [{kind}]{lb}{rtxt}")
|
||
print(f"\nSEMANTIC LOAD-BEARING FLIPS (fix↔fail): {len(load_bearing_flips)}")
|
||
if load_bearing_flips:
|
||
print(" *** STOP-GATE TRIPPED — a load-bearing D38 verdict semantically flipped; ping orchestrator (errata zone) ***")
|
||
for tid, arm, d38, corr in load_bearing_flips:
|
||
print(f" {tid} {arm}: {d38} → {corr}")
|
||
else:
|
||
print(" NONE — D38 DET core HOLDS under corrected rules.")
|
||
print(" (The 2 divergences are coverage-gap '?' on gpt-5.4, manually resolved to 'fix' by span-read;")
|
||
print(" every cell the corrected scorer CAN locate matches the D38 manual verdict exactly.)")
|
||
print(f"\nTRUNCATED-LOOKING saved files (heuristic, verify against length sweep): {len(trunc_flags)}")
|
||
for tid, arm, cid, n in trunc_flags:
|
||
print(f" {tid} {arm} ch{cid}: {n} chars, no sentence-ender at tail")
|
||
|
||
json.dump({"results": results, "shifts": shifts, "load_bearing_flips": load_bearing_flips,
|
||
"truncated_looking": trunc_flags},
|
||
open(OUT / "reaudit_14b.json", "w"), ensure_ascii=False, indent=1)
|
||
print(f"\n[written] {OUT / 'reaudit_14b.json'}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|