91 lines
4.3 KiB
Python
91 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
||
"""WS3 (г) verification — re-run the frozen miner-v1 (A3 = V-C) on the pinned exp16
|
||
inputs and confirm it reproduces the golden fixtures that pin the FUTURE Go port.
|
||
|
||
Does NOT overwrite the golden (writes only to design11). Asserts:
|
||
- A3 catastrophe ranks {方源:0, 蛊:1, 蛊师:2, 古月:21} on the full slice (PASS <50);
|
||
- A3 recall@proposed = 0.965 (full) / 0.932 (test-half);
|
||
- A3 candidate count = 13618 (full).
|
||
Compares value-by-value against the stored golden JSON (the byte-parity harness in
|
||
mem_select-reverse: Python = reference, Go port must reproduce).
|
||
|
||
$0: deterministic, no LLM, no network. Reuses eval/exp16 miner-v1 (frozen SHAs §1.10).
|
||
Writes /home/ubuntu/books/gu-zhenren/design11/ws3_miner_verify.json (OUT of git).
|
||
"""
|
||
import json, os, sys
|
||
|
||
EVAL = "/home/ubuntu/projects/textmachine/eval"
|
||
# exp15 first, then exp16 at position 0 so exp16/arms.py wins over exp15/arms.py
|
||
# (name collision the exp16_common self-review flagged; it uses sys.path.append for exp15/mem_select).
|
||
sys.path.insert(0, f"{EVAL}/exp15")
|
||
sys.path.insert(0, f"{EVAL}/exp16")
|
||
|
||
import exp16_common as X
|
||
import arms as A
|
||
|
||
GOLD_DIR = "/home/ubuntu/books/gu-zhenren/exp16"
|
||
OUT = "/home/ubuntu/books/gu-zhenren/design11/ws3_miner_verify.json"
|
||
CAT_EXPECT = {"方源": 0, "蛊": 1, "蛊师": 2, "古月": 21}
|
||
|
||
def rank_of(ranked, srcnorm):
|
||
for i, c in enumerate(ranked):
|
||
if c.src == srcnorm:
|
||
return i
|
||
return None
|
||
|
||
def main():
|
||
chunks = X.load_chunks()
|
||
gt = X.load_gt()
|
||
contrast = X.Contrast()
|
||
cfg = A.FROZEN
|
||
|
||
arms = A.Arms(chunks, contrast, cfg, use_spread=True)
|
||
a3_full = arms.arm_A3(cfg["lam"])
|
||
|
||
# catastrophe ranks on full A3
|
||
cat_ranks = {}
|
||
for term in X.CATASTROPHE:
|
||
cat_ranks[term] = rank_of(a3_full, X.norm(term))
|
||
cat_pass = all(r is not None and r < 50 for r in cat_ranks.values())
|
||
|
||
# recall@proposed full
|
||
rec_full, _ = A.recall_by(a3_full, gt, chunks, top_k=None, include_annotation=True)
|
||
overall_full = rec_full["overall"][0]
|
||
|
||
# test half (ch16-25) — GT filtered to entities occurring in test chapters (run_arms phase_test)
|
||
test_chunks = [c for c in chunks if c.chapter in X.TEST_CH]
|
||
gt_t = A.gt_in_chapters(gt, chunks, X.TEST_CH)
|
||
arms_t = A.Arms(test_chunks, contrast, cfg, use_spread=True)
|
||
a3_test = arms_t.arm_A3(cfg["lam"])
|
||
rec_test, _ = A.recall_by(a3_test, gt_t, test_chunks, top_k=None, include_annotation=True)
|
||
overall_test = rec_test["overall"][0]
|
||
|
||
# load stored golden for value-by-value compare
|
||
gold_full = json.load(open(f"{GOLD_DIR}/full_slice_result.json"))
|
||
gold_test = json.load(open(f"{GOLD_DIR}/test_half_result.json"))
|
||
gold_a3_full = gold_full.get("A3", {})
|
||
gold_a3_test = gold_test.get("A3", {})
|
||
n_cands_full = len(a3_full)
|
||
|
||
result = {
|
||
"catastrophe_ranks_A3_full": cat_ranks,
|
||
"catastrophe_expected": CAT_EXPECT,
|
||
"catastrophe_match": cat_ranks == CAT_EXPECT,
|
||
"catastrophe_screen_pass": cat_pass,
|
||
"recall_at_proposed_A3_full": round(overall_full, 4) if overall_full is not None else None,
|
||
"recall_at_proposed_A3_test": round(overall_test, 4) if overall_test is not None else None,
|
||
"n_candidates_A3_full": n_cands_full,
|
||
"golden_full_overall": gold_a3_full.get("recall_proposed", {}).get("overall") if isinstance(gold_a3_full.get("recall_proposed"), dict) else gold_a3_full.get("recall_proposed"),
|
||
"golden_test_overall": gold_a3_test.get("recall_proposed", {}).get("overall") if isinstance(gold_a3_test.get("recall_proposed"), dict) else gold_a3_test.get("recall_proposed"),
|
||
"top20_A3_full": [{"src": c.src, "score": round(c.score, 2), "freq": c.freq,
|
||
"types": c.types, "from_pattern": c.from_pattern} for c in a3_full[:20]],
|
||
}
|
||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||
json.dump(result, open(OUT, "w"), ensure_ascii=False, indent=2)
|
||
print(json.dumps({k: v for k, v in result.items() if k != "top20_A3_full"}, ensure_ascii=False, indent=2))
|
||
print("\ntop-5 A3 full:", [(c.src, round(c.score,1), c.freq) for c in a3_full[:5]])
|
||
print(f"\nVERDICT: catastrophe {'MATCH' if result['catastrophe_match'] else 'MISMATCH'} {cat_ranks}; "
|
||
f"n_cands={n_cands_full} (golden 13618); recall_full={result['recall_at_proposed_A3_full']} (golden 0.965).")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|