textmachine/eval/exp15/trap_curate.py

#!/usr/bin/env python3
"""exp15 — trap curation -> ledger (design §D1). Takes mined candidates + A0 greedy boundaries and:
  - locates each candidate's anchor & dependent spans in S2' (exact .find; drop if not locatable)
  - COMMON DENOMINATOR: keep candidates whose dependency crosses an A0 greedy cut (A0 might fail them);
    record whether it ALSO crosses a scene-shift seam (covariate, not filter)
  - stratify by antecedent distance: <=3 sentences (carryover zone) vs >3 (window/Ф2 zone)
  - dedupe near-identical spans; assign type; report quota status (T-ana/T-ent/T-tense >=20, others >=10)
  - emit the trap ledger with per-trap EXPECTED BEHAVIOR (recorded BEFORE any arm/judge spend)

Output is a REVIEW ARTIFACT for the checkpoint (manual curation follows: the polygon signs off each trap).
Run: python trap_curate.py
"""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
from chunker import split_source_sentences, greedy_chunks

S2 = Path("/home/ubuntu/books/gu-zhenren/exp15/S2prime.txt")
CANDS = Path("/home/ubuntu/books/gu-zhenren/exp15/trap_candidates.json")
SEAMS = Path("/home/ubuntu/books/gu-zhenren/exp15/S2prime_seams.json")
ANCHORS = Path("/home/ubuntu/books/gu-zhenren/exp15/anchors/anchors_analysis.json")
LABELS = Path("/home/ubuntu/books/gu-zhenren/exp15/S2prime_seam_labels.json")
OUT = Path("/home/ubuntu/books/gu-zhenren/exp15/trap_ledger.json")

QUOTA = {"T-ana": 20, "T-ent": 20, "T-tense": 20, "T-cat": 10, "T-ell": 10, "T-gen": 10, "T-reg": 10}


def sentence_index(text):
    """Return list of (start_rune, end_rune) for each source sentence, for distance computation."""
    spans, cur = [], 0
    for s in split_source_sentences(text):
        spans.append((cur, cur + len(s)))
        cur += len(s)
    return spans


def sentence_of_offset(spans, off):
    for i, (a, b) in enumerate(spans):
        if a <= off < b:
            return i
    return len(spans) - 1


def main():
    text = S2.read_text(encoding="utf-8")
    spans = sentence_index(text)
    seams = json.load(open(SEAMS))["seams"]
    seam_offs = {s["seam_id"]: s["rune_offset"] for s in seams}
    labels = json.load(open(LABELS))["labels"]
    # A0 (flat, blind-greedy) cut offsets in S2prime.txt SPACE — locate each greedy chunk head in `text`
    # so cuts and trap spans (also located via text.find) share one coordinate system.
    gch = greedy_chunks(text)
    a0_cuts, pos = [], 0
    for i, c in enumerate(gch):
        idx = text.find(c[:25], max(0, pos - 5))
        if idx < 0:
            idx = text.find(c[:25])
        if i > 0 and idx >= 0:
            a0_cuts.append(idx)
        pos = idx + len(c) if idx >= 0 else pos + len(c)
    cands_raw = json.load(open(CANDS))["results"] if CANDS.exists() else []

    ledger = []
    dropped = {"not_located": 0, "no_a0_cross": 0, "dup": 0}
    seen_keys = set()
    for res in cands_raw:
        for c in res.get("candidates", []):
            a_span = (c.get("anchor_span") or "").strip()
            d_span = (c.get("dependent_span") or "").strip()
            if not a_span or not d_span:
                dropped["not_located"] += 1
                continue
            ai = text.find(a_span[:30])
            di = text.find(d_span[:30])
            if ai < 0 or di < 0:
                dropped["not_located"] += 1
                continue
            a_end = ai + len(a_span)
            # dependency region = [lo, hi); a cut at offset c splits it iff lo < c < hi STRICTLY (a cut
            # exactly at hi lands right AFTER the dependency -> A0 kept it intact -> NOT crossing). BUG1 fix.
            lo, hi = min(ai, di), max(a_end, di + len(d_span))
            crosses_a0 = any(lo < cut < hi for cut in a0_cuts)
            crossed_cuts = [cut for cut in a0_cuts if lo < cut < hi]
            crosses_seam = [sid for sid, off in seam_offs.items() if lo < off < hi]
            # BUG2 fix: dedup on span PROXIMITY (same type + anchor & dependent within 15 runes of a kept
            # trap), not a fixed 20-rune grid (grid mis-buckets across edges -> false keep AND false drop).
            typ = c.get("type")
            if any(t["type"] == typ and abs(t["anchor_offset"] - ai) < 15 and abs(t["dependent_offset"] - di) < 15
                   for t in ledger):
                dropped["dup"] += 1
                continue
            # antecedent distance in sentences
            si_a = sentence_of_offset(spans, ai)
            si_d = sentence_of_offset(spans, di)
            dist = abs(si_d - si_a)
            stratum = "near(<=3)" if dist <= 3 else "far(>3)"
            seam_classes = [labels[str(s)]["class"] for s in crosses_seam if str(s) in labels]
            ledger.append({
                "id": f"{c.get('type')}-{len(ledger)+1}",
                "type": c.get("type"),
                "seam_id": c.get("seam_id"),
                "anchor_span": a_span, "dependent_span": d_span,
                "anchor_offset": ai, "dependent_offset": di,
                "antecedent_distance_sentences": dist, "stratum": stratum,
                "crosses_A0_cut": crosses_a0, "crossed_A0_cut_offsets": crossed_cuts,
                "crosses_seam_ids": crosses_seam, "seam_classes": seam_classes,
                "phenomenon": c.get("phenomenon"), "expected_ru": c.get("expected_ru"),
                "failure_if_split": c.get("failure_if_split"),
                # expected behavior (pre-registered before scoring):
                "expected_behavior": {
                    "A0_at_risk": crosses_a0,   # A0's greedy cut splits the dependency -> may fail
                    "logical_helps_if": "cut aligns to scene-shift seam and keeps the dependency intact"
                                        if any(cl == "scene-shift" for cl in seam_classes) else
                                        "avoids cutting a continuation seam mid-dependency"
                                        if any(cl == "continuation" for cl in seam_classes) else
                                        "boundary alignment orthogonal to this trap (weak Q1 discriminator)",
                },
            })

    # quota report
    from collections import Counter
    by_type_all = Counter(t["type"] for t in ledger)
    by_type_cross = Counter(t["type"] for t in ledger if t["crosses_A0_cut"])
    quota_status = {}
    for typ, need in QUOTA.items():
        have_cross = by_type_cross.get(typ, 0)
        quota_status[typ] = {"need": need, "have_all": by_type_all.get(typ, 0),
                             "have_crossing_A0": have_cross, "meets_quota": have_cross >= need}
    strata = Counter((t["type"], t["stratum"]) for t in ledger if t["crosses_A0_cut"])
    result = {
        "n_candidates_mined": sum(len(r.get("candidates", [])) for r in cands_raw),
        "n_in_ledger": len(ledger), "dropped": dropped,
        "a0_cut_offsets": a0_cuts,
        "quota_status": quota_status,
        "crossing_A0_by_type": dict(by_type_cross),
        "strata_crossing_A0": {f"{k[0]}/{k[1]}": v for k, v in strata.items()},
        "ledger": ledger,
    }
    OUT.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"candidates mined: {result['n_candidates_mined']}  -> ledger: {len(ledger)}  dropped: {dropped}")
    print("\nquota status (crossing an A0 cut = common denominator):")
    for typ, q in quota_status.items():
        mark = "OK" if q["meets_quota"] else "SHORT"
        print(f"  {typ:8} need>={q['need']:2}  have_all={q['have_all']:2}  crossing_A0={q['have_crossing_A0']:2}  [{mark}]")
    print(f"\nstrata (crossing A0): {result['strata_crossing_A0']}")
    print(f"-> {OUT}")
    print("\nNOTE: any SHORT type -> mine more windows OR mark 'Q undecidable a priori for this type' "
          "BEFORE paid arm/judge spend (design §D1).")


if __name__ == "__main__":
    main()