110 lines
6.3 KiB
Python
110 lines
6.3 KiB
Python
#!/usr/bin/env python3
|
||
"""exp15 — DETERMINISTIC-first straddle-trap builder (guard 1, owner/orch 2026-07-18). Finds cohesion
|
||
traps that ACTUALLY straddle a flat-A0-30 greedy boundary (the common denominator), from the seed +
|
||
surface heuristics — no flaky/expensive LLM for the deterministic core. Kimi ENRICHES T-ana/T-ell only.
|
||
|
||
Types built here:
|
||
T-ent (NEGATIVE CONTROL): a seed entity in chunk_i tail AND chunk_{i+1} head. NOTE the glossary
|
||
injection already enforces its canonical dst per-chunk, so carryover should add ~nothing — a
|
||
negative control that confirms the trap set discriminates (T-ana should move, T-ent should not).
|
||
T-ana (PRIMARY, carryover-discriminating): an anaphor (他/她/它/自己/这/那) at chunk_{i+1} head whose
|
||
antecedent (a person-entity) is in chunk_i tail. Without carryover the chunk_{i+1} translator lacks
|
||
the antecedent -> may mis-resolve gender/referent (ru past tense marks gender).
|
||
Undecidable a-priori on this material (guard 3, do NOT imitate): T-tense/T-cat/T-gen/T-reg (density
|
||
3/80 in mining = prose property, not slice size).
|
||
|
||
Each trap: id, type, boundary_idx, anchor_span (chunk_i tail sent), dependent_span (chunk_{i+1} head
|
||
sent), phenomenon, expected_ru, crosses_A0_cut=True. Run: python straddle_traps.py
|
||
"""
|
||
from __future__ import annotations
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
from chunker import greedy_chunks, split_source_sentences
|
||
from mem_select import build_bank, normalize_source_key
|
||
|
||
SD = Path("/home/ubuntu/books/gu-zhenren/exp15")
|
||
FLAT = SD / "S2prime_big_flat.txt"
|
||
SEED = "/home/ubuntu/books/gu-zhenren/guzhenren-seed-v2.yaml"
|
||
OUT = SD / "straddle_ledger.json"
|
||
ANAPHORS = ("他", "她", "它", "自己", "这", "那", "其", "此")
|
||
|
||
|
||
def person_gender(bank, src):
|
||
for e in bank.entries:
|
||
if e.src == src:
|
||
# gender is on the seed row; look it up from the raw seed (bank doesn't keep it)
|
||
return None
|
||
return None
|
||
|
||
|
||
def main():
|
||
import yaml
|
||
seed = {t["src"]: t for t in yaml.safe_load(Path(SEED).read_text(encoding="utf-8"))["terms"]}
|
||
bank = build_bank(SEED)
|
||
text = FLAT.read_text(encoding="utf-8")
|
||
ch = greedy_chunks(text)
|
||
traps = []
|
||
for i in range(len(ch) - 1):
|
||
tail_sents = split_source_sentences(ch[i])[-2:]
|
||
head_sents = split_source_sentences(ch[i + 1])[:2]
|
||
tail = "".join(tail_sents)
|
||
head = "".join(head_sents)
|
||
nt, nh = normalize_source_key(tail), normalize_source_key(head)
|
||
# T-ent straddle (negative control)
|
||
for e in bank.entries:
|
||
if e.status != "approved" or not e.norm_keys or not e.dst:
|
||
continue
|
||
if any(k in nt for k in e.norm_keys) and any(k in nh for k in e.norm_keys):
|
||
traps.append({
|
||
"id": f"T-ent-{len(traps)+1}", "type": "T-ent", "control": True,
|
||
"boundary_idx": i, "entity_src": e.src, "entity_dst": e.dst,
|
||
"anchor_span": tail, "dependent_span": head,
|
||
"phenomenon": f"Сущность {e.src} введена в чанке {i}, повторена в чанке {i+1} через разрез "
|
||
f"— НО глоссарий-инъекция даёт «{e.dst}» в обоих чанках (NEGATIVE CONTROL: "
|
||
"carryover не должен ничего добавить).",
|
||
"expected_ru": f"{e.dst} — консистентная каноническая форма в обоих чанках.",
|
||
"crosses_A0_cut": True,
|
||
})
|
||
break # one T-ent per boundary
|
||
# T-ana straddle (primary): anaphor at head, antecedent = last person-name in tail
|
||
hstart = head.lstrip(" \n")[:3]
|
||
if any(hstart.startswith(p) for p in ANAPHORS):
|
||
# find the last person-name in the tail
|
||
antecedent = None
|
||
for e in bank.entries:
|
||
if e.status == "approved" and seed.get(e.src, {}).get("type") in ("name",) and e.norm_keys \
|
||
and any(k in nt for k in e.norm_keys):
|
||
g = seed.get(e.src, {}).get("gender", "")
|
||
antecedent = (e.src, e.dst, g)
|
||
g = antecedent[2] if antecedent else ""
|
||
gtag = {"male": "муж. род (пошёл/сказал)", "female": "жен. род (пошла/сказала)",
|
||
"hidden": "род скрыт", "": "род по контексту"}.get(g, "род по контексту")
|
||
traps.append({
|
||
"id": f"T-ana-{len(traps)+1}", "type": "T-ana", "control": False,
|
||
"boundary_idx": i,
|
||
"antecedent": antecedent[0] if antecedent else None,
|
||
"antecedent_dst": antecedent[1] if antecedent else None,
|
||
"anchor_span": tail, "dependent_span": head,
|
||
"phenomenon": f"Анафора «{hstart}…» в начале чанка {i+1}; антецедент "
|
||
f"{'('+antecedent[0]+')' if antecedent else '(в чанке '+str(i)+')'} — по ДРУГУЮ "
|
||
"сторону разреза. Без carryover переводчик чанка i+1 не видит антецедент.",
|
||
"expected_ru": f"Верный референт/{gtag}"
|
||
+ (f'; антецедент = {antecedent[1]}' if antecedent else ''),
|
||
"crosses_A0_cut": True,
|
||
})
|
||
from collections import Counter
|
||
by = Counter(t["type"] for t in traps)
|
||
OUT.write_text(json.dumps({"n": len(traps), "by_type": dict(by),
|
||
"note": "deterministic-first (guard 1); T-ent=negative control (glossary-handled); "
|
||
"T-ana=primary carryover-discriminating; T-tense/cat/gen/reg=undecidable a-priori (guard 3)",
|
||
"traps": traps}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
print(f"deterministic straddle traps: {dict(by)} (total {len(traps)}) -> {OUT}")
|
||
print(" T-ent = negative control (glossary enforces); T-ana = primary (carryover-discriminating)")
|
||
print(f" T-ana deterministic = {by.get('T-ana',0)}; quota >=20 -> kimi enrichment needed for +{max(0,20-by.get('T-ana',0))}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|