89 lines
3.6 KiB
Python
89 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
||
"""exp16 — ADDITIVE canon extractor: base-rate-corrected (lift/PMI) dst-variants (research/20 §B1 mitigation
|
||
for ubiquitous terms). $0. NOT frozen — an analysis-layer improvement over the pre-registered chunk-Dice
|
||
(spread.dst_variants), which degenerates for terms present in ~every chunk (方源/蛊/古月 — no 'chunks
|
||
without X' contrast, so every frequent ru lemma looks associated).
|
||
|
||
lift(lemma, term) = |chunks(term) ∩ chunks(lemma)| / expected, expected = |chunks(term)|*|chunks(lemma)|/N.
|
||
A common ru word (год, everywhere) has lift≈1; the term's actual rendering (гу — only near 蛊) has high
|
||
lift. This surfaces the rendering even for ubiquitous terms. Reported ALONGSIDE the frozen method so the
|
||
pre-reg canon number and the true recoverability are both visible (§D4-а, no manufactured convergence).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import exp16_common as X
|
||
import spread as SP
|
||
import palladius as PAL
|
||
|
||
|
||
def dst_variants_lift(sm: SP.SpreadModel, cand_norm: str, top=6, min_lift=1.5, min_cooccur=2):
|
||
cidx = set(sm.cand_chunk_indices(cand_norm))
|
||
N = len(sm.chunks)
|
||
if len(cidx) < 1:
|
||
return []
|
||
from collections import Counter
|
||
counts = Counter()
|
||
for i in cidx:
|
||
counts.update(sm.chunk_lemmas[i])
|
||
scored = []
|
||
for lm, co in counts.items():
|
||
if co < min_cooccur:
|
||
continue
|
||
base = len(sm.lemma_chunks[lm])
|
||
expected = len(cidx) * base / N
|
||
lift = co / expected if expected else 0.0
|
||
if lift >= min_lift:
|
||
scored.append((lm, round(lift, 2), co))
|
||
scored.sort(key=lambda t: (-t[1], -t[2], t[0]))
|
||
return scored[:top]
|
||
|
||
|
||
def canon_propose_lift(sm, ent, top=6):
|
||
lifted = dst_variants_lift(sm, X.norm(ent.src), top=top)
|
||
lemmas = [lm for lm, _, _ in lifted]
|
||
if ent.typ in ("name", "place", "nickname"):
|
||
proposed = [lm for lm in lemmas if PAL.is_palladius_token(lm, 1) and sm.is_name_lemma(lm)]
|
||
head = [lm for lm in lemmas if lm in ("гора", "деревня")]
|
||
proposed = head[:1] + proposed if ent.typ == "place" else proposed
|
||
else:
|
||
proposed = lemmas[:4]
|
||
return proposed, lifted
|
||
|
||
|
||
def canon_recovery_lift(sm, ent, cover_frac=0.5):
|
||
import canon as CANON
|
||
signed = set(CANON.signed_content_lemmas(ent.dst))
|
||
if not signed:
|
||
return None, [], []
|
||
proposed, lifted = canon_propose_lift(sm, ent)
|
||
assoc = {lm for lm, _, _ in lifted}
|
||
covered = {s for s in signed if s in set(proposed) or s in assoc}
|
||
return (len(covered) / len(signed) >= cover_frac), sorted(signed), proposed
|
||
|
||
|
||
def run(chunks, gt):
|
||
sm = SP.SpreadModel(chunks)
|
||
rec = tot = 0
|
||
rows = []
|
||
for e in gt:
|
||
if not e.dst or X.gt_occurrences(e, chunks) == 0:
|
||
continue
|
||
r, signed, proposed = canon_recovery_lift(sm, e)
|
||
if r is None:
|
||
continue
|
||
tot += 1
|
||
rec += bool(r)
|
||
rows.append(dict(src=e.src, typ=e.typ, recovered=bool(r), signed=signed, proposed=proposed))
|
||
return rows, (rec / tot if tot else None), tot
|
||
|
||
|
||
if __name__ == "__main__":
|
||
chunks = X.load_chunks(); gt = X.load_gt()
|
||
sm = SP.SpreadModel(chunks)
|
||
print("lift-based dst extraction (fixes ubiquitous-term degeneration) — full 57ch:\n")
|
||
for s in ["蛊", "方源", "古月", "蛊师", "转", "元石", "族长"]:
|
||
e = next(x for x in gt if x.src == s)
|
||
prop, lifted = canon_propose_lift(sm, e)
|
||
print(f" {s:<4} signed=«{e.dst}» -> lift-proposed={prop} top-lift={[(l,v) for l,v,_ in lifted[:4]]}")
|
||
rows, rate, tot = run(chunks, gt)
|
||
print(f"\ncanon-recovery (lift) on records.json full 57ch: {rate:.3f} (n={tot})")
|