93 lines
4.4 KiB
Python
93 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
||
"""exp16 — canon consolidation §C2 + canon-recovery metric §D3-3. miner-v1. $0.
|
||
|
||
Canon by ALL occurrences (NOT first-wins, anti-LTCR): the winning dst for a term is the rendering
|
||
scored by frequency-across-all-chunks × Palladius-conformity (type name/place) × case-lemma coverage —
|
||
the VARIANT wins, not the first occurrence. Majority NEVER yields approved (canon proposal = draft).
|
||
|
||
canon-recovery(ent): would the §C2 procedure, run over the drafts, propose a dst whose content lemmas
|
||
COVER the owner-signed seed dst (memnorm + lemma level)? On records.json (injected) this is trivialized
|
||
(spread≈0 — the drafts already carry the signed forms; §D1 confound). The ECOLOGICAL measurement is on
|
||
the cold-start drafts (no injection): does canon recover the signed dst from an un-anchored draft?
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import regex
|
||
|
||
import exp16_common as X
|
||
import spread as SP
|
||
import palladius as PAL
|
||
|
||
_COMMON_HEAD = {"гора", "деревня", "село", "селение", "стан", "клан", "род", "море", "первобытный",
|
||
"камень", "истинный", "ци", "апертура"} # generic heads still counted, but not name-distinctive
|
||
|
||
|
||
def signed_content_lemmas(dst: str) -> list[str]:
|
||
"""Distinctive lemmas of the signed dst (drop 2-char function words; keep name + key term words)."""
|
||
lem = SP.lemmatize_ru(dst)
|
||
return [l for l in lem if len(l) >= 3]
|
||
|
||
|
||
def canon_propose(sm: SP.SpreadModel, ent: X.GTEntity, top=6):
|
||
"""Reconstruct the canon dst proposal from the drafts by ALL occurrences.
|
||
For name/place: ordered Palladius name lemmas among the top associates. For term/title: the top
|
||
content associates. Returns (proposed_lemmas, evidence)."""
|
||
variants, cidx = sm.dst_variants(X.norm(ent.src), top=top)
|
||
lemmas = [lm for lm, d, _ in variants]
|
||
if ent.typ in ("name", "place", "nickname"):
|
||
# canon = Palladius-conformant capitalized-name lemmas (the transliteration), by Dice rank
|
||
proposed = [lm for lm in lemmas if PAL.is_palladius_token(lm, 1) and sm.is_name_lemma(lm)]
|
||
# for places, prepend the generic head if strongly associated (гора/деревня)
|
||
head = [lm for lm in lemmas if lm in ("гора", "деревня")]
|
||
proposed = head[:1] + proposed if ent.typ == "place" else proposed
|
||
else:
|
||
proposed = [lm for lm in lemmas if len(lm) >= 3][:4]
|
||
return proposed, variants
|
||
|
||
|
||
def canon_recovery(sm: SP.SpreadModel, ent: X.GTEntity, cover_frac=0.5):
|
||
"""Recovered if >= cover_frac of the signed distinctive lemmas appear among the proposed lemmas."""
|
||
signed = set(signed_content_lemmas(ent.dst))
|
||
if not signed:
|
||
return None, [], [] # nothing to recover (e.g. empty dst)
|
||
proposed, variants = canon_propose(sm, ent)
|
||
proposed_set = set(proposed)
|
||
# also allow coverage by ANY top associate lemma (the content word may not be a name token)
|
||
assoc = {lm for lm, d, _ in variants}
|
||
covered = {s for s in signed if s in proposed_set or s in assoc}
|
||
rec = len(covered) / len(signed) >= cover_frac
|
||
return rec, sorted(signed), proposed
|
||
|
||
|
||
def run(chunks, gt, label=""):
|
||
sm = SP.SpreadModel(chunks)
|
||
rows = []
|
||
rec_n = tot = 0
|
||
for e in gt:
|
||
if not e.dst:
|
||
continue
|
||
occ = X.gt_occurrences(e, chunks)
|
||
if occ == 0:
|
||
continue
|
||
rec, signed, proposed = canon_recovery(sm, e)
|
||
if rec is None:
|
||
continue
|
||
tot += 1
|
||
rec_n += bool(rec)
|
||
rows.append(dict(src=e.src, dst=e.dst, typ=e.typ, occ=occ, recovered=bool(rec),
|
||
signed=signed, proposed=proposed))
|
||
return rows, (rec_n / tot if tot else None), tot
|
||
|
||
|
||
if __name__ == "__main__":
|
||
chunks = X.load_chunks(); gt = X.load_gt()
|
||
rows, rate, tot = run(chunks, gt, "records.json (INJECTED — confounded upper bound, §D1)")
|
||
print(f"canon-recovery on records.json (INJECTED drafts — confound §D1 trivializes this): "
|
||
f"{sum(r['recovered'] for r in rows)}/{tot} = {rate:.3f}\n")
|
||
print("misses (signed vs proposed):")
|
||
for r in rows:
|
||
if not r["recovered"]:
|
||
print(f" {r['src']:<6}({r['typ']:<8}) signed={r['signed']} proposed={r['proposed']}")
|
||
print("\nsample recovered:")
|
||
for r in [x for x in rows if x["recovered"]][:12]:
|
||
print(f" {r['src']:<6}({r['typ']:<8}) signed={r['signed']} -> proposed={r['proposed']}")
|