#!/usr/bin/env python3
"""exp16 — ADDITIVE canon extractor: base-rate-corrected (lift/PMI) dst-variants (research/20 §B1 mitigation
for ubiquitous terms). $0. NOT frozen — an analysis-layer improvement over the pre-registered chunk-Dice
(spread.dst_variants), which degenerates for terms present in ~every chunk (方源/蛊/古月 — no 'chunks
without X' contrast, so every frequent ru lemma looks associated).

lift(lemma, term) = |chunks(term) ∩ chunks(lemma)| / expected, expected = |chunks(term)|*|chunks(lemma)|/N.
A common ru word (год, everywhere) has lift≈1; the term's actual rendering (гу — only near 蛊) has high
lift. This surfaces the rendering even for ubiquitous terms. Reported ALONGSIDE the frozen method so the
pre-reg canon number and the true recoverability are both visible (§D4-а, no manufactured convergence).
"""
from __future__ import annotations

import exp16_common as X
import spread as SP
import palladius as PAL


def dst_variants_lift(sm: SP.SpreadModel, cand_norm: str, top=6, min_lift=1.5, min_cooccur=2):
    cidx = set(sm.cand_chunk_indices(cand_norm))
    N = len(sm.chunks)
    if len(cidx) < 1:
        return []
    from collections import Counter
    counts = Counter()
    for i in cidx:
        counts.update(sm.chunk_lemmas[i])
    scored = []
    for lm, co in counts.items():
        if co < min_cooccur:
            continue
        base = len(sm.lemma_chunks[lm])
        expected = len(cidx) * base / N
        lift = co / expected if expected else 0.0
        if lift >= min_lift:
            scored.append((lm, round(lift, 2), co))
    scored.sort(key=lambda t: (-t[1], -t[2], t[0]))
    return scored[:top]


def canon_propose_lift(sm, ent, top=6):
    lifted = dst_variants_lift(sm, X.norm(ent.src), top=top)
    lemmas = [lm for lm, _, _ in lifted]
    if ent.typ in ("name", "place", "nickname"):
        proposed = [lm for lm in lemmas if PAL.is_palladius_token(lm, 1) and sm.is_name_lemma(lm)]
        head = [lm for lm in lemmas if lm in ("гора", "деревня")]
        proposed = head[:1] + proposed if ent.typ == "place" else proposed
    else:
        proposed = lemmas[:4]
    return proposed, lifted


def canon_recovery_lift(sm, ent, cover_frac=0.5):
    import canon as CANON
    signed = set(CANON.signed_content_lemmas(ent.dst))
    if not signed:
        return None, [], []
    proposed, lifted = canon_propose_lift(sm, ent)
    assoc = {lm for lm, _, _ in lifted}
    covered = {s for s in signed if s in set(proposed) or s in assoc}
    return (len(covered) / len(signed) >= cover_frac), sorted(signed), proposed


def run(chunks, gt):
    sm = SP.SpreadModel(chunks)
    rec = tot = 0
    rows = []
    for e in gt:
        if not e.dst or X.gt_occurrences(e, chunks) == 0:
            continue
        r, signed, proposed = canon_recovery_lift(sm, e)
        if r is None:
            continue
        tot += 1
        rec += bool(r)
        rows.append(dict(src=e.src, typ=e.typ, recovered=bool(r), signed=signed, proposed=proposed))
    return rows, (rec / tot if tot else None), tot


if __name__ == "__main__":
    chunks = X.load_chunks(); gt = X.load_gt()
    sm = SP.SpreadModel(chunks)
    print("lift-based dst extraction (fixes ubiquitous-term degeneration) — full 57ch:\n")
    for s in ["蛊", "方源", "古月", "蛊师", "转", "元石", "族长"]:
        e = next(x for x in gt if x.src == s)
        prop, lifted = canon_propose_lift(sm, e)
        print(f"  {s:<4} signed=«{e.dst}» -> lift-proposed={prop}  top-lift={[(l,v) for l,v,_ in lifted[:4]]}")
    rows, rate, tot = run(chunks, gt)
    print(f"\ncanon-recovery (lift) on records.json full 57ch: {rate:.3f} (n={tot})")