#!/usr/bin/env python3
"""exp16 — deterministic detectors V-A / V-B / V-C (research/20 §B1). miner-v1. $0.

Common substrate: Han char n-grams 1–6 (exp16_common), freq floor 3, memnorm normalization, c-value
nested discount with g(L)=log2(L+1) (avoids the |a|=1 degeneracy that would zero out 蛊/转 — §B1),
GT occurrence counting via AC-equivalent WITHOUT suppressContained.

  V-A  freq × contrast(weirdness, general-zh) × c-value(nested discount).  Output: ranked src candidates.
  V-B  V-A + translation-spread signal from the ru drafts (Dice/LTCR-style; pymorphy3 lemmas). +dst variants.
  V-C  V-B + NER patterns / structural morphology / Palladius ru-side channel (patterns.py, palladius.py).

Thresholds (floor, top-K, λ, m) are frozen after tuning on chapters 1–15; only chapters 16–25 are the
reported test half (§D1 anti-overfit). This module exposes the scorer; tuning/arms live in arms.py.
"""
from __future__ import annotations

import math
from collections import defaultdict
from dataclasses import dataclass, field

import exp16_common as X


# ── c-value (Frantzi et al. 2000, §A1) with §B1 length-multiplier fix ──────────────────────────────
def length_mult(L: int) -> float:
    """g(L) = log2(L+1). §B1: NOT log2(L) — that zeros |a|=1 (蛊/转) and breaks the catastrophe screen."""
    return math.log2(L + 1)


def compute_cvalue(cand_freq: dict[str, int]) -> dict[str, float]:
    """C-value with nested discount. For a nested in longer candidates T_a:
       cval(a) = g(|a|)*(f(a) - (1/|T_a|)*Σ_{b∈T_a} f(b)); non-nested: g(|a|)*f(a)."""
    cands = set(cand_freq)
    tcount = defaultdict(int)      # |T_a|
    tfreqsum = defaultdict(int)    # Σ f(b)
    for b, fb in cand_freq.items():
        Lb = len(b)
        if Lb < 2:
            continue
        subs = set()
        for n in range(1, Lb):
            for i in range(0, Lb - n + 1):
                a = b[i:i + n]
                if a in cands and a != b:
                    subs.add(a)
        for a in subs:
            tcount[a] += 1
            tfreqsum[a] += fb
    cval = {}
    for a, fa in cand_freq.items():
        g = length_mult(len(a))
        if tcount[a] > 0:
            cval[a] = g * (fa - tfreqsum[a] / tcount[a])
        else:
            cval[a] = g * fa
    return cval


# ── V-A ───────────────────────────────────────────────────────────────────────────────────────────
@dataclass
class Candidate:
    src: str            # normalized n-gram
    freq: int
    cvalue: float
    weirdness: float
    termhood: float
    score: float
    length: int
    chapters: set = field(default_factory=set)
    # V-B/V-C enrichments
    dst_variants: dict = field(default_factory=dict)   # lemma -> count
    spread: float = 0.0
    pattern_types: list = field(default_factory=list)   # name/title/place/term hints
    pattern_evidence: list = field(default_factory=list)


SUBSUME_ALPHA = 0.80   # drop candidate a if a longer container b has f(b) >= alpha*f(a): a is not independent

def subsumed_candidates(cand_freq: dict[str, int], alpha=SUBSUME_ALPHA) -> set[str]:
    """Substring subsumption (nested consolidation for ranking): a candidate `a` that is (almost)
    always part of a longer candidate `b` (f(b) >= alpha*f(a)) is a boundary fragment, not an
    independent term — drop it. 方源(559) survives 方源的(85) [ratio 0.15]; 花酒行(65) is dropped by
    花酒行者(65) [ratio 1.0]; 光蛊(96) dropped by 月光蛊(96). Occurrence counting for GT is unaffected
    (that path is suppress-free, D24.2) — this only cleans the candidate RANKING."""
    cands = set(cand_freq)
    drop = set()
    # index candidates by length for efficient container lookup
    bylen = defaultdict(list)
    for c in cands:
        bylen[len(c)].append(c)
    for a, fa in cand_freq.items():
        La = len(a)
        # any longer candidate b that strictly contains a with f(b) >= alpha*f(a) subsumes a
        found = False
        for Lb in range(La + 1, X.NGRAM_MAX + 1):
            for b in bylen[Lb]:
                if a in b and cand_freq[b] >= alpha * fa:
                    found = True
                    break
            if found:
                break
        if found:
            drop.add(a)
    return drop


class VA:
    """Frequency × contrast × c-value. Params tunable; frozen after tuning half."""

    def __init__(self, chunks, contrast: X.Contrast, freq_floor=X.FREQ_FLOOR,
                 weird_floor=1e-9, termhood_w=1.0, subsume_alpha=SUBSUME_ALPHA):
        self.chunks = chunks
        self.C = contrast
        self.freq_floor = freq_floor
        self.weird_floor = weird_floor
        self.termhood_w = termhood_w
        self.subsume_alpha = subsume_alpha
        self.cand_freq = {}
        self.N_book = 0

    def build(self):
        allc = X.enumerate_candidates(self.chunks)
        self.cand_freq = {k: v for k, v in allc.items() if v >= self.freq_floor}
        self.N_book = sum(allc.values())    # total n-gram token mass (all lengths) — weirdness denominator scale
        self.subsumed = subsumed_candidates(self.cand_freq, self.subsume_alpha)
        self.cval = compute_cvalue(self.cand_freq)
        return self

    def weirdness(self, a: str, f: int) -> float:
        """P_book(a) / P_general(a). P_general = max(word_rel, char_indep) smoothed (§A1 OOV smoothing)."""
        p_book = f / self.N_book
        gen = max(self.C.word_rel(a), self.C.char_indep_rel(a), self.weird_floor)
        return p_book / gen

    def score_all(self, drop_subsumed=True) -> list[Candidate]:
        out = []
        for a, f in self.cand_freq.items():
            if drop_subsumed and a in self.subsumed:
                continue
            w = self.weirdness(a, f)
            termhood = math.log1p(w)
            cv = self.cval[a]
            # score: unithood (c-value) gated by termhood (domain specificity). Negative c-value
            # (over-nested substring) floored at 0 — it is dominated by its container, not a term itself.
            score = max(cv, 0.0) * (termhood ** self.termhood_w)
            out.append(Candidate(src=a, freq=f, cvalue=cv, weirdness=w, termhood=termhood,
                                 score=score, length=len(a)))
        out.sort(key=lambda c: (-c.score, -c.freq, c.src))
        return out


# ── recall harness ────────────────────────────────────────────────────────────────────────────────
def rank_index(ranked: list[Candidate]) -> dict[str, int]:
    return {c.src: i for i, c in enumerate(ranked)}


def gt_recall(ranked: list[Candidate], gt: list[X.GTEntity], chunks, top_k=None,
              stratum_filter=None, include_annotation=True):
    """Recall = fraction of GT entities whose src OR any alias surface appears in ranked[:top_k].
    Returns (recall, hit_entities, miss_entities, per_entity_rank)."""
    idx = rank_index(ranked)
    limit = top_k if top_k is not None else len(ranked)
    hits, misses, per = [], [], {}
    considered = 0
    for e in gt:
        f = X.gt_occurrences(e, chunks, include_annotation)
        if stratum_filter and X.freq_stratum(f) not in stratum_filter:
            continue
        considered += 1
        best = None
        for surf in e.norm_surfaces:
            r = idx.get(surf)
            if r is not None and r < limit and (best is None or r < best):
                best = r
        per[e.src] = best
        if best is not None:
            hits.append(e)
        else:
            misses.append(e)
    recall = len(hits) / considered if considered else 0.0
    return recall, hits, misses, per


def catastrophe_ok(ranked: list[Candidate], top_k=50):
    idx = rank_index(ranked)
    res = {}
    for s in X.CATASTROPHE:
        r = idx.get(X.norm(s))
        res[s] = r
    ok = all(r is not None and r < top_k for r in res.values())
    return ok, res


if __name__ == "__main__":
    import sys
    chunks = X.load_chunks()
    gt = X.load_gt()
    C = X.Contrast()
    va = VA(chunks, C).build()
    ranked = va.score_all()
    print(f"V-A: {len(ranked)} scored candidates (freq>={va.freq_floor})\n")

    print("== top-30 V-A candidates ==")
    gtsurf = {X.norm(s) for e in gt for s in e.surfaces}
    for i, c in enumerate(ranked[:30]):
        mark = "GT" if c.src in gtsurf else "  "
        print(f"  {i:>3} [{mark}] {c.src:<8} f={c.freq:<4} cval={c.cvalue:8.1f} weird={c.weirdness:10.1f} score={c.score:10.1f}")

    print("\n== catastrophe screen (top-50) ==")
    ok, res = catastrophe_ok(ranked, 50)
    for s, r in res.items():
        print(f"  {s}: rank {r}")
    print(f"  screen {'PASS' if ok else 'FAIL'}")

    for K in (50, 80, 120, 200, 400):
        r_all, _, _, _ = gt_recall(ranked, gt, chunks, top_k=K)
        r_f3, _, miss_f3, _ = gt_recall(ranked, gt, chunks, top_k=K, stratum_filter={"f>=10", "f3-9"})
        print(f"\n== top-{K}: recall(all GT)={r_all:.3f}  recall(f>=3)={r_f3:.3f}  misses(f>=3)={[e.src for e in miss_f3]}")
