#!/usr/bin/env python3 """exp16 — deterministic detectors V-A / V-B / V-C (research/20 §B1). miner-v1. $0. Common substrate: Han char n-grams 1–6 (exp16_common), freq floor 3, memnorm normalization, c-value nested discount with g(L)=log2(L+1) (avoids the |a|=1 degeneracy that would zero out 蛊/转 — §B1), GT occurrence counting via AC-equivalent WITHOUT suppressContained. V-A freq × contrast(weirdness, general-zh) × c-value(nested discount). Output: ranked src candidates. V-B V-A + translation-spread signal from the ru drafts (Dice/LTCR-style; pymorphy3 lemmas). +dst variants. V-C V-B + NER patterns / structural morphology / Palladius ru-side channel (patterns.py, palladius.py). Thresholds (floor, top-K, λ, m) are frozen after tuning on chapters 1–15; only chapters 16–25 are the reported test half (§D1 anti-overfit). This module exposes the scorer; tuning/arms live in arms.py. """ from __future__ import annotations import math from collections import defaultdict from dataclasses import dataclass, field import exp16_common as X # ── c-value (Frantzi et al. 2000, §A1) with §B1 length-multiplier fix ────────────────────────────── def length_mult(L: int) -> float: """g(L) = log2(L+1). §B1: NOT log2(L) — that zeros |a|=1 (蛊/转) and breaks the catastrophe screen.""" return math.log2(L + 1) def compute_cvalue(cand_freq: dict[str, int]) -> dict[str, float]: """C-value with nested discount. For a nested in longer candidates T_a: cval(a) = g(|a|)*(f(a) - (1/|T_a|)*Σ_{b∈T_a} f(b)); non-nested: g(|a|)*f(a).""" cands = set(cand_freq) tcount = defaultdict(int) # |T_a| tfreqsum = defaultdict(int) # Σ f(b) for b, fb in cand_freq.items(): Lb = len(b) if Lb < 2: continue subs = set() for n in range(1, Lb): for i in range(0, Lb - n + 1): a = b[i:i + n] if a in cands and a != b: subs.add(a) for a in subs: tcount[a] += 1 tfreqsum[a] += fb cval = {} for a, fa in cand_freq.items(): g = length_mult(len(a)) if tcount[a] > 0: cval[a] = g * (fa - tfreqsum[a] / tcount[a]) else: cval[a] = g * fa return cval # ── V-A ─────────────────────────────────────────────────────────────────────────────────────────── @dataclass class Candidate: src: str # normalized n-gram freq: int cvalue: float weirdness: float termhood: float score: float length: int chapters: set = field(default_factory=set) # V-B/V-C enrichments dst_variants: dict = field(default_factory=dict) # lemma -> count spread: float = 0.0 pattern_types: list = field(default_factory=list) # name/title/place/term hints pattern_evidence: list = field(default_factory=list) SUBSUME_ALPHA = 0.80 # drop candidate a if a longer container b has f(b) >= alpha*f(a): a is not independent def subsumed_candidates(cand_freq: dict[str, int], alpha=SUBSUME_ALPHA) -> set[str]: """Substring subsumption (nested consolidation for ranking): a candidate `a` that is (almost) always part of a longer candidate `b` (f(b) >= alpha*f(a)) is a boundary fragment, not an independent term — drop it. 方源(559) survives 方源的(85) [ratio 0.15]; 花酒行(65) is dropped by 花酒行者(65) [ratio 1.0]; 光蛊(96) dropped by 月光蛊(96). Occurrence counting for GT is unaffected (that path is suppress-free, D24.2) — this only cleans the candidate RANKING.""" cands = set(cand_freq) drop = set() # index candidates by length for efficient container lookup bylen = defaultdict(list) for c in cands: bylen[len(c)].append(c) for a, fa in cand_freq.items(): La = len(a) # any longer candidate b that strictly contains a with f(b) >= alpha*f(a) subsumes a found = False for Lb in range(La + 1, X.NGRAM_MAX + 1): for b in bylen[Lb]: if a in b and cand_freq[b] >= alpha * fa: found = True break if found: break if found: drop.add(a) return drop class VA: """Frequency × contrast × c-value. Params tunable; frozen after tuning half.""" def __init__(self, chunks, contrast: X.Contrast, freq_floor=X.FREQ_FLOOR, weird_floor=1e-9, termhood_w=1.0, subsume_alpha=SUBSUME_ALPHA): self.chunks = chunks self.C = contrast self.freq_floor = freq_floor self.weird_floor = weird_floor self.termhood_w = termhood_w self.subsume_alpha = subsume_alpha self.cand_freq = {} self.N_book = 0 def build(self): allc = X.enumerate_candidates(self.chunks) self.cand_freq = {k: v for k, v in allc.items() if v >= self.freq_floor} self.N_book = sum(allc.values()) # total n-gram token mass (all lengths) — weirdness denominator scale self.subsumed = subsumed_candidates(self.cand_freq, self.subsume_alpha) self.cval = compute_cvalue(self.cand_freq) return self def weirdness(self, a: str, f: int) -> float: """P_book(a) / P_general(a). P_general = max(word_rel, char_indep) smoothed (§A1 OOV smoothing).""" p_book = f / self.N_book gen = max(self.C.word_rel(a), self.C.char_indep_rel(a), self.weird_floor) return p_book / gen def score_all(self, drop_subsumed=True) -> list[Candidate]: out = [] for a, f in self.cand_freq.items(): if drop_subsumed and a in self.subsumed: continue w = self.weirdness(a, f) termhood = math.log1p(w) cv = self.cval[a] # score: unithood (c-value) gated by termhood (domain specificity). Negative c-value # (over-nested substring) floored at 0 — it is dominated by its container, not a term itself. score = max(cv, 0.0) * (termhood ** self.termhood_w) out.append(Candidate(src=a, freq=f, cvalue=cv, weirdness=w, termhood=termhood, score=score, length=len(a))) out.sort(key=lambda c: (-c.score, -c.freq, c.src)) return out # ── recall harness ──────────────────────────────────────────────────────────────────────────────── def rank_index(ranked: list[Candidate]) -> dict[str, int]: return {c.src: i for i, c in enumerate(ranked)} def gt_recall(ranked: list[Candidate], gt: list[X.GTEntity], chunks, top_k=None, stratum_filter=None, include_annotation=True): """Recall = fraction of GT entities whose src OR any alias surface appears in ranked[:top_k]. Returns (recall, hit_entities, miss_entities, per_entity_rank).""" idx = rank_index(ranked) limit = top_k if top_k is not None else len(ranked) hits, misses, per = [], [], {} considered = 0 for e in gt: f = X.gt_occurrences(e, chunks, include_annotation) if stratum_filter and X.freq_stratum(f) not in stratum_filter: continue considered += 1 best = None for surf in e.norm_surfaces: r = idx.get(surf) if r is not None and r < limit and (best is None or r < best): best = r per[e.src] = best if best is not None: hits.append(e) else: misses.append(e) recall = len(hits) / considered if considered else 0.0 return recall, hits, misses, per def catastrophe_ok(ranked: list[Candidate], top_k=50): idx = rank_index(ranked) res = {} for s in X.CATASTROPHE: r = idx.get(X.norm(s)) res[s] = r ok = all(r is not None and r < top_k for r in res.values()) return ok, res if __name__ == "__main__": import sys chunks = X.load_chunks() gt = X.load_gt() C = X.Contrast() va = VA(chunks, C).build() ranked = va.score_all() print(f"V-A: {len(ranked)} scored candidates (freq>={va.freq_floor})\n") print("== top-30 V-A candidates ==") gtsurf = {X.norm(s) for e in gt for s in e.surfaces} for i, c in enumerate(ranked[:30]): mark = "GT" if c.src in gtsurf else " " print(f" {i:>3} [{mark}] {c.src:<8} f={c.freq:<4} cval={c.cvalue:8.1f} weird={c.weirdness:10.1f} score={c.score:10.1f}") print("\n== catastrophe screen (top-50) ==") ok, res = catastrophe_ok(ranked, 50) for s, r in res.items(): print(f" {s}: rank {r}") print(f" screen {'PASS' if ok else 'FAIL'}") for K in (50, 80, 120, 200, 400): r_all, _, _, _ = gt_recall(ranked, gt, chunks, top_k=K) r_f3, _, miss_f3, _ = gt_recall(ranked, gt, chunks, top_k=K, stratum_filter={"f>=10", "f3-9"}) print(f"\n== top-{K}: recall(all GT)={r_all:.3f} recall(f>=3)={r_f3:.3f} misses(f>=3)={[e.src for e in miss_f3]}")