218 lines
9.1 KiB
Python
218 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
||
"""exp16 — deterministic detectors V-A / V-B / V-C (research/20 §B1). miner-v1. $0.
|
||
|
||
Common substrate: Han char n-grams 1–6 (exp16_common), freq floor 3, memnorm normalization, c-value
|
||
nested discount with g(L)=log2(L+1) (avoids the |a|=1 degeneracy that would zero out 蛊/转 — §B1),
|
||
GT occurrence counting via AC-equivalent WITHOUT suppressContained.
|
||
|
||
V-A freq × contrast(weirdness, general-zh) × c-value(nested discount). Output: ranked src candidates.
|
||
V-B V-A + translation-spread signal from the ru drafts (Dice/LTCR-style; pymorphy3 lemmas). +dst variants.
|
||
V-C V-B + NER patterns / structural morphology / Palladius ru-side channel (patterns.py, palladius.py).
|
||
|
||
Thresholds (floor, top-K, λ, m) are frozen after tuning on chapters 1–15; only chapters 16–25 are the
|
||
reported test half (§D1 anti-overfit). This module exposes the scorer; tuning/arms live in arms.py.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import math
|
||
from collections import defaultdict
|
||
from dataclasses import dataclass, field
|
||
|
||
import exp16_common as X
|
||
|
||
|
||
# ── c-value (Frantzi et al. 2000, §A1) with §B1 length-multiplier fix ──────────────────────────────
|
||
def length_mult(L: int) -> float:
|
||
"""g(L) = log2(L+1). §B1: NOT log2(L) — that zeros |a|=1 (蛊/转) and breaks the catastrophe screen."""
|
||
return math.log2(L + 1)
|
||
|
||
|
||
def compute_cvalue(cand_freq: dict[str, int]) -> dict[str, float]:
|
||
"""C-value with nested discount. For a nested in longer candidates T_a:
|
||
cval(a) = g(|a|)*(f(a) - (1/|T_a|)*Σ_{b∈T_a} f(b)); non-nested: g(|a|)*f(a)."""
|
||
cands = set(cand_freq)
|
||
tcount = defaultdict(int) # |T_a|
|
||
tfreqsum = defaultdict(int) # Σ f(b)
|
||
for b, fb in cand_freq.items():
|
||
Lb = len(b)
|
||
if Lb < 2:
|
||
continue
|
||
subs = set()
|
||
for n in range(1, Lb):
|
||
for i in range(0, Lb - n + 1):
|
||
a = b[i:i + n]
|
||
if a in cands and a != b:
|
||
subs.add(a)
|
||
for a in subs:
|
||
tcount[a] += 1
|
||
tfreqsum[a] += fb
|
||
cval = {}
|
||
for a, fa in cand_freq.items():
|
||
g = length_mult(len(a))
|
||
if tcount[a] > 0:
|
||
cval[a] = g * (fa - tfreqsum[a] / tcount[a])
|
||
else:
|
||
cval[a] = g * fa
|
||
return cval
|
||
|
||
|
||
# ── V-A ───────────────────────────────────────────────────────────────────────────────────────────
|
||
@dataclass
|
||
class Candidate:
|
||
src: str # normalized n-gram
|
||
freq: int
|
||
cvalue: float
|
||
weirdness: float
|
||
termhood: float
|
||
score: float
|
||
length: int
|
||
chapters: set = field(default_factory=set)
|
||
# V-B/V-C enrichments
|
||
dst_variants: dict = field(default_factory=dict) # lemma -> count
|
||
spread: float = 0.0
|
||
pattern_types: list = field(default_factory=list) # name/title/place/term hints
|
||
pattern_evidence: list = field(default_factory=list)
|
||
|
||
|
||
SUBSUME_ALPHA = 0.80 # drop candidate a if a longer container b has f(b) >= alpha*f(a): a is not independent
|
||
|
||
def subsumed_candidates(cand_freq: dict[str, int], alpha=SUBSUME_ALPHA) -> set[str]:
|
||
"""Substring subsumption (nested consolidation for ranking): a candidate `a` that is (almost)
|
||
always part of a longer candidate `b` (f(b) >= alpha*f(a)) is a boundary fragment, not an
|
||
independent term — drop it. 方源(559) survives 方源的(85) [ratio 0.15]; 花酒行(65) is dropped by
|
||
花酒行者(65) [ratio 1.0]; 光蛊(96) dropped by 月光蛊(96). Occurrence counting for GT is unaffected
|
||
(that path is suppress-free, D24.2) — this only cleans the candidate RANKING."""
|
||
cands = set(cand_freq)
|
||
drop = set()
|
||
# index candidates by length for efficient container lookup
|
||
bylen = defaultdict(list)
|
||
for c in cands:
|
||
bylen[len(c)].append(c)
|
||
for a, fa in cand_freq.items():
|
||
La = len(a)
|
||
# any longer candidate b that strictly contains a with f(b) >= alpha*f(a) subsumes a
|
||
found = False
|
||
for Lb in range(La + 1, X.NGRAM_MAX + 1):
|
||
for b in bylen[Lb]:
|
||
if a in b and cand_freq[b] >= alpha * fa:
|
||
found = True
|
||
break
|
||
if found:
|
||
break
|
||
if found:
|
||
drop.add(a)
|
||
return drop
|
||
|
||
|
||
class VA:
|
||
"""Frequency × contrast × c-value. Params tunable; frozen after tuning half."""
|
||
|
||
def __init__(self, chunks, contrast: X.Contrast, freq_floor=X.FREQ_FLOOR,
|
||
weird_floor=1e-9, termhood_w=1.0, subsume_alpha=SUBSUME_ALPHA):
|
||
self.chunks = chunks
|
||
self.C = contrast
|
||
self.freq_floor = freq_floor
|
||
self.weird_floor = weird_floor
|
||
self.termhood_w = termhood_w
|
||
self.subsume_alpha = subsume_alpha
|
||
self.cand_freq = {}
|
||
self.N_book = 0
|
||
|
||
def build(self):
|
||
allc = X.enumerate_candidates(self.chunks)
|
||
self.cand_freq = {k: v for k, v in allc.items() if v >= self.freq_floor}
|
||
self.N_book = sum(allc.values()) # total n-gram token mass (all lengths) — weirdness denominator scale
|
||
self.subsumed = subsumed_candidates(self.cand_freq, self.subsume_alpha)
|
||
self.cval = compute_cvalue(self.cand_freq)
|
||
return self
|
||
|
||
def weirdness(self, a: str, f: int) -> float:
|
||
"""P_book(a) / P_general(a). P_general = max(word_rel, char_indep) smoothed (§A1 OOV smoothing)."""
|
||
p_book = f / self.N_book
|
||
gen = max(self.C.word_rel(a), self.C.char_indep_rel(a), self.weird_floor)
|
||
return p_book / gen
|
||
|
||
def score_all(self, drop_subsumed=True) -> list[Candidate]:
|
||
out = []
|
||
for a, f in self.cand_freq.items():
|
||
if drop_subsumed and a in self.subsumed:
|
||
continue
|
||
w = self.weirdness(a, f)
|
||
termhood = math.log1p(w)
|
||
cv = self.cval[a]
|
||
# score: unithood (c-value) gated by termhood (domain specificity). Negative c-value
|
||
# (over-nested substring) floored at 0 — it is dominated by its container, not a term itself.
|
||
score = max(cv, 0.0) * (termhood ** self.termhood_w)
|
||
out.append(Candidate(src=a, freq=f, cvalue=cv, weirdness=w, termhood=termhood,
|
||
score=score, length=len(a)))
|
||
out.sort(key=lambda c: (-c.score, -c.freq, c.src))
|
||
return out
|
||
|
||
|
||
# ── recall harness ────────────────────────────────────────────────────────────────────────────────
|
||
def rank_index(ranked: list[Candidate]) -> dict[str, int]:
|
||
return {c.src: i for i, c in enumerate(ranked)}
|
||
|
||
|
||
def gt_recall(ranked: list[Candidate], gt: list[X.GTEntity], chunks, top_k=None,
|
||
stratum_filter=None, include_annotation=True):
|
||
"""Recall = fraction of GT entities whose src OR any alias surface appears in ranked[:top_k].
|
||
Returns (recall, hit_entities, miss_entities, per_entity_rank)."""
|
||
idx = rank_index(ranked)
|
||
limit = top_k if top_k is not None else len(ranked)
|
||
hits, misses, per = [], [], {}
|
||
considered = 0
|
||
for e in gt:
|
||
f = X.gt_occurrences(e, chunks, include_annotation)
|
||
if stratum_filter and X.freq_stratum(f) not in stratum_filter:
|
||
continue
|
||
considered += 1
|
||
best = None
|
||
for surf in e.norm_surfaces:
|
||
r = idx.get(surf)
|
||
if r is not None and r < limit and (best is None or r < best):
|
||
best = r
|
||
per[e.src] = best
|
||
if best is not None:
|
||
hits.append(e)
|
||
else:
|
||
misses.append(e)
|
||
recall = len(hits) / considered if considered else 0.0
|
||
return recall, hits, misses, per
|
||
|
||
|
||
def catastrophe_ok(ranked: list[Candidate], top_k=50):
|
||
idx = rank_index(ranked)
|
||
res = {}
|
||
for s in X.CATASTROPHE:
|
||
r = idx.get(X.norm(s))
|
||
res[s] = r
|
||
ok = all(r is not None and r < top_k for r in res.values())
|
||
return ok, res
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import sys
|
||
chunks = X.load_chunks()
|
||
gt = X.load_gt()
|
||
C = X.Contrast()
|
||
va = VA(chunks, C).build()
|
||
ranked = va.score_all()
|
||
print(f"V-A: {len(ranked)} scored candidates (freq>={va.freq_floor})\n")
|
||
|
||
print("== top-30 V-A candidates ==")
|
||
gtsurf = {X.norm(s) for e in gt for s in e.surfaces}
|
||
for i, c in enumerate(ranked[:30]):
|
||
mark = "GT" if c.src in gtsurf else " "
|
||
print(f" {i:>3} [{mark}] {c.src:<8} f={c.freq:<4} cval={c.cvalue:8.1f} weird={c.weirdness:10.1f} score={c.score:10.1f}")
|
||
|
||
print("\n== catastrophe screen (top-50) ==")
|
||
ok, res = catastrophe_ok(ranked, 50)
|
||
for s, r in res.items():
|
||
print(f" {s}: rank {r}")
|
||
print(f" screen {'PASS' if ok else 'FAIL'}")
|
||
|
||
for K in (50, 80, 120, 200, 400):
|
||
r_all, _, _, _ = gt_recall(ranked, gt, chunks, top_k=K)
|
||
r_f3, _, miss_f3, _ = gt_recall(ranked, gt, chunks, top_k=K, stratum_filter={"f>=10", "f3-9"})
|
||
print(f"\n== top-{K}: recall(all GT)={r_all:.3f} recall(f>=3)={r_f3:.3f} misses(f>=3)={[e.src for e in miss_f3]}")
|