236 lines
12 KiB
Python
236 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""exp16 — V-C pattern channels (research/20 §B1 channels 1–4/6). miner-v1. $0.
|
||
|
||
Language×genre pattern pack for zh (§B5 plugin P3). Versioned data (surnames / title affixes / topo
|
||
suffixes) + a book-adaptive productive-morphology detector (channel 4 — auto-detects the domain formant,
|
||
NOT hardcoded 蛊). Each channel proposes TYPED candidates (name/title/place/term) with evidence, closing
|
||
V-A's frequency-blind classes (rare surname-anchored names, rank/grade titles, one-off realia).
|
||
|
||
Channels:
|
||
(1) surname anchor 百家姓 + compound surnames, + 1–2 Han window right -> name
|
||
(2) title affixes suffix 公子/大人/长老/前辈/族长/嬷嬷…, prefix 老/小/阿 + ordinals -> title/name
|
||
(3) topo suffixes 山/寨/村/疆/谷/城/门/宗… -> place
|
||
(4) productive morphology char that binds with >=m distinct n-grams (蛊/等/转…) -> term/title (auto)
|
||
(6) genre lexicon pack xianxia suffix/title lexicon (this file IS the zh-xianxia pack)
|
||
|
||
Owner decision 18.07: genre PACKS are NOT built (deferred to book 2); channel (6) runs only on UNIVERSAL
|
||
channels — 百家姓 anchor, title/topo suffixes, productive morphology (auto-detected formant, not hardcoded
|
||
蛊), Palladius on the ru side. This module encodes exactly that universal set.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from collections import Counter, defaultdict
|
||
|
||
import exp16_common as X
|
||
|
||
# ── (1) surnames: 百家姓 single-char subset + compound surnames (versioned inventory) ───────────────
|
||
# Standard 百家姓 opening + the surnames present in this book's cast. Compound surnames listed explicitly.
|
||
SURNAMES_SINGLE = set(
|
||
"赵钱孙李周吴郑王冯陈褚卫蒋沈韩杨朱秦尤许何吕施张孔曹严华金魏陶姜"
|
||
"戚谢邹喻柏水窦章云苏潘葛奚范彭郎鲁韦昌马苗凤花方俞任袁柳酆鲍史唐"
|
||
"费廉岑薛雷贺倪汤滕殷罗毕郝邬安常乐于时傅皮卞齐康伍余元卜顾孟平黄"
|
||
"和穆萧尹姚邵湛汪祁毛禹狄米贝明臧计伏成戴谈宋茅庞熊纪舒屈项祝董梁"
|
||
"杜阮蓝闵席季麻强贾路娄危江童颜郭梅盛林刁钟徐邱骆高夏蔡田樊胡凌霍"
|
||
"虞万支柯昝管卢莫经房裘缪干解应宗丁宣贲邓郁单杭洪包诸左石崔吉钮龚"
|
||
"白凝" # 白 (Bai clan). 凝 is not a surname but kept out — handled below via compound guard.
|
||
)
|
||
SURNAMES_SINGLE.discard("凝")
|
||
SURNAMES_COMPOUND = {"古月", "欧阳", "司马", "上官", "夏侯", "诸葛", "东方", "皇甫", "尉迟", "公孙",
|
||
"慕容", "长孙", "宇文", "司徒", "鲜于", "南宫"}
|
||
|
||
# ── (2) title affixes (universal honorific/rank suffixes+prefixes; §A4 Cao) ─────────────────────────
|
||
TITLE_SUFFIX = ["公子", "大人", "长老", "前辈", "族长", "家老", "老祖", "祖师", "真人", "上人",
|
||
"道人", "先生", "夫人", "娘子", "姑娘", "嬷嬷", "师傅", "师父", "掌门", "宗主",
|
||
"少爷", "老爷", "小姐", "婆婆", "大娘", "大爷"]
|
||
TITLE_PREFIX = ["老", "小", "阿"]
|
||
ORDINAL = ["一代", "二代", "三代", "四代", "五代", "六代", "七代", "八代", "九代", "十代",
|
||
"第一", "第二", "第三", "第四", "第五"]
|
||
|
||
# ── (3) topo suffixes -> place ─────────────────────────────────────────────────────────────────────
|
||
TOPO_SUFFIX = ["山", "寨", "村", "疆", "谷", "城", "门", "宗", "府", "洞", "峰", "岭", "河",
|
||
"江", "湖", "海", "林", "原", "岛", "潭", "殿", "阁", "楼", "院", "堂", "祠"]
|
||
|
||
# ── (2c) rank/grade compositional pattern (numeral/sequential + rank-word) -> title ────────────────
|
||
# Grade/rank words are COMMON chars (等 over_rep 1.1, 转 3.9) — not formant-detectable; caught by the
|
||
# sequential-prefix + rank-word composition instead. Universal (numeral+rank), not genre-specific.
|
||
GRADE_PREFIX = list("甲乙丙丁戊己庚辛壬癸") # sequential grade markers
|
||
NUMERAL = list("一二三四五六七八九十零百千") # Chinese numerals
|
||
RANK_WORD = ["等", "转", "阶", "级", "重", "品", "段", "层"] # rank/grade head words
|
||
|
||
PACK_VERSION = "zh-universal-v1"
|
||
|
||
|
||
def is_surname_start(s: str):
|
||
"""Return the surname prefix (compound preferred) if s starts with one, else None."""
|
||
for cs in SURNAMES_COMPOUND:
|
||
if s.startswith(cs):
|
||
return cs
|
||
if s and s[0] in SURNAMES_SINGLE:
|
||
return s[0]
|
||
return None
|
||
|
||
|
||
# ── (4) productive-morphology (formant) auto-detection ─────────────────────────────────────────────
|
||
def book_char_freq(chunks) -> Counter:
|
||
bc = Counter()
|
||
for c in chunks:
|
||
for ch in c.nsource:
|
||
if X._RE_HAN.match(ch):
|
||
bc[ch] += 1
|
||
return bc
|
||
|
||
|
||
def detect_formants(chunks, cand_freq: dict[str, int], contrast: X.Contrast, min_partners=3,
|
||
min_over_rep=15.0) -> dict[str, dict]:
|
||
"""A Han char c is a productive DOMAIN formant if it binds (suffix OR prefix) with >= min_partners
|
||
DISTINCT content morphemes among candidates AND is over-represented in-book vs general zh
|
||
(p_book/p_general >= min_over_rep). Over-representation — NOT char-rarity — is the domain signal:
|
||
it isolates 蛊(2723×)/窍(88×)/虫(32×) while rejecting common chars 师/花/等(1.1×). Auto-detected,
|
||
not hardcoded (§B1 channel 4). Common rank-words (等/转) are handled by the compositional channel."""
|
||
bc = book_char_freq(chunks)
|
||
tot_book = sum(bc.values()) or 1
|
||
suf = defaultdict(set)
|
||
pre = defaultdict(set)
|
||
for a in cand_freq:
|
||
if len(a) < 2:
|
||
continue
|
||
suf[a[-1]].add(a[:-1])
|
||
pre[a[0]].add(a[1:])
|
||
formants = {}
|
||
for c in set(suf) | set(pre):
|
||
p_book = bc.get(c, 0) / tot_book
|
||
p_gen = (contrast.char_freq.get(c, 0) + 1) / (contrast.total_char + len(contrast.char_freq))
|
||
over_rep = p_book / p_gen if p_gen else 0.0
|
||
if over_rep < min_over_rep:
|
||
continue
|
||
ns, npr = len(suf.get(c, ())), len(pre.get(c, ()))
|
||
role = None
|
||
if ns >= min_partners:
|
||
role = "suffix"
|
||
if npr >= min_partners:
|
||
role = "both" if role else "prefix"
|
||
if role:
|
||
formants[c] = {"role": role, "over_rep": round(over_rep, 1),
|
||
"suffix_partners": ns, "prefix_partners": npr}
|
||
return formants
|
||
|
||
|
||
# formant -> candidate type (universal heuristic; not genre-conditioned)
|
||
def formant_type(c: str) -> str:
|
||
if c in TOPO_SUFFIX:
|
||
return "place"
|
||
if c in ("等", "转", "阶"):
|
||
return "title"
|
||
return "term"
|
||
|
||
|
||
# ── channel application: produce typed pattern candidates from the source ──────────────────────────
|
||
def pattern_candidates(chunks, cand_freq: dict[str, int], contrast: X.Contrast,
|
||
min_partners=3, min_over_rep=15.0):
|
||
"""Return {cand_norm: {'types': [...], 'evidence': [...]}} for pattern-detected candidates.
|
||
Operates over the ALREADY-normalized source; proposes candidates that may be BELOW the V-A freq
|
||
floor (that is the point — patterns close the rare-term blind spot)."""
|
||
out = defaultdict(lambda: {"types": [], "evidence": []})
|
||
|
||
def add(cand, typ, ev):
|
||
cand = X.norm(cand)
|
||
if not cand or not X.han_only(cand):
|
||
return
|
||
if typ not in out[cand]["types"]:
|
||
out[cand]["types"].append(typ)
|
||
if ev not in out[cand]["evidence"]:
|
||
out[cand]["evidence"].append(ev)
|
||
|
||
# scan each source for surname / title / topo patterns over Han runs
|
||
for c in chunks:
|
||
for run in X.han_runs(c.nsource):
|
||
L = len(run)
|
||
for i in range(L):
|
||
# (1) surname anchor: surname + 1–2 Han given-name window
|
||
sn = is_surname_start(run[i:])
|
||
if sn:
|
||
base = i + len(sn)
|
||
for gl in (1, 2):
|
||
if base + gl <= L:
|
||
full = run[i:base + gl]
|
||
if 2 <= len(full) <= 4:
|
||
add(full, "name", f"surname:{sn}")
|
||
# (2) title suffix: content + suffix
|
||
for suf in TITLE_SUFFIX:
|
||
if run.startswith(suf, i):
|
||
# take up to 3 Han to the LEFT as the titled base (e.g. 沈+嬷嬷, 四代+族长)
|
||
for left in (3, 2, 1, 0):
|
||
if i - left >= 0:
|
||
cand = run[i - left:i + len(suf)]
|
||
if 2 <= len(cand) <= 6:
|
||
add(cand, "title", f"title_suffix:{suf}")
|
||
add(suf, "title", f"title_bare:{suf}")
|
||
# (2b) ordinal + title (四代族长-class)
|
||
for od in ORDINAL:
|
||
if run.startswith(od, i):
|
||
for suf in TITLE_SUFFIX:
|
||
end = i + len(od)
|
||
if run.startswith(suf, end):
|
||
add(run[i:end + len(suf)], "title", f"ordinal_title:{od}+{suf}")
|
||
# (2c) rank/grade compositional: (numeral|grade-prefix) + rank-word -> title
|
||
for rw in RANK_WORD:
|
||
if run.startswith(rw, i) and i >= 1:
|
||
left = run[i - 1]
|
||
if left in NUMERAL or left in GRADE_PREFIX:
|
||
add(run[i - 1:i + len(rw)], "title", f"rank_grade:{left}+{rw}")
|
||
# (3) topo suffix: 1–3 Han base + topo char
|
||
if run[i] in TOPO_SUFFIX and i >= 1:
|
||
for left in (3, 2, 1):
|
||
if i - left >= 0:
|
||
cand = run[i - left:i + 1]
|
||
if 2 <= len(cand) <= 4:
|
||
add(cand, "place", f"topo_suffix:{run[i]}")
|
||
|
||
# (4) productive morphology: for each detected formant, propose all binding n-grams
|
||
formants = detect_formants(chunks, cand_freq, contrast, min_partners, min_over_rep)
|
||
for c in chunks:
|
||
for run in X.han_runs(c.nsource):
|
||
L = len(run)
|
||
for i in range(L):
|
||
if run[i] not in formants:
|
||
continue
|
||
info = formants[run[i]]
|
||
typ = formant_type(run[i])
|
||
if info["role"] in ("suffix", "both"):
|
||
for left in (3, 2, 1):
|
||
if i - left >= 0:
|
||
cand = run[i - left:i + 1]
|
||
if 2 <= len(cand) <= 4:
|
||
add(cand, typ, f"formant_suffix:{run[i]}")
|
||
if info["role"] in ("prefix", "both"):
|
||
for r in (2, 3):
|
||
if i + r <= L:
|
||
cand = run[i:i + r]
|
||
if 2 <= len(cand) <= 4:
|
||
add(cand, typ, f"formant_prefix:{run[i]}")
|
||
return dict(out), formants
|
||
|
||
|
||
if __name__ == "__main__":
|
||
chunks = X.load_chunks()
|
||
gt = X.load_gt()
|
||
C = X.Contrast()
|
||
import detectors as D
|
||
va = D.VA(chunks, C).build()
|
||
pats, formants = pattern_candidates(chunks, va.cand_freq, C)
|
||
print(f"pattern candidates: {len(pats)}")
|
||
print(f"\ndetected formants ({len(formants)}):")
|
||
for c, info in sorted(formants.items(), key=lambda kv: -kv[1]["over_rep"]):
|
||
print(f" {c} {info}")
|
||
# how many f>=3-miss GT terms does the pattern layer now propose?
|
||
gtsurf = {X.norm(s): e for e in gt for s in e.surfaces}
|
||
hit = [s for s in gtsurf if s in pats]
|
||
print(f"\nGT surfaces proposed by patterns: {len(hit)}/{len(gtsurf)}")
|
||
# specifically the V-A f>=3 misses
|
||
focus = ["转", "一转", "三转", "五转", "六转", "甲等", "乙等", "丙等", "丁等", "白凝冰", "江牙",
|
||
"沈嬷嬷", "四代族长", "族长", "青茅山", "古月山寨", "希望蛊", "月影蛊", "月光蛊", "人祖"]
|
||
print("\nfocus (V-A structural misses) — proposed by patterns?")
|
||
for s in focus:
|
||
sn = X.norm(s)
|
||
info = pats.get(sn)
|
||
print(f" {s:<6} {'YES ' + str(info['types']) + ' ' + str(info['evidence'][:2]) if info else 'no'}")
|