219 lines
10 KiB
Python
219 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""exp16 — owner-facing deliverables (research/20 §C2-8 карта подписи, §E-1 мини-голд, §D1 precision@30). $0.
|
||
|
||
Emits three markdown sheets for the owner (sidecars, NOT the seed schema — miner never writes approved):
|
||
1. signature_map.md — per proposed entity: canon dst-proposal (by ALL occurrences), dst-variants,
|
||
top-KWIC contexts (H15 lever), gender counters (他/她), zone flags Z1-Z5, weak edges.
|
||
2. alias_minigold.md — candidate entity-edges for the ~20-30 min owner touchpoint (strike/add).
|
||
3. precision_at30.md — top-30 NON-seed A3 candidates for owner adjudication (§D1 precision@30 primary path).
|
||
|
||
Uses cold-start drafts for canon/spread if present (ecological), else injected records.json (confounded).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from pathlib import Path
|
||
|
||
import exp16_common as X
|
||
import spread as SP
|
||
import canon as CANON
|
||
import palladius as PAL
|
||
import arms as A
|
||
import alias as ALIAS
|
||
|
||
BOOK = Path("/home/ubuntu/books/gu-zhenren")
|
||
OUT = BOOK / "exp16"
|
||
COLD_PLAIN = OUT / "coldstart" / "plain"
|
||
|
||
_RE_HE, _RE_SHE = re.compile("他"), re.compile("她")
|
||
|
||
|
||
def kwic(term, chunks, window=18, maxn=3):
|
||
tn = X.norm(term)
|
||
out = []
|
||
for c in chunks:
|
||
s = c.source
|
||
for m in re.finditer(re.escape(term), s):
|
||
i = m.start()
|
||
ctx = s[max(0, i - window):i + len(term) + window].replace("\n", " ")
|
||
out.append(f"…{ctx}…")
|
||
if len(out) >= maxn:
|
||
return out
|
||
return out
|
||
|
||
|
||
def gender_counts(term, chunks, window=40):
|
||
"""他/她 counts within `window` chars of each occurrence — EVIDENCE ONLY (verdict = owner, D19.3)."""
|
||
he = she = 0
|
||
for c in chunks:
|
||
s = c.source
|
||
for m in re.finditer(re.escape(term), s):
|
||
seg = s[max(0, m.start() - window):m.end() + window]
|
||
he += len(_RE_HE.findall(seg))
|
||
she += len(_RE_SHE.findall(seg))
|
||
return he, she
|
||
|
||
|
||
def zone_flags(ent, cand, sm, chunks):
|
||
"""§B4 zone tags for the candidate (evidence, not verdict)."""
|
||
flags = []
|
||
sn = X.norm(ent.src if hasattr(ent, "src") else cand.src)
|
||
sp = sm.spread(sn)
|
||
variants, _ = sm.dst_variants(sn)
|
||
top_dst = variants[0][0] if variants else ""
|
||
# Z1 suspicious_stable: high score + spread~0 + dst not a Palladius name and not capitalized-name
|
||
is_name_dst = PAL.is_palladius_token(top_dst, 2) and sm.is_name_lemma(top_dst)
|
||
if cand and cand.score > 500 and sp < 0.05 and top_dst and not is_name_dst and \
|
||
(hasattr(ent, "typ") and ent.typ in ("term", "title")):
|
||
flags.append("Z1?suspicious_stable")
|
||
if sp >= 0.3:
|
||
flags.append("Z2?sense_split/spread")
|
||
f = X.gt_occurrences(ent, chunks) if hasattr(ent, "norm_surfaces") else cand.freq
|
||
if f < 3:
|
||
flags.append("Z3:rare")
|
||
return flags
|
||
|
||
|
||
def load_banknote_dst():
|
||
"""Aggregate translator-emitted src->dst from the cold-start banknote drafts (reliable dst; §3.3)."""
|
||
bdir = OUT / "coldstart" / "banknote"
|
||
dst = {}
|
||
for f in sorted(bdir.glob("*.banknote.json")):
|
||
for e in json.load(open(f))["entries"]:
|
||
sn = X.norm(e["src"])
|
||
dst.setdefault(sn, []).append(e["dst"])
|
||
# majority dst per src
|
||
from collections import Counter
|
||
return {sn: Counter(v).most_common(1)[0][0] for sn, v in dst.items()}
|
||
|
||
|
||
def build_signature_map(gt, chunks, sm, a3, tag, bank_dst=None):
|
||
bank_dst = bank_dst or {}
|
||
lines = [f"# Карта подписи exp16 (miner-v1, {tag}) — сайдкар владельцу",
|
||
"",
|
||
"> Майнер НИКОГДА не пишет `approved`. Ниже — предложения `draft` по кластерам: канон-предложение "
|
||
"(по ВСЕМ вхождениям × Палладий-конформность), dst-варианты, KWIC-контексты (детерминированный "
|
||
"H15-рычаг), счётчики пола (他/她 — evidence, вердикт ваш, D19.3), флаги зон §B4. "
|
||
"Владелец: approve / правка / оставить draft / расклеить.",
|
||
""]
|
||
a3_by = {c.src: c for c in a3}
|
||
# cover GT entities (the measurable set) + top non-GT name/place candidates
|
||
gt_by_norm = {X.norm(s): e for e in gt for s in e.surfaces}
|
||
seen = set()
|
||
for e in sorted(gt, key=lambda e: -X.gt_occurrences(e, chunks)):
|
||
sn = X.norm(e.src)
|
||
if sn in seen:
|
||
continue
|
||
seen.add(sn)
|
||
cand = a3_by.get(sn)
|
||
proposed, variants = CANON.canon_propose(sm, e)
|
||
vstr = ", ".join(f"{lm}({d})" for lm, d, _ in variants[:4])
|
||
he, she = gender_counts(e.src, chunks)
|
||
zf = zone_flags(e, cand, sm, chunks)
|
||
occ = X.gt_occurrences(e, chunks)
|
||
# dst proposal priority: banknote (translator-emitted, reliable) > Palladius-name canon > co-occ note
|
||
bank = bank_dst.get(sn)
|
||
if bank:
|
||
canon_str = f"«{bank}» (banknote-канал)"
|
||
elif proposed:
|
||
canon_str = f"«{' '.join(proposed)}» (Палладий-канон)"
|
||
else:
|
||
canon_str = "(dst — банкнота/облако: co-occ извлечение вырождено, §3.3)"
|
||
lines.append(f"### {e.src} — предл. dst: {canon_str} "
|
||
f"[{e.typ}, occ={occ}, статус сида={e.status}, подписанный dst=«{e.dst}»]")
|
||
lines.append(f"- co-occ dst-варианты (справочно): {vstr}")
|
||
if he or she:
|
||
lines.append(f"- пол-evidence: 他={he} 她={she} (вердикт — владелец)")
|
||
if zf:
|
||
lines.append(f"- зоны §B4: {', '.join(zf)}")
|
||
for k in kwic(e.src, chunks):
|
||
lines.append(f"- KWIC: `{k}`")
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def build_minigold(gt, chunks, sm):
|
||
gt_by_norm = {X.norm(s): e for e in gt for s in e.surfaces}
|
||
ar = A.Arms(chunks, X.Contrast())
|
||
a3 = ar.arm_A3()
|
||
subsumed = ar.va.subsumed
|
||
gt_surf = {X.norm(s) for e in gt for s in e.surfaces}
|
||
PARTICLE = ALIAS.__dict__.get("PARTICLE", set())
|
||
name_cands = [c.src for c in a3[:200] if any(t in c.types for t in ("name", "place", "title"))
|
||
and c.freq >= 5 and c.src not in subsumed and len(c.src) >= 2]
|
||
surface_srcs = list(gt_surf | set(name_cands))
|
||
surfaces = ALIAS.build_surfaces(surface_srcs, sm, gt_by_norm)
|
||
ident, family, weak = ALIAS.propose_edges(surfaces, chunks, sm, frozenset(gt_surf))
|
||
clusters = ALIAS.cluster(ident, list(surfaces))
|
||
lines = ["# Мини-голд алиасов exp16 — тачпойнт владельца (~20–30 мин)",
|
||
"",
|
||
"> Код ПРЕДЛОЖИЛ рёбра сущностей слайса-25. Вычеркните ложные, допишите пропущенные. Только после "
|
||
"этого меряется recall кластеризации (сейчас — только precision предложенных). Тир-1 имеет "
|
||
"известную ошибку (§B2): спорное держим раздельным draft.", "",
|
||
"## Предложенные IDENTITY-кластеры (одна сущность):"]
|
||
for cl in clusters:
|
||
lines.append(f"- [ ] {{ {', '.join(cl)} }}")
|
||
lines += ["", "## Предложенные FAMILY-суперкластеры (общая фамилия, НЕ тождество):"]
|
||
fam_pairs = sorted({tuple(sorted((e['a'], e['b']))) for e in family})
|
||
for a, b in fam_pairs[:40]:
|
||
lines.append(f"- [ ] {a} ~ {b}")
|
||
lines += ["", "## Слабые/tier-2 рёбра (со-оккуренция, решения нет — предложения к связи):"]
|
||
for e in weak[:30]:
|
||
lines.append(f"- [ ] {e['a']} ? {e['b']} [{e.get('rule','')}]")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def build_precision30(gt, chunks, sm, a3):
|
||
gt_surf = {X.norm(s) for e in gt for s in e.surfaces}
|
||
non_seed = [c for c in a3 if c.src not in gt_surf][:30]
|
||
lines = ["# precision@30 — топ-30 НЕ-сидовых кандидатов A3 (V-C) для адъюдикации владельца",
|
||
"",
|
||
"> Отметьте: T = валидная сущность банка (имя/титул/место/термин), F = мусор/фрагмент, "
|
||
"? = спорно. Это первичный путь precision@30 (§D1; precision@сид НЕ метрика — сид не исчерпывающий).",
|
||
"", "| # | кандидат | freq | типы | evidence | dst-намёк | T/F/? |",
|
||
"|---|---|---|---|---|---|---|"]
|
||
for i, c in enumerate(non_seed):
|
||
variants, _ = sm.dst_variants(c.src, top=2)
|
||
dsth = variants[0][0] if variants else ""
|
||
ev = ",".join(c.evidence[:2])
|
||
lines.append(f"| {i+1} | {c.src} | {c.freq} | {'/'.join(c.types) or '-'} | {ev} | {dsth} | |")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def main():
|
||
gt = X.load_gt()
|
||
# prefer cold-start drafts (ecological) for canon/spread; fall back to injected
|
||
from coldstart_analyze import load_coldstart_chunks, COLD_CHAPTERS
|
||
cold = load_coldstart_chunks("plain")
|
||
if cold:
|
||
chunks_for_canon = cold
|
||
tag = f"cold-start ch{COLD_CHAPTERS}"
|
||
else:
|
||
chunks_for_canon = X.load_chunks()
|
||
tag = "injected records.json (confounded — cold-start refines)"
|
||
full = X.load_chunks()
|
||
sm = SP.SpreadModel(chunks_for_canon)
|
||
ar = A.Arms(full, X.Contrast())
|
||
a3 = ar.arm_A3()
|
||
|
||
# signature map over the canon-source chunks
|
||
gt_canon = [e for e in gt if X.gt_occurrences(e, chunks_for_canon) >= 1]
|
||
bank_dst = load_banknote_dst()
|
||
sig = build_signature_map(gt_canon, chunks_for_canon, sm, a3, tag, bank_dst=bank_dst)
|
||
(OUT / "signature_map.md").write_text(sig, encoding="utf-8")
|
||
|
||
sm_full = SP.SpreadModel(full)
|
||
mg = build_minigold(gt, full, sm_full)
|
||
(OUT / "alias_minigold.md").write_text(mg, encoding="utf-8")
|
||
|
||
p30 = build_precision30(gt, full, sm_full, a3)
|
||
(OUT / "precision_at30.md").write_text(p30, encoding="utf-8")
|
||
|
||
print(f"[written] {OUT}/signature_map.md ({len(gt_canon)} entities, canon source: {tag})")
|
||
print(f"[written] {OUT}/alias_minigold.md")
|
||
print(f"[written] {OUT}/precision_at30.md (top-30 non-seed A3)")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|