151 lines
8.2 KiB
Python
151 lines
8.2 KiB
Python
#!/usr/bin/env python3
|
|
"""kana-precision measurement (D16-tail / E1-protocol extension; POLYGON handoff Task 5).
|
|
|
|
QUESTION (memory.go:37-48): minKeyLenPhonetic=3 is a "conservative default pending a ja-kana
|
|
precision measurement". This quantifies matcher PRECISION on kana name-keys at MIN ∈ {2,3,4},
|
|
so the prod-ja decision (confirm 3 / raise / split by kana type) is grounded, not asserted.
|
|
|
|
The failure mode: a short kana name-key (glossary key) firing as a FALSE POSITIVE inside an
|
|
unrelated common Japanese word — the kana analog of a Han homograph trap. Two distinct causes,
|
|
tagged in kana_traps.json (the agent partitioned them, and they behave DIFFERENTLY under a floor):
|
|
- substring: the name hides inside a LONGER unrelated word (リン ⊂ リンゴ apple). Killed by a
|
|
high-enough MIN floor.
|
|
- homograph: the name IS a whole common noun (ひかる=光る shine, あさひ=morning sun, ひまわり=
|
|
sunflower). LENGTH-INVARIANT — a MIN floor cannot filter it; it is a full substring at every
|
|
length. Needs POS / known-word gating, not a longer key.
|
|
|
|
Matcher variants measured:
|
|
- substring: the CURRENT Go source-side matcher (Aho-Corasick = pure substring, no boundary).
|
|
- script_boundary: emulates the D16.3 fix ("script/word-boundary on the source side for
|
|
phonetic keys"). For kana this means "the match must be flanked by NON-kana (kanji/latin/
|
|
punct) or a string edge". This is the critical ja finding: unlike Latin (rose⊂roseanne,
|
|
where spaces/letters bound words), Japanese has NO word spaces and names take kana PARTICLES
|
|
(メロスは…, 名前が…), so a kana-run boundary rule SUPPRESSES real names followed by a particle
|
|
→ recall collapse. Measured on both so the orchestrator sees the trade, not just the win.
|
|
|
|
Both operate on the NORMALIZED (kana-folded) text — imported from the memory_eval Go-mirror so
|
|
this measures the SAME normalization the hot path uses (katakana→hiragana, NFKC, ignorables).
|
|
|
|
Usage: eval/.venv/bin/python eval/pilot/kana_precision.py
|
|
"""
|
|
from __future__ import annotations
|
|
import json
|
|
from pathlib import Path
|
|
import memory_eval as me # reuse the Go-mirror normalization (norm_src, significant_len, any_han)
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
TRAPS = json.loads((ROOT / "kana_traps.json").read_text(encoding="utf-8"))
|
|
|
|
|
|
def is_kana(c: str) -> bool:
|
|
"""Hiragana/katakana (incl. prolonged mark ー). After norm_src katakana folds to hiragana,
|
|
so the flanking test operates on a hiragana run; ー (U+30FC) is not folded, still kana."""
|
|
return 0x3040 <= ord(c) <= 0x30FF
|
|
|
|
|
|
def eligible(name_norm: str, min_phonetic: int) -> bool:
|
|
"""A kana key (no Han) is eligible only if significantLen >= the phonetic floor under test
|
|
(min_key_len_for with minKeyLenPhonetic := min_phonetic). Han keys are out of scope here."""
|
|
floor = 2 if me.any_han(name_norm) else min_phonetic
|
|
return me.significant_len(name_norm) >= floor
|
|
|
|
|
|
def collision_prone_at(name_norm: str, min_phonetic: int) -> bool:
|
|
"""Mirror of collisionProneKey with the floor == the const under test: a fired key of
|
|
significantLen <= min_phonetic is AMBIGUOUS (softened + forced post-check); a longer key
|
|
fires CONFIRMED (authoritative). The dangerous FP is the CONFIRMED one."""
|
|
return (not me.any_han(name_norm)) and me.significant_len(name_norm) <= min_phonetic
|
|
|
|
|
|
def fires_substring(name_norm: str, text_norm: str) -> bool:
|
|
return name_norm in text_norm
|
|
|
|
|
|
def fires_script_boundary(name_norm: str, text_norm: str) -> bool:
|
|
"""D16.3 emulation for kana: a substring hit is valid only if flanked by non-kana or edge."""
|
|
i = text_norm.find(name_norm)
|
|
n = len(name_norm)
|
|
while i != -1:
|
|
left_ok = i == 0 or not is_kana(text_norm[i - 1])
|
|
j = i + n
|
|
right_ok = j == len(text_norm) or not is_kana(text_norm[j])
|
|
if left_ok and right_ok:
|
|
return True
|
|
i = text_norm.find(name_norm, i + 1)
|
|
return False
|
|
|
|
|
|
def measure(matcher, min_phonetic: int) -> dict:
|
|
"""Precision over the trap set at one floor with one matcher. FP from collision_traps,
|
|
TP from true_positive_contexts. Partition FP by cause (substring/homograph) and by
|
|
disposition (ambiguous=softened / confirmed=authoritative)."""
|
|
fp = {"substring": 0, "homograph": 0, "confirmed": 0, "ambiguous": 0, "banned": 0, "items": []}
|
|
for t in TRAPS["collision_traps"]:
|
|
nk = me.norm_src(t["name_kana"])
|
|
cw = me.norm_src(t["containing_word_kana"])
|
|
if not eligible(nk, min_phonetic):
|
|
fp["banned"] += 1
|
|
continue
|
|
if matcher(nk, cw):
|
|
fp[t["type"]] += 1
|
|
disp = "ambiguous" if collision_prone_at(nk, min_phonetic) else "confirmed"
|
|
fp[disp] += 1
|
|
fp["items"].append(f'{t["name_kana"]}({t["len"]})⊂{t["containing_word_kana"]}[{t["type"]},{disp}]')
|
|
tp = {"fired": 0, "banned": 0, "missed_boundary": 0}
|
|
for c in TRAPS["true_positive_contexts"]:
|
|
nk = me.norm_src(c["name_kana"])
|
|
sent = me.norm_src(c["sentence_kana"])
|
|
if not eligible(nk, min_phonetic):
|
|
tp["banned"] += 1
|
|
continue
|
|
if matcher(nk, sent):
|
|
tp["fired"] += 1
|
|
else:
|
|
tp["missed_boundary"] += 1 # eligible but the boundary rule suppressed a REAL name
|
|
n_fp = fp["substring"] + fp["homograph"]
|
|
prec = round(tp["fired"] / (tp["fired"] + n_fp), 3) if (tp["fired"] + n_fp) else None
|
|
# CONFIRMED-precision: among AUTHORITATIVE firings only (AMBIGUOUS is softened by ⟨проверить⟩)
|
|
tp_conf = sum(1 for c in TRAPS["true_positive_contexts"]
|
|
if eligible(me.norm_src(c["name_kana"]), min_phonetic)
|
|
and not collision_prone_at(me.norm_src(c["name_kana"]), min_phonetic)
|
|
and matcher(me.norm_src(c["name_kana"]), me.norm_src(c["sentence_kana"])))
|
|
conf_prec = round(tp_conf / (tp_conf + fp["confirmed"]), 3) if (tp_conf + fp["confirmed"]) else None
|
|
return {"min": min_phonetic, "tp_fired": tp["fired"], "tp_banned": tp["banned"],
|
|
"tp_missed_boundary": tp["missed_boundary"], "fp_total": n_fp,
|
|
"fp_substring": fp["substring"], "fp_homograph": fp["homograph"],
|
|
"fp_confirmed": fp["confirmed"], "fp_ambiguous": fp["ambiguous"], "fp_banned": fp["banned"],
|
|
"precision": prec, "confirmed_precision": conf_prec, "fp_items": fp["items"]}
|
|
|
|
|
|
def main():
|
|
print("kana-precision (D16-tail). Trap set: "
|
|
f'{len(TRAPS["collision_traps"])} collision pairs, '
|
|
f'{len(TRAPS["true_positive_contexts"])} true-positive contexts. '
|
|
"FP by cause; disposition confirmed=authoritative / ambiguous=softened.\n")
|
|
for label, matcher in (("substring (CURRENT Go)", fires_substring),
|
|
("script_boundary (D16.3 emulation)", fires_script_boundary)):
|
|
print(f"### matcher = {label}")
|
|
print("MIN | TP_fire | TP_missed(bound) | FP_tot | FP_substr | FP_homog | FP_CONFIRMED | FP_ambig | precision | conf_prec")
|
|
rows = [measure(matcher, m) for m in (2, 3, 4)]
|
|
for r in rows:
|
|
print(f" {r['min']} | {r['tp_fired']:2} | {r['tp_missed_boundary']:2} |"
|
|
f" {r['fp_total']:2} | {r['fp_substring']:2} | {r['fp_homograph']:2} |"
|
|
f" {r['fp_confirmed']:2} | {r['fp_ambiguous']:2} | "
|
|
f"{r['precision'] if r['precision'] is not None else ' -'} | "
|
|
f"{r['confirmed_precision'] if r['confirmed_precision'] is not None else ' -'}")
|
|
print()
|
|
# dump machine-readable for the report / re-run comparison
|
|
out = {"substring": [measure(fires_substring, m) for m in (2, 3, 4)],
|
|
"script_boundary": [measure(fires_script_boundary, m) for m in (2, 3, 4)]}
|
|
outp = ROOT.parent / "data" / "pilot" / "kana_precision.json"
|
|
outp.parent.mkdir(parents=True, exist_ok=True)
|
|
outp.write_text(json.dumps(out, ensure_ascii=False, indent=2))
|
|
print(f"wrote {outp}")
|
|
# headline read
|
|
s3 = out["substring"][1]
|
|
print(f"\nHEADLINE (substring, MIN=3): precision={s3['precision']} conf_prec={s3['confirmed_precision']} "
|
|
f"| FP_confirmed(len>MIN)={s3['fp_confirmed']} are {[i for i in s3['fp_items'] if 'confirmed' in i]}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|