183 lines
9.9 KiB
Python
183 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
||
"""Decl-aware (склонение-осознанная) consistency metric for the Ф2.5 memory-eval.
|
||
|
||
WHY decl-aware is MANDATORY (research/14 §2, quantified; backend memory_e1_test.go):
|
||
a naive word-boundary regexp on the base form gives 18-67 pct FALSE flags on inflected Russian — forms
|
||
rendered CORRECTLY ("Вэйчжуане", "Бородатого Вана", "Дэна", "сюцая") that a boundary regexp
|
||
on the lemma misses. Without decl-awareness the eval is noise and the bank-vs-long-context
|
||
verdict is untrustworthy. This module is the eval-side analogue of the Go post-check
|
||
(backend/internal/pipeline/mempostcheck.go): whole-word, decl-tolerant, letter-boundary.
|
||
|
||
Two matchers, both reusable:
|
||
- pymorphy3 lemma-set: a name is "rendered" if its canonical LEMMA(s) appear among the
|
||
lemmatized words of the output (folds Вэйчжуане→вэйчжуан, Вана→ван). Mirrors the Go
|
||
post-check's intent with an OOV-tolerant morphological analyzer (pymorphy3 lemmatizes
|
||
even unknown translit names).
|
||
- hyphen-translit regex fallback: for hyphenated translit names (А-кью, У-ма) pymorphy3
|
||
splits on the hyphen, so a regex like "А[-space]?кью" is the correct decl-tolerant matcher.
|
||
|
||
Also exposes GO-STYLE decl-form whole-word matching (given stored decl forms), so the eval
|
||
can compare "stored-decl completeness" (Go's real mechanism) against pymorphy fallback —
|
||
the E1 measurement the handoff asks the polygon to validate on real books.
|
||
|
||
D13.5 SYNC WITH GO (07-strategic-review §4.5; POLYGON handoff Task 1e-tail + package-2 Task 1b):
|
||
normalize_target and containsWholeWord are now a FAITHFUL mirror of memnorm.go normalizeTargetForm /
|
||
mempostcheck.go containsWholeWord — NFC → dash/hyphen-variant fold → apostrophe-variant fold
|
||
(‘ ’ ʼ ＇ → ') → default-ignorable strip (Cf + variation selectors) → whitespace-collapse → lower
|
||
→ ё→е, and a boundary test on the Unicode IsLetter equivalent (category L*) rather than
|
||
Cyrillic-only. Before this, a translit name adjacent to a Latin letter or split by a zero-width
|
||
joiner false-missed/false-hit vs Go; the apostrophe fold (package-2 Task 1b) closes a false MISS
|
||
on д'Артаньян vs д’Артаньян — critical for the en→ru pilot arm.
|
||
"""
|
||
from __future__ import annotations
|
||
import re
|
||
import unicodedata
|
||
import pymorphy3
|
||
|
||
_MORPH = pymorphy3.MorphAnalyzer()
|
||
_WORD_RE = re.compile(r"[А-Яа-яЁё]+(?:-[А-Яа-яЁё]+)?")
|
||
|
||
# Dash/hyphen/minus variants that fold to ASCII '-' (mirror of Go isDashVariant): hyphen,
|
||
# non-breaking hyphen, figure dash, en/em dash, horizontal bar, math minus, soft hyphen.
|
||
_DASH_VARIANTS = frozenset("‐‑‒–—―−")
|
||
|
||
# Typographic apostrophe/single-quote variants that fold to ASCII '\'' (mirror of Go
|
||
# isApostropheVariant): U+2018/2019 curly single quotes, U+02BC modifier apostrophe, U+FF07
|
||
# fullwidth apostrophe. Symmetric with the source side so д'Артаньян vs д’Артаньян is never a
|
||
# false post-check MISS (critical for the en→ru pilot arm).
|
||
_APOSTROPHE_VARIANTS = frozenset("‘’ʼ＇")
|
||
|
||
|
||
def _is_ignorable(c: str) -> bool:
|
||
"""Mirror of Go isIgnorableForMatch: default-ignorable / format code points (unicode.Cf:
|
||
ZWSP/ZWNJ/ZWJ/WJ/BOM/bidi) plus the variation selectors (VS1-16 U+FE00..FE0F and the
|
||
ideographic supplement U+E0100..E01EF). Dropped before matching so a stored «вана» is not
|
||
a false MISS against «ва<WJ>на». Soft hyphen (also Cf) is folded to '-' BEFORE this."""
|
||
o = ord(c)
|
||
return unicodedata.category(c) == "Cf" or (0xFE00 <= o <= 0xFE0F) or (0xE0100 <= o <= 0xE01EF)
|
||
|
||
|
||
def _is_letter(c: str) -> bool:
|
||
"""unicode.IsLetter equivalent: Unicode general category L* (Lu/Ll/Lt/Lm/Lo). Used for
|
||
the whole-word boundary so a Latin-letter neighbour bounds a translit name too (Go parity;
|
||
the old Cyrillic-only [А-Яа-яЁё] test missed Latin boundaries)."""
|
||
return unicodedata.category(c)[0] == "L"
|
||
|
||
|
||
def normalize_target(s: str) -> str:
|
||
"""NFC → dash/hyphen-variant fold → apostrophe-variant fold → drop default-ignorables →
|
||
whitespace-collapse → lower → ё→е. Symmetric mirror of Go normalizeTargetForm (memnorm.go),
|
||
applied to BOTH the stored decl forms AND the model output so a correctly-rendered term is
|
||
NEVER a false MISS (the dash/apostrophe/ignorable folds are separate steps, matching Go)."""
|
||
s = unicodedata.normalize("NFC", s)
|
||
out: list[str] = []
|
||
prev_space = False
|
||
for c in s:
|
||
if c in _DASH_VARIANTS:
|
||
c = "-" # soft hyphen folds here, so it is kept (not dropped below)
|
||
if c in _APOSTROPHE_VARIANTS:
|
||
c = "'" # symmetric with the source side (д'Артаньян) — no false MISS
|
||
if _is_ignorable(c):
|
||
continue
|
||
if c.isspace():
|
||
if not prev_space:
|
||
out.append(" ")
|
||
prev_space = True
|
||
continue
|
||
prev_space = False
|
||
if c == "İ": # U+0130: Go unicode.ToLower→'i' (Python str.lower→'i̇'); match Go (parity-review #1)
|
||
out.append("i")
|
||
continue
|
||
c = c.lower()
|
||
if c == "ё":
|
||
c = "е"
|
||
out.append(c)
|
||
return "".join(out).strip()
|
||
|
||
|
||
def lemmas(text: str) -> set[str]:
|
||
"""Lemma set of the Cyrillic words in text (ё→е folded)."""
|
||
out = set()
|
||
for w in _WORD_RE.findall(text):
|
||
w = w.replace("ё", "е")
|
||
out.add(_MORPH.parse(w)[0].normal_form)
|
||
# also add the raw lowered form (for hyphenated/translit that pymorphy keeps whole)
|
||
out.add(w.lower())
|
||
return out
|
||
|
||
|
||
def rendered_pymorphy(output: str, lemma_keys: list[str], hyphen_pat: str | None = None) -> bool:
|
||
"""A term is rendered if ALL its canonical lemma keys appear in the output's lemma set.
|
||
hyphen_pat: a regex for hyphenated translit (А-кью) matched directly on the output."""
|
||
if hyphen_pat:
|
||
return bool(re.search(hyphen_pat, output, re.I))
|
||
lem = lemmas(output)
|
||
return all(k.replace("ё", "е").lower() in lem for k in lemma_keys)
|
||
|
||
|
||
def contains_whole_word(text_norm: str, form_norm: str) -> bool:
|
||
"""Go-style whole-word (letter-boundary) containment (mirror of Go containsWholeWord).
|
||
Boundary = Unicode IsLetter (category L*) on both ends. Both args must be normalize_target()'d."""
|
||
if not form_norm:
|
||
return False
|
||
n = len(form_norm)
|
||
i = text_norm.find(form_norm)
|
||
while i != -1:
|
||
left_ok = i == 0 or not _is_letter(text_norm[i - 1])
|
||
j = i + n
|
||
right_ok = j == len(text_norm) or not _is_letter(text_norm[j])
|
||
if left_ok and right_ok:
|
||
return True
|
||
i = text_norm.find(form_norm, i + 1)
|
||
return False
|
||
|
||
|
||
def rendered_stored_decl(output: str, decl_forms: list[str], base_dst: str) -> bool:
|
||
"""Go post-check's REAL mechanism: whole-word match against STORED decl forms
|
||
(fallback to the base dst when decl is empty — the naive case E1 quantifies)."""
|
||
nout = normalize_target(output)
|
||
forms = [normalize_target(f) for f in decl_forms] or [normalize_target(base_dst)]
|
||
return any(f and contains_whole_word(nout, f) for f in forms)
|
||
|
||
|
||
def consistency(output: str, entries: dict) -> dict:
|
||
"""Consistency score over the glossary entries whose SRC is present in the chunk.
|
||
entries: {src: {"dst":..., "lemma_keys":[...], "hyphen_pat":..., "decl_forms":[...]}}.
|
||
Returns per-matcher rendered/missed sets + score. `present` filter is the caller's job
|
||
(pass only entries whose src fired in this chunk)."""
|
||
result = {"n": len(entries), "pymorphy": {}, "stored_decl": {}}
|
||
for name, matcher_fn in (("pymorphy", _score_pymorphy), ("stored_decl", _score_stored)):
|
||
rendered = [s for s, e in entries.items() if matcher_fn(output, e)]
|
||
result[name] = {"rendered": rendered, "missed": [s for s in entries if s not in rendered],
|
||
"score": round(len(rendered) / max(1, len(entries)), 3)}
|
||
return result
|
||
|
||
|
||
def _score_pymorphy(output: str, e: dict) -> bool:
|
||
return rendered_pymorphy(output, e.get("lemma_keys", []), e.get("hyphen_pat"))
|
||
|
||
|
||
def _score_stored(output: str, e: dict) -> bool:
|
||
return rendered_stored_decl(output, e.get("decl_forms", []), e["dst"])
|
||
|
||
|
||
if __name__ == "__main__": # smoke: inflected forms must NOT false-flag; Latin-boundary parity
|
||
out = "Бородатого Вана встретили в Вэйчжуане; Дэна и сюцая тоже."
|
||
ents = {
|
||
"王胡": {"dst": "Бородатый Ван", "lemma_keys": ["ван"], "decl_forms": ["Бородатого Вана", "Бородатый Ван"]},
|
||
"未庄": {"dst": "Вэйчжуан", "lemma_keys": ["вэйчжуан"], "decl_forms": ["Вэйчжуане", "Вэйчжуан"]},
|
||
"小D": {"dst": "Маленький Дэн", "lemma_keys": ["дэн"], "decl_forms": ["Дэна", "Дэн"]},
|
||
"秀才": {"dst": "сюцай", "lemma_keys": ["сюцай"], "decl_forms": ["сюцая", "сюцай"]},
|
||
}
|
||
import json
|
||
print(json.dumps(consistency(out, ents), ensure_ascii=False, indent=2))
|
||
# boundary + normalization parity asserts (Go containsWholeWord / normalizeTargetForm)
|
||
assert contains_whole_word(normalize_target("Ван ушёл"), normalize_target("Ван"))
|
||
assert not contains_whole_word(normalize_target("караВАН ушёл"), normalize_target("Ван"))
|
||
assert contains_whole_word(normalize_target("А‑кью"), normalize_target("А-кью")) # NB-hyphen folds
|
||
assert contains_whole_word(normalize_target("Вана"), normalize_target("вана")) # ZWSP stripped
|
||
assert not contains_whole_word(normalize_target("Ivan"), normalize_target("van")) # Latin boundary
|
||
# apostrophe-variant fold parity (package-2 Task 1b): typographic ≡ ASCII on the target side
|
||
assert normalize_target("д’Артаньян") == normalize_target("д'Артаньян")
|
||
assert contains_whole_word(normalize_target("вот д’Артаньян идёт"), normalize_target("д'Артаньян"))
|
||
print("declmetric parity asserts: OK")
|