111 lines
5.6 KiB
Python
111 lines
5.6 KiB
Python
#!/usr/bin/env python3
|
||
"""Decl-aware (склонение-осознанная) consistency metric for the Ф2.5 memory-eval.
|
||
|
||
WHY decl-aware is MANDATORY (research/14 §2, quantified; backend memory_e1_test.go):
|
||
a naive word-boundary regexp on the base form gives 18-67 pct FALSE flags on inflected Russian — forms
|
||
rendered CORRECTLY ("Вэйчжуане", "Бородатого Вана", "Дэна", "сюцая") that a boundary regexp
|
||
on the lemma misses. Without decl-awareness the eval is noise and the bank-vs-long-context
|
||
verdict is untrustworthy. This module is the eval-side analogue of the Go post-check
|
||
(backend/internal/pipeline/mempostcheck.go): whole-word, decl-tolerant, letter-boundary.
|
||
|
||
Two matchers, both reusable:
|
||
- pymorphy3 lemma-set: a name is "rendered" if its canonical LEMMA(s) appear among the
|
||
lemmatized words of the output (folds Вэйчжуане→вэйчжуан, Вана→ван). Mirrors the Go
|
||
post-check's intent with an OOV-tolerant morphological analyzer (pymorphy3 lemmatizes
|
||
even unknown translit names).
|
||
- hyphen-translit regex fallback: for hyphenated translit names (А-кью, У-ма) pymorphy3
|
||
splits on the hyphen, so a regex like "А[-space]?кью" is the correct decl-tolerant matcher.
|
||
|
||
Also exposes GO-STYLE decl-form whole-word matching (given stored decl forms), so the eval
|
||
can compare "stored-decl completeness" (Go's real mechanism) against pymorphy fallback —
|
||
the E1 measurement the handoff asks the polygon to validate on real books.
|
||
"""
|
||
from __future__ import annotations
|
||
import re
|
||
import pymorphy3
|
||
|
||
_MORPH = pymorphy3.MorphAnalyzer()
|
||
_WORD_RE = re.compile(r"[А-Яа-яЁё]+(?:-[А-Яа-яЁё]+)?")
|
||
_CYR = re.compile(r"[А-Яа-яЁё]")
|
||
|
||
|
||
def normalize_target(s: str) -> str:
|
||
"""ё→е fold + lower + whitespace collapse (mirror of Go normalizeTargetForm, target side)."""
|
||
return re.sub(r"\s+", " ", s).lower().replace("ё", "е").strip()
|
||
|
||
|
||
def lemmas(text: str) -> set[str]:
|
||
"""Lemma set of the Cyrillic words in text (ё→е folded)."""
|
||
out = set()
|
||
for w in _WORD_RE.findall(text):
|
||
w = w.replace("ё", "е")
|
||
out.add(_MORPH.parse(w)[0].normal_form)
|
||
# also add the raw lowered form (for hyphenated/translit that pymorphy keeps whole)
|
||
out.add(w.lower())
|
||
return out
|
||
|
||
|
||
def rendered_pymorphy(output: str, lemma_keys: list[str], hyphen_pat: str | None = None) -> bool:
|
||
"""A term is rendered if ALL its canonical lemma keys appear in the output's lemma set.
|
||
hyphen_pat: a regex for hyphenated translit (А-кью) matched directly on the output."""
|
||
if hyphen_pat:
|
||
return bool(re.search(hyphen_pat, output, re.I))
|
||
lem = lemmas(output)
|
||
return all(k.replace("ё", "е").lower() in lem for k in lemma_keys)
|
||
|
||
|
||
def contains_whole_word(text_norm: str, form_norm: str) -> bool:
|
||
"""Go-style whole-word (letter-boundary) containment (mirror of Go containsWholeWord).
|
||
Both args must be normalize_target()'d."""
|
||
if not form_norm:
|
||
return False
|
||
i = text_norm.find(form_norm)
|
||
while i != -1:
|
||
left_ok = i == 0 or not _CYR.match(text_norm[i - 1])
|
||
j = i + len(form_norm)
|
||
right_ok = j == len(text_norm) or not _CYR.match(text_norm[j])
|
||
if left_ok and right_ok:
|
||
return True
|
||
i = text_norm.find(form_norm, i + 1)
|
||
return False
|
||
|
||
|
||
def rendered_stored_decl(output: str, decl_forms: list[str], base_dst: str) -> bool:
|
||
"""Go post-check's REAL mechanism: whole-word match against STORED decl forms
|
||
(fallback to the base dst when decl is empty — the naive case E1 quantifies)."""
|
||
nout = normalize_target(output)
|
||
forms = [normalize_target(f) for f in decl_forms] or [normalize_target(base_dst)]
|
||
return any(f and contains_whole_word(nout, f) for f in forms)
|
||
|
||
|
||
def consistency(output: str, entries: dict) -> dict:
|
||
"""Consistency score over the glossary entries whose SRC is present in the chunk.
|
||
entries: {src: {"dst":..., "lemma_keys":[...], "hyphen_pat":..., "decl_forms":[...]}}.
|
||
Returns per-matcher rendered/missed sets + score. `present` filter is the caller's job
|
||
(pass only entries whose src fired in this chunk)."""
|
||
result = {"n": len(entries), "pymorphy": {}, "stored_decl": {}}
|
||
for name, matcher_fn in (("pymorphy", _score_pymorphy), ("stored_decl", _score_stored)):
|
||
rendered = [s for s, e in entries.items() if matcher_fn(output, e)]
|
||
result[name] = {"rendered": rendered, "missed": [s for s in entries if s not in rendered],
|
||
"score": round(len(rendered) / max(1, len(entries)), 3)}
|
||
return result
|
||
|
||
|
||
def _score_pymorphy(output: str, e: dict) -> bool:
|
||
return rendered_pymorphy(output, e.get("lemma_keys", []), e.get("hyphen_pat"))
|
||
|
||
|
||
def _score_stored(output: str, e: dict) -> bool:
|
||
return rendered_stored_decl(output, e.get("decl_forms", []), e["dst"])
|
||
|
||
|
||
if __name__ == "__main__": # smoke: inflected forms must NOT false-flag
|
||
out = "Бородатого Вана встретили в Вэйчжуане; Дэна и сюцая тоже."
|
||
ents = {
|
||
"王胡": {"dst": "Бородатый Ван", "lemma_keys": ["ван"], "decl_forms": ["Бородатого Вана", "Бородатый Ван"]},
|
||
"未庄": {"dst": "Вэйчжуан", "lemma_keys": ["вэйчжуан"], "decl_forms": ["Вэйчжуане", "Вэйчжуан"]},
|
||
"小D": {"dst": "Маленький Дэн", "lemma_keys": ["дэн"], "decl_forms": ["Дэна", "Дэн"]},
|
||
"秀才": {"dst": "сюцай", "lemma_keys": ["сюцай"], "decl_forms": ["сюцая", "сюцай"]},
|
||
}
|
||
import json
|
||
print(json.dumps(consistency(out, ents), ensure_ascii=False, indent=2))
|