#!/usr/bin/env python3 """Decl-aware (склонение-осознанная) consistency metric for the Ф2.5 memory-eval. WHY decl-aware is MANDATORY (research/14 §2, quantified; backend memory_e1_test.go): a naive word-boundary regexp on the base form gives 18-67 pct FALSE flags on inflected Russian — forms rendered CORRECTLY ("Вэйчжуане", "Бородатого Вана", "Дэна", "сюцая") that a boundary regexp on the lemma misses. Without decl-awareness the eval is noise and the bank-vs-long-context verdict is untrustworthy. This module is the eval-side analogue of the Go post-check (backend/internal/pipeline/mempostcheck.go): whole-word, decl-tolerant, letter-boundary. Two matchers, both reusable: - pymorphy3 lemma-set: a name is "rendered" if its canonical LEMMA(s) appear among the lemmatized words of the output (folds Вэйчжуане→вэйчжуан, Вана→ван). Mirrors the Go post-check's intent with an OOV-tolerant morphological analyzer (pymorphy3 lemmatizes even unknown translit names). - hyphen-translit regex fallback: for hyphenated translit names (А-кью, У-ма) pymorphy3 splits on the hyphen, so a regex like "А[-space]?кью" is the correct decl-tolerant matcher. Also exposes GO-STYLE decl-form whole-word matching (given stored decl forms), so the eval can compare "stored-decl completeness" (Go's real mechanism) against pymorphy fallback — the E1 measurement the handoff asks the polygon to validate on real books. D13.5 SYNC WITH GO (07-strategic-review §4.5; POLYGON handoff Task 1e-tail): normalize_target and containsWholeWord are now a FAITHFUL mirror of memnorm.go normalizeTargetForm / mempostcheck.go containsWholeWord — NFC → dash/hyphen-variant fold → default-ignorable strip (Cf + variation selectors) → whitespace-collapse → lower → ё→е, and a boundary test on the Unicode IsLetter equivalent (category L*) rather than Cyrillic-only. Before this, a translit name adjacent to a Latin letter or split by a zero-width joiner false-missed/false-hit vs Go. """ from __future__ import annotations import re import unicodedata import pymorphy3 _MORPH = pymorphy3.MorphAnalyzer() _WORD_RE = re.compile(r"[А-Яа-яЁё]+(?:-[А-Яа-яЁё]+)?") # Dash/hyphen/minus variants that fold to ASCII '-' (mirror of Go isDashVariant): hyphen, # non-breaking hyphen, figure dash, en/em dash, horizontal bar, math minus, soft hyphen. _DASH_VARIANTS = frozenset("‐‑‒–—―−­") def _is_ignorable(c: str) -> bool: """Mirror of Go isIgnorableForMatch: default-ignorable / format code points (unicode.Cf: ZWSP/ZWNJ/ZWJ/WJ/BOM/bidi) plus the variation selectors (VS1-16 U+FE00..FE0F and the ideographic supplement U+E0100..E01EF). Dropped before matching so a stored «вана» is not a false MISS against «вана». Soft hyphen (also Cf) is folded to '-' BEFORE this.""" o = ord(c) return unicodedata.category(c) == "Cf" or (0xFE00 <= o <= 0xFE0F) or (0xE0100 <= o <= 0xE01EF) def _is_letter(c: str) -> bool: """unicode.IsLetter equivalent: Unicode general category L* (Lu/Ll/Lt/Lm/Lo). Used for the whole-word boundary so a Latin-letter neighbour bounds a translit name too (Go parity; the old Cyrillic-only [А-Яа-яЁё] test missed Latin boundaries).""" return unicodedata.category(c)[0] == "L" def normalize_target(s: str) -> str: """NFC → dash/hyphen-variant fold → drop default-ignorables → whitespace-collapse → lower → ё→е. Symmetric mirror of Go normalizeTargetForm (memnorm.go), applied to BOTH the stored decl forms AND the model output so a correctly-rendered term is NEVER a false MISS.""" s = unicodedata.normalize("NFC", s) out: list[str] = [] prev_space = False for c in s: if c in _DASH_VARIANTS: c = "-" # soft hyphen folds here, so it is kept (not dropped below) elif _is_ignorable(c): continue if c.isspace(): if not prev_space: out.append(" ") prev_space = True continue prev_space = False c = c.lower() if c == "ё": c = "е" out.append(c) return "".join(out).strip() def lemmas(text: str) -> set[str]: """Lemma set of the Cyrillic words in text (ё→е folded).""" out = set() for w in _WORD_RE.findall(text): w = w.replace("ё", "е") out.add(_MORPH.parse(w)[0].normal_form) # also add the raw lowered form (for hyphenated/translit that pymorphy keeps whole) out.add(w.lower()) return out def rendered_pymorphy(output: str, lemma_keys: list[str], hyphen_pat: str | None = None) -> bool: """A term is rendered if ALL its canonical lemma keys appear in the output's lemma set. hyphen_pat: a regex for hyphenated translit (А-кью) matched directly on the output.""" if hyphen_pat: return bool(re.search(hyphen_pat, output, re.I)) lem = lemmas(output) return all(k.replace("ё", "е").lower() in lem for k in lemma_keys) def contains_whole_word(text_norm: str, form_norm: str) -> bool: """Go-style whole-word (letter-boundary) containment (mirror of Go containsWholeWord). Boundary = Unicode IsLetter (category L*) on both ends. Both args must be normalize_target()'d.""" if not form_norm: return False n = len(form_norm) i = text_norm.find(form_norm) while i != -1: left_ok = i == 0 or not _is_letter(text_norm[i - 1]) j = i + n right_ok = j == len(text_norm) or not _is_letter(text_norm[j]) if left_ok and right_ok: return True i = text_norm.find(form_norm, i + 1) return False def rendered_stored_decl(output: str, decl_forms: list[str], base_dst: str) -> bool: """Go post-check's REAL mechanism: whole-word match against STORED decl forms (fallback to the base dst when decl is empty — the naive case E1 quantifies).""" nout = normalize_target(output) forms = [normalize_target(f) for f in decl_forms] or [normalize_target(base_dst)] return any(f and contains_whole_word(nout, f) for f in forms) def consistency(output: str, entries: dict) -> dict: """Consistency score over the glossary entries whose SRC is present in the chunk. entries: {src: {"dst":..., "lemma_keys":[...], "hyphen_pat":..., "decl_forms":[...]}}. Returns per-matcher rendered/missed sets + score. `present` filter is the caller's job (pass only entries whose src fired in this chunk).""" result = {"n": len(entries), "pymorphy": {}, "stored_decl": {}} for name, matcher_fn in (("pymorphy", _score_pymorphy), ("stored_decl", _score_stored)): rendered = [s for s, e in entries.items() if matcher_fn(output, e)] result[name] = {"rendered": rendered, "missed": [s for s in entries if s not in rendered], "score": round(len(rendered) / max(1, len(entries)), 3)} return result def _score_pymorphy(output: str, e: dict) -> bool: return rendered_pymorphy(output, e.get("lemma_keys", []), e.get("hyphen_pat")) def _score_stored(output: str, e: dict) -> bool: return rendered_stored_decl(output, e.get("decl_forms", []), e["dst"]) if __name__ == "__main__": # smoke: inflected forms must NOT false-flag; Latin-boundary parity out = "Бородатого Вана встретили в Вэйчжуане; Дэна и сюцая тоже." ents = { "王胡": {"dst": "Бородатый Ван", "lemma_keys": ["ван"], "decl_forms": ["Бородатого Вана", "Бородатый Ван"]}, "未庄": {"dst": "Вэйчжуан", "lemma_keys": ["вэйчжуан"], "decl_forms": ["Вэйчжуане", "Вэйчжуан"]}, "小D": {"dst": "Маленький Дэн", "lemma_keys": ["дэн"], "decl_forms": ["Дэна", "Дэн"]}, "秀才": {"dst": "сюцай", "lemma_keys": ["сюцай"], "decl_forms": ["сюцая", "сюцай"]}, } import json print(json.dumps(consistency(out, ents), ensure_ascii=False, indent=2)) # boundary + normalization parity asserts (Go containsWholeWord / normalizeTargetForm) assert contains_whole_word(normalize_target("Ван ушёл"), normalize_target("Ван")) assert not contains_whole_word(normalize_target("караВАН ушёл"), normalize_target("Ван")) assert contains_whole_word(normalize_target("А‑кью"), normalize_target("А-кью")) # NB-hyphen folds assert contains_whole_word(normalize_target("Ва​на"), normalize_target("вана")) # ZWSP stripped assert not contains_whole_word(normalize_target("Ivan"), normalize_target("van")) # Latin boundary print("declmetric parity asserts: OK")