#!/usr/bin/env python3
"""Decl-aware (склонение-осознанная) consistency metric for the Ф2.5 memory-eval.

WHY decl-aware is MANDATORY (research/14 §2, quantified; backend memory_e1_test.go):
a naive word-boundary regexp on the base form gives 18-67 pct FALSE flags on inflected Russian — forms
rendered CORRECTLY ("Вэйчжуане", "Бородатого Вана", "Дэна", "сюцая") that a boundary regexp
on the lemma misses. Without decl-awareness the eval is noise and the bank-vs-long-context
verdict is untrustworthy. This module is the eval-side analogue of the Go post-check
(backend/internal/pipeline/mempostcheck.go): whole-word, decl-tolerant, letter-boundary.

Two matchers, both reusable:
  - pymorphy3 lemma-set: a name is "rendered" if its canonical LEMMA(s) appear among the
    lemmatized words of the output (folds Вэйчжуане→вэйчжуан, Вана→ван). Mirrors the Go
    post-check's intent with an OOV-tolerant morphological analyzer (pymorphy3 lemmatizes
    even unknown translit names).
  - hyphen-translit regex fallback: for hyphenated translit names (А-кью, У-ма) pymorphy3
    splits on the hyphen, so a regex like "А[-space]?кью" is the correct decl-tolerant matcher.

Also exposes GO-STYLE decl-form whole-word matching (given stored decl forms), so the eval
can compare "stored-decl completeness" (Go's real mechanism) against pymorphy fallback —
the E1 measurement the handoff asks the polygon to validate on real books.

D13.5 SYNC WITH GO (07-strategic-review §4.5; POLYGON handoff Task 1e-tail + package-2 Task 1b):
normalize_target and containsWholeWord are now a FAITHFUL mirror of memnorm.go normalizeTargetForm /
mempostcheck.go containsWholeWord — NFC → dash/hyphen-variant fold → apostrophe-variant fold
(‘ ’ ʼ ＇ → ') → default-ignorable strip (Cf + variation selectors) → whitespace-collapse → lower
→ ё→е, and a boundary test on the Unicode IsLetter equivalent (category L*) rather than
Cyrillic-only. Before this, a translit name adjacent to a Latin letter or split by a zero-width
joiner false-missed/false-hit vs Go; the apostrophe fold (package-2 Task 1b) closes a false MISS
on д'Артаньян vs д’Артаньян — critical for the en→ru pilot arm.
"""
from __future__ import annotations
import re
import unicodedata
import pymorphy3

_MORPH = pymorphy3.MorphAnalyzer()
_WORD_RE = re.compile(r"[А-Яа-яЁё]+(?:-[А-Яа-яЁё]+)?")

# Dash/hyphen/minus variants that fold to ASCII '-' (mirror of Go isDashVariant): hyphen,
# non-breaking hyphen, figure dash, en/em dash, horizontal bar, math minus, soft hyphen.
_DASH_VARIANTS = frozenset("‐‑‒–—―−­")

# Typographic apostrophe/single-quote variants that fold to ASCII '\'' (mirror of Go
# isApostropheVariant): U+2018/2019 curly single quotes, U+02BC modifier apostrophe, U+FF07
# fullwidth apostrophe. Symmetric with the source side so д'Артаньян vs д’Артаньян is never a
# false post-check MISS (critical for the en→ru pilot arm).
_APOSTROPHE_VARIANTS = frozenset("‘’ʼ＇")


def _is_ignorable(c: str) -> bool:
    """Mirror of Go isIgnorableForMatch: default-ignorable / format code points (unicode.Cf:
    ZWSP/ZWNJ/ZWJ/WJ/BOM/bidi) plus the variation selectors (VS1-16 U+FE00..FE0F and the
    ideographic supplement U+E0100..E01EF). Dropped before matching so a stored «вана» is not
    a false MISS against «ва<WJ>на». Soft hyphen (also Cf) is folded to '-' BEFORE this."""
    o = ord(c)
    return unicodedata.category(c) == "Cf" or (0xFE00 <= o <= 0xFE0F) or (0xE0100 <= o <= 0xE01EF)


def _is_letter(c: str) -> bool:
    """unicode.IsLetter equivalent: Unicode general category L* (Lu/Ll/Lt/Lm/Lo). Used for
    the whole-word boundary so a Latin-letter neighbour bounds a translit name too (Go parity;
    the old Cyrillic-only [А-Яа-яЁё] test missed Latin boundaries)."""
    return unicodedata.category(c)[0] == "L"


def normalize_target(s: str) -> str:
    """NFC → dash/hyphen-variant fold → apostrophe-variant fold → drop default-ignorables →
    whitespace-collapse → lower → ё→е. Symmetric mirror of Go normalizeTargetForm (memnorm.go),
    applied to BOTH the stored decl forms AND the model output so a correctly-rendered term is
    NEVER a false MISS (the dash/apostrophe/ignorable folds are separate steps, matching Go)."""
    s = unicodedata.normalize("NFC", s)
    out: list[str] = []
    prev_space = False
    for c in s:
        if c in _DASH_VARIANTS:
            c = "-"                    # soft hyphen folds here, so it is kept (not dropped below)
        if c in _APOSTROPHE_VARIANTS:
            c = "'"                    # symmetric with the source side (д'Артаньян) — no false MISS
        if _is_ignorable(c):
            continue
        if c.isspace():
            if not prev_space:
                out.append(" ")
                prev_space = True
            continue
        prev_space = False
        if c == "İ":  # U+0130: Go unicode.ToLower→'i' (Python str.lower→'i̇'); match Go (parity-review #1)
            out.append("i")
            continue
        c = c.lower()
        if c == "ё":
            c = "е"
        out.append(c)
    return "".join(out).strip()


def lemmas(text: str) -> set[str]:
    """Lemma set of the Cyrillic words in text (ё→е folded)."""
    out = set()
    for w in _WORD_RE.findall(text):
        w = w.replace("ё", "е")
        out.add(_MORPH.parse(w)[0].normal_form)
        # also add the raw lowered form (for hyphenated/translit that pymorphy keeps whole)
        out.add(w.lower())
    return out


def rendered_pymorphy(output: str, lemma_keys: list[str], hyphen_pat: str | None = None) -> bool:
    """A term is rendered if ALL its canonical lemma keys appear in the output's lemma set.
    hyphen_pat: a regex for hyphenated translit (А-кью) matched directly on the output."""
    if hyphen_pat:
        return bool(re.search(hyphen_pat, output, re.I))
    lem = lemmas(output)
    return all(k.replace("ё", "е").lower() in lem for k in lemma_keys)


def contains_whole_word(text_norm: str, form_norm: str) -> bool:
    """Go-style whole-word (letter-boundary) containment (mirror of Go containsWholeWord).
    Boundary = Unicode IsLetter (category L*) on both ends. Both args must be normalize_target()'d."""
    if not form_norm:
        return False
    n = len(form_norm)
    i = text_norm.find(form_norm)
    while i != -1:
        left_ok = i == 0 or not _is_letter(text_norm[i - 1])
        j = i + n
        right_ok = j == len(text_norm) or not _is_letter(text_norm[j])
        if left_ok and right_ok:
            return True
        i = text_norm.find(form_norm, i + 1)
    return False


def rendered_stored_decl(output: str, decl_forms: list[str], base_dst: str) -> bool:
    """Go post-check's REAL mechanism: whole-word match against STORED decl forms
    (fallback to the base dst when decl is empty — the naive case E1 quantifies)."""
    nout = normalize_target(output)
    forms = [normalize_target(f) for f in decl_forms] or [normalize_target(base_dst)]
    return any(f and contains_whole_word(nout, f) for f in forms)


def consistency(output: str, entries: dict) -> dict:
    """Consistency score over the glossary entries whose SRC is present in the chunk.
    entries: {src: {"dst":..., "lemma_keys":[...], "hyphen_pat":..., "decl_forms":[...]}}.
    Returns per-matcher rendered/missed sets + score. `present` filter is the caller's job
    (pass only entries whose src fired in this chunk)."""
    result = {"n": len(entries), "pymorphy": {}, "stored_decl": {}}
    for name, matcher_fn in (("pymorphy", _score_pymorphy), ("stored_decl", _score_stored)):
        rendered = [s for s, e in entries.items() if matcher_fn(output, e)]
        result[name] = {"rendered": rendered, "missed": [s for s in entries if s not in rendered],
                        "score": round(len(rendered) / max(1, len(entries)), 3)}
    return result


def _score_pymorphy(output: str, e: dict) -> bool:
    return rendered_pymorphy(output, e.get("lemma_keys", []), e.get("hyphen_pat"))


def _score_stored(output: str, e: dict) -> bool:
    return rendered_stored_decl(output, e.get("decl_forms", []), e["dst"])


if __name__ == "__main__":  # smoke: inflected forms must NOT false-flag; Latin-boundary parity
    out = "Бородатого Вана встретили в Вэйчжуане; Дэна и сюцая тоже."
    ents = {
        "王胡": {"dst": "Бородатый Ван", "lemma_keys": ["ван"], "decl_forms": ["Бородатого Вана", "Бородатый Ван"]},
        "未庄": {"dst": "Вэйчжуан", "lemma_keys": ["вэйчжуан"], "decl_forms": ["Вэйчжуане", "Вэйчжуан"]},
        "小D": {"dst": "Маленький Дэн", "lemma_keys": ["дэн"], "decl_forms": ["Дэна", "Дэн"]},
        "秀才": {"dst": "сюцай", "lemma_keys": ["сюцай"], "decl_forms": ["сюцая", "сюцай"]},
    }
    import json
    print(json.dumps(consistency(out, ents), ensure_ascii=False, indent=2))
    # boundary + normalization parity asserts (Go containsWholeWord / normalizeTargetForm)
    assert contains_whole_word(normalize_target("Ван ушёл"), normalize_target("Ван"))
    assert not contains_whole_word(normalize_target("караВАН ушёл"), normalize_target("Ван"))
    assert contains_whole_word(normalize_target("А‑кью"), normalize_target("А-кью"))  # NB-hyphen folds
    assert contains_whole_word(normalize_target("Ва​на"), normalize_target("вана"))   # ZWSP stripped
    assert not contains_whole_word(normalize_target("Ivan"), normalize_target("van"))       # Latin boundary
    # apostrophe-variant fold parity (package-2 Task 1b): typographic ≡ ASCII on the target side
    assert normalize_target("д’Артаньян") == normalize_target("д'Артаньян")
    assert contains_whole_word(normalize_target("вот д’Артаньян идёт"), normalize_target("д'Артаньян"))
    print("declmetric parity asserts: OK")
