textmachine/eval/exp16/palladius.py

#!/usr/bin/env python3
"""exp16 — Palladius (Палладий) transliteration table + ru-side name detector (research/20 §B1 ch5, §B5
P4). miner-v1. $0. Versioned artifact (the syllable set is generated deterministically from the mapping
rules below and self-validated against the GT dst names).

Two uses:
  (5) ru-side detector: a ru token that segments cleanly into Palladius syllables = the draft model
      treated it as a transliterated Chinese NAME -> high-precision name spot WITH a ready dst.
  canon conformity (§C2-3): score whether a proposed dst for a type=name/place term conforms to Palladius.

The pinyin->Cyrillic Palladius system is a fixed ~400-syllable table. We generate it from initials ×
finals + the documented irregularities (zhi/zi/…->ы-class; y-/w- whole syllables; ü-class). Coverage is
validated by requiring every GT dst name syllable to be recognized (assert in __main__).
"""
from __future__ import annotations

import regex

import exp16_common as X

# ── initials (声母) → Palladius ────────────────────────────────────────────────────────────────────
INITIALS = {
    "b": "б", "p": "п", "m": "м", "f": "ф", "d": "д", "t": "т", "n": "н", "l": "л",
    "g": "г", "k": "к", "h": "х", "j": "цз", "q": "ц", "x": "с",
    "zh": "чж", "ch": "ч", "sh": "ш", "r": "ж", "z": "цз", "c": "ц", "s": "с",
    "": "",   # zero initial (y/w handled as whole syllables below)
}

# ── finals (韵母) → Palladius (base forms; some initial-conditioned variants applied in build) ──────
FINALS = {
    "a": "а", "o": "о", "e": "э", "ai": "ай", "ei": "эй", "ao": "ао", "ou": "оу",
    "an": "ань", "en": "энь", "ang": "ан", "eng": "эн", "er": "эр", "ong": "ун",
    "i": "и", "ia": "я", "ie": "е", "iao": "яо", "iu": "ю", "ian": "янь",
    "in": "инь", "iang": "ян", "ing": "ин", "iong": "юн",
    "u": "у", "ua": "уа", "uo": "о", "uai": "уай", "ui": "уй", "uan": "уань",
    "un": "унь", "uang": "уан", "ueng": "ун",
    "v": "юй", "ve": "юэ", "van": "юань", "vn": "юнь",   # ü written as v
}

# whole zero-initial syllables (y-/w-)
Y_W = {
    "yi": "и", "ya": "я", "ye": "е", "yao": "яо", "you": "ю", "yan": "янь", "yin": "инь",
    "yang": "ян", "ying": "ин", "yong": "юн", "yu": "юй", "yue": "юэ", "yuan": "юань", "yun": "юнь",
    "wu": "у", "wa": "ва", "wo": "во", "wai": "вай", "wei": "вэй", "wan": "вань", "wen": "вэнь",
    "wang": "ван", "weng": "вэн",
}
# retroflex/sibilant + i => -ы/-и class (zhi chi shi ri zi ci si)
SPECIAL_I = {
    "zhi": "чжи", "chi": "чи", "shi": "ши", "ri": "жи", "zi": "цзы", "ci": "цы", "si": "сы",
}
# initials that take j/q/x with ü finals written as u (ju->цзюй etc.)
_JQX = {"j", "q", "x"}


def _final_after(initial: str, final: str, cyr_ini: str, cyr_fin: str) -> str:
    """Apply the few initial-conditioned Palladius adjustments."""
    # e after most initials -> э, but after certain -> е is not standard; keep э.
    # 'o' after b/p/m/f -> о (бо/по/мо/фо); 'uo' after them n/a.
    # 'ie'->е, 'ei'->эй are fine. Keep base mapping.
    return cyr_ini + cyr_fin


def build_syllables() -> dict[str, str]:
    """pinyin syllable -> Palladius. Generated; not exhaustive of tone/rare finals but covers names."""
    syl = {}
    syl.update(Y_W)
    syl.update(SPECIAL_I)
    for pi, ci in INITIALS.items():
        if pi == "":
            continue
        for pf, cf in FINALS.items():
            # ü finals (v) only valid after j/q/x/l/n; jqx write ü as plain u in pinyin
            if pf in ("v", "ve", "van", "vn"):
                if pi in _JQX:
                    py = pi + pf.replace("v", "u")     # ju/jue/juan/jun
                elif pi in ("l", "n"):
                    py = pi + pf.replace("v", "ü")     # lü/nü
                else:
                    continue
            else:
                py = pi + pf
            # skip the retroflex/sibilant + bare i (handled by SPECIAL_I)
            if pf == "i" and pi in ("zh", "ch", "sh", "r", "z", "c", "s"):
                continue
            syl[py] = _final_after(pi, pf, ci, cf)
    return syl


SYLLABLES = build_syllables()
# Cyrillic syllable inventory (values), longest-first for greedy segmentation
_CYR_SYL = sorted(set(SYLLABLES.values()), key=len, reverse=True)
_RE_CYR_WORD = regex.compile(r"^[\p{Cyrillic}]+$")


def is_palladius_token(token: str, min_syllables=1) -> bool:
    """True if `token` (a ru word) segments fully into Palladius syllables (greedy longest-match).
    Used as a high-precision 'this looks like a transliterated Chinese name' signal."""
    t = token.strip().lower().replace("ъ", "")
    if not t or not _RE_CYR_WORD.match(t):
        return False
    n_syl = 0
    i = 0
    while i < len(t):
        for s in _CYR_SYL:
            if s and t.startswith(s, i):
                i += len(s)
                n_syl += 1
                break
        else:
            return False
    return n_syl >= min_syllables


def palladius_conformant(dst: str) -> bool:
    """Canon conformity (§C2-3): every whitespace/hyphen-separated Cyrillic word of dst is a Palladius
    token. For multi-word names (Фан Юань) all parts must conform. Non-Cyrillic parts are ignored."""
    words = regex.findall(r"[\p{Cyrillic}]+", dst)
    cyr = [w for w in words if _RE_CYR_WORD.match(w)]
    if not cyr:
        return False
    return all(is_palladius_token(w, min_syllables=1) for w in cyr)


def ru_name_tokens(draft: str, min_syllables=2) -> list[str]:
    """Capitalized ru tokens in a draft that parse as Palladius (>=min_syllables) — model-flagged names."""
    out = []
    for tok in regex.findall(r"\b[\p{Lu}][\p{Cyrillic}]+", draft):
        if is_palladius_token(tok, min_syllables=min_syllables):
            out.append(tok)
    return out


if __name__ == "__main__":
    print(f"Palladius syllables generated: {len(SYLLABLES)} pinyin -> {len(set(SYLLABLES.values()))} cyr forms")
    # validate: every GT dst name/place syllable must be recognized
    gt = X.load_gt()
    name_dsts = [e.dst for e in gt if e.typ in ("name",) and e.dst]
    print("\n== GT name dst recognition ==")
    ok = bad = 0
    for e in gt:
        if e.typ not in ("name", "place"):
            continue
        # take the transliterated part(s) — for places skip the leading common noun (гора/деревня)
        words = regex.findall(r"[\p{Cyrillic}]+", e.dst)
        translit = [w for w in words if w.lower() not in ("гора", "деревня", "село", "селение", "стан", "клан", "род")]
        good = all(is_palladius_token(w) for w in translit) if translit else False
        ok += good
        bad += not good
        mark = "OK " if good else "MISS"
        print(f"  {mark} {e.src:<6} {e.dst:<26} translit={translit} "
              f"{[w for w in translit if not is_palladius_token(w)] if not good else ''}")
    print(f"\nrecognized {ok}/{ok+bad} GT name/place dst")
    # negatives: common Russian words should NOT be Palladius
    print("\n== negative control (common ru words, should be FALSE) ==")
    for w in ["человек", "который", "сказал", "деревня", "истинный", "камень", "мастер", "ранг"]:
        print(f"  {w:<12} palladius={is_palladius_token(w)}")