textmachine/backend/internal/miner/miner_palladius.go

package miner

import (
	"sort"
	"strings"
	"unicode"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/text"
)

// miner_palladius.go: the Palladius (Палладий) transliteration syllable GENERATOR + ru-side name-shape
// detector (WS3 — a Go port of exp16 palladius.py). A ru token that segments cleanly into Palladius
// syllables looks like a transliterated Chinese NAME. The pinyin→Cyrillic TABLE is DATA (a pair pack,
// configs/langpacks/<pair>/palladius.txt, loaded via internal/lang); this file keeps only the ALGORITHM —
// the syllable-inventory generator (over the table) and the greedy segmenter (D39.15: data as files,
// algorithm in code).
//
// In the ratified DEFAULT-B miner the ru-side name CONFIRMATION sub-channel is OFF (it also needs the
// pymorphy3 is_name_lemma gate to block the «найти»=най+ти false-positive class), so isPalladiusToken is
// not wired into the default-B detector pipeline. It is kept as a deterministic, tested artifact — the
// name-shape signal a future faithful-A variant (with a Go morphology backend) would consume, and the
// pattern P4 transliteration-by-pair seam (§B5).

// buildPalladiusCyrSyllables derives the distinct Cyrillic syllable set, longest-first, for greedy
// segmentation (palladius._CYR_SYL) from the pack's pinyin→Cyrillic table. Ties in length never both match
// a position (distinct strings), so the order among equal-length forms is immaterial — sorted (len desc,
// then value) for a stable artifact. The table (initials/finals/Y_W/SPECIAL_I) AND the phonotactic
// constraints (retroflex / ü-finals / their legal initials) are langpack DATA (pair-14).
func buildPalladiusCyrSyllables(p *lang.Pack) []string {
	set := map[string]bool{}
	pal := p.Palladius
	for _, v := range pal.YW {
		set[v] = true
	}
	for _, v := range pal.SpecialI {
		set[v] = true
	}
	for pi, ci := range pal.Initials {
		for pf, cf := range pal.Finals {
			// ü finals (written v) are valid only after a vfinal_initial (pinyin writes ü as plain u).
			if pal.VFinal[pf] && !pal.VFinalInitial[pi] {
				continue
			}
			// skip the retroflex/sibilant + bare i (handled by SPECIAL_I).
			if pf == "i" && pal.Retroflex[pi] {
				continue
			}
			set[ci+cf] = true
		}
	}
	out := make([]string, 0, len(set))
	for s := range set {
		if s != "" {
			out = append(out, s)
		}
	}
	sort.Slice(out, func(i, j int) bool {
		if len([]rune(out[i])) != len([]rune(out[j])) {
			return len([]rune(out[i])) > len([]rune(out[j]))
		}
		return out[i] < out[j]
	})
	return out
}

// isPalladiusToken reports whether a ru word segments fully into Palladius syllables (greedy longest-
// match, palladius.is_palladius_token): a high-precision "this looks like a transliterated Chinese name"
// signal. ъ is dropped before segmentation. minSyllables is the minimum syllable count. syllables is the
// longest-first inventory from buildPalladiusCyrSyllables(pack).
func isPalladiusToken(token string, minSyllables int, syllables []string) bool {
	t := strings.ToLower(strings.TrimSpace(token))
	t = strings.ReplaceAll(t, "ъ", "")
	if t == "" || !allCyrillic(t) {
		return false
	}
	tr := []rune(t)
	nSyl := 0
	for i := 0; i < len(tr); {
		matched := false
		for _, s := range syllables {
			sr := []rune(s)
			if i+len(sr) <= len(tr) && text.RunesEqual(tr[i:i+len(sr)], sr) {
				i += len(sr)
				nSyl++
				matched = true
				break
			}
		}
		if !matched {
			return false
		}
	}
	return nSyl >= minSyllables
}

// PalladiusConformance scores how much of a rendering looks like a transliteration under the pair's
// table: the share of its target-script word tokens that segment cleanly into Palladius syllables, in
// [0,1]. It is the "conformance" factor of the §C2-3 consolidation formula (pack-20) — «Фан Юань» for
// 方源 scores 1.0, a semantic rendering scores 0.
//
// It is exported (and only here) so the terminologist consumes the SAME frozen segmenter the miner ships
// rather than a second copy: a divergent second implementation of a transliteration check is precisely the
// drift the langpack move was done to prevent. A nil pack, or a rendering with no word tokens, scores 0 —
// the factor then goes neutral at the caller, never negative.
func PalladiusConformance(dst string, pack *lang.Pack) float64 {
	if pack == nil || strings.TrimSpace(dst) == "" {
		return 0
	}
	syl := buildPalladiusCyrSyllables(pack)
	if len(syl) == 0 {
		return 0
	}
	total, ok := 0, 0
	for _, tok := range strings.FieldsFunc(dst, func(r rune) bool { return !unicode.IsLetter(r) }) {
		total++
		if isPalladiusToken(tok, 1, syl) {
			ok++
		}
	}
	if total == 0 {
		return 0
	}
	return float64(ok) / float64(total)
}

// allCyrillic reports whether every rune of s is Cyrillic (palladius._RE_CYR_WORD).
func allCyrillic(s string) bool {
	if s == "" {
		return false
	}
	for _, r := range s {
		if !unicode.Is(unicode.Cyrillic, r) {
			return false
		}
	}
	return true
}