package pipeline import ( "sort" "strings" "unicode" "textmachine/backend/internal/lang" ) // miner_palladius.go: the Palladius (Палладий) transliteration syllable GENERATOR + ru-side name-shape // detector (WS3 — a Go port of exp16 palladius.py). A ru token that segments cleanly into Palladius // syllables looks like a transliterated Chinese NAME. The pinyin→Cyrillic TABLE is DATA (a pair pack, // configs/langpacks//palladius.txt, loaded via internal/lang); this file keeps only the ALGORITHM — // the syllable-inventory generator (over the table) and the greedy segmenter (D39.15: data as files, // algorithm in code). // // In the ratified DEFAULT-B miner the ru-side name CONFIRMATION sub-channel is OFF (it also needs the // pymorphy3 is_name_lemma gate to block the «найти»=най+ти false-positive class), so isPalladiusToken is // not wired into the default-B detector pipeline. It is kept as a deterministic, tested artifact — the // name-shape signal a future faithful-A variant (with a Go morphology backend) would consume, and the // pattern P4 transliteration-by-pair seam (§B5). var palladiusJQX = map[string]bool{"j": true, "q": true, "x": true} // buildPalladiusCyrSyllables derives the distinct Cyrillic syllable set, longest-first, for greedy // segmentation (palladius._CYR_SYL) from the pack's pinyin→Cyrillic table. Ties in length never both match // a position (distinct strings), so the order among equal-length forms is immaterial — sorted (len desc, // then value) for a stable artifact. The table (initials/finals/Y_W/SPECIAL_I) is langpack DATA. func buildPalladiusCyrSyllables(p *lang.Pack) []string { set := map[string]bool{} for _, v := range p.PalladiusYW { set[v] = true } for _, v := range p.PalladiusSpecialI { set[v] = true } retroflex := map[string]bool{"zh": true, "ch": true, "sh": true, "r": true, "z": true, "c": true, "s": true} for pi, ci := range p.PalladiusInitials { for pf, cf := range p.PalladiusFinals { // ü finals (written v) are valid only after j/q/x (pinyin writes ü as plain u) or l/n. switch pf { case "v", "ve", "van", "vn": if !palladiusJQX[pi] && pi != "l" && pi != "n" { continue } } // skip the retroflex/sibilant + bare i (handled by SPECIAL_I). if pf == "i" && retroflex[pi] { continue } set[ci+cf] = true } } out := make([]string, 0, len(set)) for s := range set { if s != "" { out = append(out, s) } } sort.Slice(out, func(i, j int) bool { if len([]rune(out[i])) != len([]rune(out[j])) { return len([]rune(out[i])) > len([]rune(out[j])) } return out[i] < out[j] }) return out } // isPalladiusToken reports whether a ru word segments fully into Palladius syllables (greedy longest- // match, palladius.is_palladius_token): a high-precision "this looks like a transliterated Chinese name" // signal. ъ is dropped before segmentation. minSyllables is the minimum syllable count. syllables is the // longest-first inventory from buildPalladiusCyrSyllables(pack). func isPalladiusToken(token string, minSyllables int, syllables []string) bool { t := strings.ToLower(strings.TrimSpace(token)) t = strings.ReplaceAll(t, "ъ", "") if t == "" || !allCyrillic(t) { return false } tr := []rune(t) nSyl := 0 for i := 0; i < len(tr); { matched := false for _, s := range syllables { sr := []rune(s) if i+len(sr) <= len(tr) && runesEqual(tr[i:i+len(sr)], sr) { i += len(sr) nSyl++ matched = true break } } if !matched { return false } } return nSyl >= minSyllables } // allCyrillic reports whether every rune of s is Cyrillic (palladius._RE_CYR_WORD). func allCyrillic(s string) bool { if s == "" { return false } for _, r := range s { if !unicode.Is(unicode.Cyrillic, r) { return false } } return true }