138 lines
5.1 KiB
Go
138 lines
5.1 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"sort"
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
// miner_palladius.go: the Palladius (Палладий) transliteration syllable table + ru-side name-shape
|
||
// detector (WS3 — a Go port of exp16 palladius.py). A ru token that segments cleanly into Palladius
|
||
// syllables looks like a transliterated Chinese NAME. The pinyin→Cyrillic table is generated
|
||
// deterministically from initials × finals + the documented irregularities (a versioned artifact).
|
||
//
|
||
// In the ratified DEFAULT-B miner the ru-side name CONFIRMATION sub-channel is OFF (it also needs the
|
||
// pymorphy3 is_name_lemma gate to block the «найти»=най+ти false-positive class), so isPalladiusToken is
|
||
// not wired into the default-B detector pipeline. It is kept as a deterministic, tested artifact — the
|
||
// name-shape signal a future faithful-A variant (with a Go morphology backend) would consume, and the
|
||
// pattern P4 transliteration-by-pair seam (§B5).
|
||
|
||
// palladius initials (声母) → Palladius Cyrillic (palladius.INITIALS).
|
||
var palladiusInitials = map[string]string{
|
||
"b": "б", "p": "п", "m": "м", "f": "ф", "d": "д", "t": "т", "n": "н", "l": "л",
|
||
"g": "г", "k": "к", "h": "х", "j": "цз", "q": "ц", "x": "с",
|
||
"zh": "чж", "ch": "ч", "sh": "ш", "r": "ж", "z": "цз", "c": "ц", "s": "с",
|
||
}
|
||
|
||
// palladius finals (韵母) → Palladius Cyrillic (palladius.FINALS). ü is written as v.
|
||
var palladiusFinals = map[string]string{
|
||
"a": "а", "o": "о", "e": "э", "ai": "ай", "ei": "эй", "ao": "ао", "ou": "оу",
|
||
"an": "ань", "en": "энь", "ang": "ан", "eng": "эн", "er": "эр", "ong": "ун",
|
||
"i": "и", "ia": "я", "ie": "е", "iao": "яо", "iu": "ю", "ian": "янь",
|
||
"in": "инь", "iang": "ян", "ing": "ин", "iong": "юн",
|
||
"u": "у", "ua": "уа", "uo": "о", "uai": "уай", "ui": "уй", "uan": "уань",
|
||
"un": "унь", "uang": "уан", "ueng": "ун",
|
||
"v": "юй", "ve": "юэ", "van": "юань", "vn": "юнь",
|
||
}
|
||
|
||
// whole zero-initial (y-/w-) syllables (palladius.Y_W).
|
||
var palladiusYW = map[string]string{
|
||
"yi": "и", "ya": "я", "ye": "е", "yao": "яо", "you": "ю", "yan": "янь", "yin": "инь",
|
||
"yang": "ян", "ying": "ин", "yong": "юн", "yu": "юй", "yue": "юэ", "yuan": "юань", "yun": "юнь",
|
||
"wu": "у", "wa": "ва", "wo": "во", "wai": "вай", "wei": "вэй", "wan": "вань", "wen": "вэнь",
|
||
"wang": "ван", "weng": "вэн",
|
||
}
|
||
|
||
// retroflex/sibilant + i → -ы/-и class (palladius.SPECIAL_I).
|
||
var palladiusSpecialI = map[string]string{
|
||
"zhi": "чжи", "chi": "чи", "shi": "ши", "ri": "жи", "zi": "цзы", "ci": "цы", "si": "сы",
|
||
}
|
||
|
||
var palladiusJQX = map[string]bool{"j": true, "q": true, "x": true}
|
||
|
||
// palladiusCyrSyllables is the distinct Cyrillic syllable set, longest-first, for greedy segmentation
|
||
// (palladius._CYR_SYL). Ties in length never both match a position (distinct strings), so the order
|
||
// among equal-length forms is immaterial — sorted (len desc, then value) for a stable artifact.
|
||
var palladiusCyrSyllables = buildPalladiusCyrSyllables()
|
||
|
||
func buildPalladiusCyrSyllables() []string {
|
||
set := map[string]bool{}
|
||
for _, v := range palladiusYW {
|
||
set[v] = true
|
||
}
|
||
for _, v := range palladiusSpecialI {
|
||
set[v] = true
|
||
}
|
||
retroflex := map[string]bool{"zh": true, "ch": true, "sh": true, "r": true, "z": true, "c": true, "s": true}
|
||
for pi, ci := range palladiusInitials {
|
||
for pf, cf := range palladiusFinals {
|
||
// ü finals (written v) are valid only after j/q/x (pinyin writes ü as plain u) or l/n.
|
||
switch pf {
|
||
case "v", "ve", "van", "vn":
|
||
if !palladiusJQX[pi] && pi != "l" && pi != "n" {
|
||
continue
|
||
}
|
||
}
|
||
// skip the retroflex/sibilant + bare i (handled by SPECIAL_I).
|
||
if pf == "i" && retroflex[pi] {
|
||
continue
|
||
}
|
||
set[ci+cf] = true
|
||
}
|
||
}
|
||
out := make([]string, 0, len(set))
|
||
for s := range set {
|
||
if s != "" {
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
sort.Slice(out, func(i, j int) bool {
|
||
if len([]rune(out[i])) != len([]rune(out[j])) {
|
||
return len([]rune(out[i])) > len([]rune(out[j]))
|
||
}
|
||
return out[i] < out[j]
|
||
})
|
||
return out
|
||
}
|
||
|
||
// isPalladiusToken reports whether a ru word segments fully into Palladius syllables (greedy longest-
|
||
// match, palladius.is_palladius_token): a high-precision "this looks like a transliterated Chinese name"
|
||
// signal. ъ is dropped before segmentation. minSyllables is the minimum syllable count.
|
||
func isPalladiusToken(token string, minSyllables int) bool {
|
||
t := strings.ToLower(strings.TrimSpace(token))
|
||
t = strings.ReplaceAll(t, "ъ", "")
|
||
if t == "" || !allCyrillic(t) {
|
||
return false
|
||
}
|
||
tr := []rune(t)
|
||
nSyl := 0
|
||
for i := 0; i < len(tr); {
|
||
matched := false
|
||
for _, s := range palladiusCyrSyllables {
|
||
sr := []rune(s)
|
||
if i+len(sr) <= len(tr) && runesEqual(tr[i:i+len(sr)], sr) {
|
||
i += len(sr)
|
||
nSyl++
|
||
matched = true
|
||
break
|
||
}
|
||
}
|
||
if !matched {
|
||
return false
|
||
}
|
||
}
|
||
return nSyl >= minSyllables
|
||
}
|
||
|
||
// allCyrillic reports whether every rune of s is Cyrillic (palladius._RE_CYR_WORD).
|
||
func allCyrillic(s string) bool {
|
||
if s == "" {
|
||
return false
|
||
}
|
||
for _, r := range s {
|
||
if !unicode.Is(unicode.Cyrillic, r) {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|