111 lines
3.8 KiB
Go
111 lines
3.8 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"sort"
|
|
"strings"
|
|
"unicode"
|
|
|
|
"textmachine/backend/internal/lang"
|
|
)
|
|
|
|
// miner_palladius.go: the Palladius (Палладий) transliteration syllable GENERATOR + ru-side name-shape
|
|
// detector (WS3 — a Go port of exp16 palladius.py). A ru token that segments cleanly into Palladius
|
|
// syllables looks like a transliterated Chinese NAME. The pinyin→Cyrillic TABLE is DATA (a pair pack,
|
|
// configs/langpacks/<pair>/palladius.txt, loaded via internal/lang); this file keeps only the ALGORITHM —
|
|
// the syllable-inventory generator (over the table) and the greedy segmenter (D39.15: data as files,
|
|
// algorithm in code).
|
|
//
|
|
// In the ratified DEFAULT-B miner the ru-side name CONFIRMATION sub-channel is OFF (it also needs the
|
|
// pymorphy3 is_name_lemma gate to block the «найти»=най+ти false-positive class), so isPalladiusToken is
|
|
// not wired into the default-B detector pipeline. It is kept as a deterministic, tested artifact — the
|
|
// name-shape signal a future faithful-A variant (with a Go morphology backend) would consume, and the
|
|
// pattern P4 transliteration-by-pair seam (§B5).
|
|
|
|
var palladiusJQX = map[string]bool{"j": true, "q": true, "x": true}
|
|
|
|
// buildPalladiusCyrSyllables derives the distinct Cyrillic syllable set, longest-first, for greedy
|
|
// segmentation (palladius._CYR_SYL) from the pack's pinyin→Cyrillic table. Ties in length never both match
|
|
// a position (distinct strings), so the order among equal-length forms is immaterial — sorted (len desc,
|
|
// then value) for a stable artifact. The table (initials/finals/Y_W/SPECIAL_I) is langpack DATA.
|
|
func buildPalladiusCyrSyllables(p *lang.Pack) []string {
|
|
set := map[string]bool{}
|
|
for _, v := range p.PalladiusYW {
|
|
set[v] = true
|
|
}
|
|
for _, v := range p.PalladiusSpecialI {
|
|
set[v] = true
|
|
}
|
|
retroflex := map[string]bool{"zh": true, "ch": true, "sh": true, "r": true, "z": true, "c": true, "s": true}
|
|
for pi, ci := range p.PalladiusInitials {
|
|
for pf, cf := range p.PalladiusFinals {
|
|
// ü finals (written v) are valid only after j/q/x (pinyin writes ü as plain u) or l/n.
|
|
switch pf {
|
|
case "v", "ve", "van", "vn":
|
|
if !palladiusJQX[pi] && pi != "l" && pi != "n" {
|
|
continue
|
|
}
|
|
}
|
|
// skip the retroflex/sibilant + bare i (handled by SPECIAL_I).
|
|
if pf == "i" && retroflex[pi] {
|
|
continue
|
|
}
|
|
set[ci+cf] = true
|
|
}
|
|
}
|
|
out := make([]string, 0, len(set))
|
|
for s := range set {
|
|
if s != "" {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
if len([]rune(out[i])) != len([]rune(out[j])) {
|
|
return len([]rune(out[i])) > len([]rune(out[j]))
|
|
}
|
|
return out[i] < out[j]
|
|
})
|
|
return out
|
|
}
|
|
|
|
// isPalladiusToken reports whether a ru word segments fully into Palladius syllables (greedy longest-
|
|
// match, palladius.is_palladius_token): a high-precision "this looks like a transliterated Chinese name"
|
|
// signal. ъ is dropped before segmentation. minSyllables is the minimum syllable count. syllables is the
|
|
// longest-first inventory from buildPalladiusCyrSyllables(pack).
|
|
func isPalladiusToken(token string, minSyllables int, syllables []string) bool {
|
|
t := strings.ToLower(strings.TrimSpace(token))
|
|
t = strings.ReplaceAll(t, "ъ", "")
|
|
if t == "" || !allCyrillic(t) {
|
|
return false
|
|
}
|
|
tr := []rune(t)
|
|
nSyl := 0
|
|
for i := 0; i < len(tr); {
|
|
matched := false
|
|
for _, s := range syllables {
|
|
sr := []rune(s)
|
|
if i+len(sr) <= len(tr) && runesEqual(tr[i:i+len(sr)], sr) {
|
|
i += len(sr)
|
|
nSyl++
|
|
matched = true
|
|
break
|
|
}
|
|
}
|
|
if !matched {
|
|
return false
|
|
}
|
|
}
|
|
return nSyl >= minSyllables
|
|
}
|
|
|
|
// allCyrillic reports whether every rune of s is Cyrillic (palladius._RE_CYR_WORD).
|
|
func allCyrillic(s string) bool {
|
|
if s == "" {
|
|
return false
|
|
}
|
|
for _, r := range s {
|
|
if !unicode.Is(unicode.Cyrillic, r) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|