101 lines
3.8 KiB
Go
101 lines
3.8 KiB
Go
package lang
|
||
|
||
import (
|
||
"sort"
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
// stemmer.go: the conservative TARGET stemmer (bank-quality §3, primitive C). It strips one inflectional
|
||
// ending off a word to get its stem, so a decl-aware check accepts an oblique case («мечом») against its
|
||
// nominative («меч») WITHOUT the seed listing every form — the fix for the decl-142/142-null false-miss
|
||
// noise, and the backstop the checker omission-detector shares. The ALGORITHM is generic; the ending REGISTRY
|
||
// is target data (decl_suffix in data/target-<tgt>.txt), so a target with none gets an inert stemmer (exact
|
||
// match only). A FLAGGER aid, biased to ACCEPT declensions; it never gates.
|
||
|
||
// minStemRunes is the shortest stem a suffix strip may leave. Below it a "stem" is too short to distinguish
|
||
// two words (стripping «а» off «дома» is fine → «дом»; stripping it off «яма» to «ям» is not worth trusting),
|
||
// so the ending is not stripped and the word stands whole. Conservative on the side of NOT conflating words.
|
||
const minStemRunes = 3
|
||
|
||
// TargetStemmer strips a registry ending off a word. The zero value is a valid inert stemmer (no endings →
|
||
// Stem is the identity → SameStem is exact equality), which is what a target with no decl_suffix data gets.
|
||
type TargetStemmer struct {
|
||
suffixes []string // registry endings, longest-first
|
||
}
|
||
|
||
// NewTargetStemmer builds the stemmer from a target's decl_suffix registry. No registry → an inert stemmer.
|
||
func NewTargetStemmer(tc TargetChecks) TargetStemmer {
|
||
sfx := append([]string(nil), tc.List("decl_suffix")...)
|
||
// Longest-first so Stem strips the most specific ending (е.g. «ому» before «у»); ties by bytes for a
|
||
// stable, deterministic order.
|
||
sort.SliceStable(sfx, func(i, j int) bool {
|
||
if a, b := len([]rune(sfx[i])), len([]rune(sfx[j])); a != b {
|
||
return a > b
|
||
}
|
||
return sfx[i] < sfx[j]
|
||
})
|
||
return TargetStemmer{suffixes: sfx}
|
||
}
|
||
|
||
// Enabled reports whether the stemmer carries any endings — false for a target with no decl_suffix data, so a
|
||
// consumer can skip the whole stem branch.
|
||
func (s TargetStemmer) Enabled() bool { return len(s.suffixes) > 0 }
|
||
|
||
// Stem returns word with its longest registry ending removed, provided the remaining stem stays ≥
|
||
// minStemRunes; otherwise the word is returned unchanged. Case-insensitive (folded to lower), so a match is
|
||
// case-blind. An empty or all-short word returns folded-as-is.
|
||
func (s TargetStemmer) Stem(word string) string {
|
||
w := strings.ToLower(word)
|
||
rs := []rune(w)
|
||
if len(rs) < minStemRunes {
|
||
return w
|
||
}
|
||
for _, suf := range s.suffixes {
|
||
sr := []rune(suf)
|
||
if len(rs)-len(sr) < minStemRunes {
|
||
continue // stripping this ending would leave too short a stem
|
||
}
|
||
if strings.HasSuffix(w, suf) {
|
||
return string(rs[:len(rs)-len(sr)])
|
||
}
|
||
}
|
||
return w
|
||
}
|
||
|
||
// SameStem reports whether two words share a stem under this stemmer — the decl-aware equality a consumer
|
||
// uses to accept one form of a word against another. Two identical words trivially match; the value the
|
||
// stemmer adds is matching «мечом»/«меча»/«меч». Never true for an inert stemmer beyond exact equality.
|
||
func (s TargetStemmer) SameStem(a, b string) bool {
|
||
la, lb := strings.ToLower(a), strings.ToLower(b)
|
||
if la == lb {
|
||
return true
|
||
}
|
||
if !s.Enabled() {
|
||
return false
|
||
}
|
||
return s.Stem(la) == s.Stem(lb)
|
||
}
|
||
|
||
// TokenizeWords splits a target string into its whole words (maximal letter runs), lower-cased. Script-
|
||
// generic (any Unicode letter), so it needs no per-language branch. Used to match a declined multi-word term
|
||
// head against the output.
|
||
func TokenizeWords(s string) []string {
|
||
var out []string
|
||
var cur []rune
|
||
flush := func() {
|
||
if len(cur) > 0 {
|
||
out = append(out, string(cur))
|
||
cur = cur[:0]
|
||
}
|
||
}
|
||
for _, r := range strings.ToLower(s) {
|
||
if unicode.IsLetter(r) {
|
||
cur = append(cur, r)
|
||
continue
|
||
}
|
||
flush()
|
||
}
|
||
flush()
|
||
return out
|
||
}
|