package lang import ( "sort" "strings" "textmachine/backend/internal/text" ) // stemmer.go: the conservative TARGET stemmer (bank-quality §3, primitive C). It strips one inflectional // ending off a word to get its stem, so a decl-aware check accepts an oblique case («мечом») against its // nominative («меч») WITHOUT the seed listing every form — the fix for the decl-142/142-null false-miss // noise, and the backstop the checker omission-detector shares. The ALGORITHM is generic; the ending REGISTRY // is target data (decl_suffix in data/target-.txt), so a target with none gets an inert stemmer (exact // match only). A FLAGGER aid, biased to ACCEPT declensions; it never gates. // minStemRunes is the shortest stem a suffix strip may leave. Below it a "stem" is too short to distinguish // two words (стripping «а» off «дома» is fine → «дом»; stripping it off «яма» to «ям» is not worth trusting), // so the ending is not stripped and the word stands whole. Conservative on the side of NOT conflating words. const minStemRunes = 3 // TargetStemmer strips a registry ending off a word. The zero value is a valid inert stemmer (no endings → // Stem is the identity → SameStem is exact equality), which is what a target with no decl_suffix data gets. type TargetStemmer struct { suffixes []string // registry endings, longest-first } // NewTargetStemmer builds the stemmer from a target's decl_suffix registry. No registry → an inert stemmer. func NewTargetStemmer(tc TargetChecks) TargetStemmer { sfx := append([]string(nil), tc.List("decl_suffix")...) // Longest-first so Stem strips the most specific ending (е.g. «ому» before «у»); ties by bytes for a // stable, deterministic order. sort.SliceStable(sfx, func(i, j int) bool { if a, b := len([]rune(sfx[i])), len([]rune(sfx[j])); a != b { return a > b } return sfx[i] < sfx[j] }) return TargetStemmer{suffixes: sfx} } // Enabled reports whether the stemmer carries any endings — false for a target with no decl_suffix data, so a // consumer can skip the whole stem branch. func (s TargetStemmer) Enabled() bool { return len(s.suffixes) > 0 } // Stem returns word with its longest registry ending removed, provided the remaining stem stays ≥ // minStemRunes; otherwise the word is returned unchanged. Case-insensitive (folded to lower), so a match is // case-blind. An empty or all-short word returns folded-as-is. func (s TargetStemmer) Stem(word string) string { w := strings.ToLower(word) rs := []rune(w) if len(rs) < minStemRunes { return w } for _, suf := range s.suffixes { sr := []rune(suf) if len(rs)-len(sr) < minStemRunes { continue // stripping this ending would leave too short a stem } if strings.HasSuffix(w, suf) { return string(rs[:len(rs)-len(sr)]) } } return w } // SameStem reports whether two words share a stem under this stemmer — the decl-aware equality a consumer // uses to accept one form of a word against another. Two identical words trivially match; the value the // stemmer adds is matching «мечом»/«меча»/«меч». Never true for an inert stemmer beyond exact equality. func (s TargetStemmer) SameStem(a, b string) bool { la, lb := strings.ToLower(a), strings.ToLower(b) if la == lb { return true } if !s.Enabled() { return false } return s.Stem(la) == s.Stem(lb) } // TokenizeWords splits a target string into its whole words (maximal letter runs), lower-cased. Script- // generic (any Unicode letter), so it needs no per-language branch. Used to match a declined multi-word term // head against the output. It delegates to text.TokenizeLetters — the ONE tokenizer, shared with the target // lexical checkers' TokenizeScript, so a stress-accented output word no longer splits mid-word here either. func TokenizeWords(s string) []string { return text.TokenizeLetters(s) }