package lang import ( "sort" "strings" "textmachine/backend/internal/text" ) // stemmer.go: the conservative TARGET stemmer (bank-quality §3, primitive C). It strips one inflectional // ending off a word to get its stem, so a decl-aware check accepts an oblique case («мечом») against its // nominative («меч») WITHOUT the seed listing every form — the fix for the decl-142/142-null false-miss // noise, and the backstop the checker omission-detector shares. The ALGORITHM is generic; the ending REGISTRY // is target data (decl_suffix in data/target-.txt), so a target with none gets an inert stemmer (exact // match only). A FLAGGER aid, biased to ACCEPT declensions; it never gates. // minStemRunes is the shortest stem a suffix strip may leave. Below it a "stem" is too short to distinguish // two words (стripping «а» off «дома» is fine → «дом»; stripping it off «яма» to «ям» is not worth trusting), // so the ending is not stripped and the word stands whole. Conservative on the side of NOT conflating words. const minStemRunes = 3 // TargetStemmer strips a registry ending off a word. The zero value is a valid inert stemmer (no endings → // Stem is the identity → SameStem is exact equality), which is what a target with no decl_suffix data gets. type TargetStemmer struct { suffixes []string // registry endings, longest-first } // NewTargetStemmer builds the stemmer from a target's decl_suffix registry. No registry → an inert stemmer. func NewTargetStemmer(tc TargetChecks) TargetStemmer { sfx := append([]string(nil), tc.List("decl_suffix")...) // Longest-first so Stem strips the most specific ending (е.g. «ому» before «у»); ties by bytes for a // stable, deterministic order. sort.SliceStable(sfx, func(i, j int) bool { if a, b := len([]rune(sfx[i])), len([]rune(sfx[j])); a != b { return a > b } return sfx[i] < sfx[j] }) return TargetStemmer{suffixes: sfx} } // Enabled reports whether the stemmer carries any endings — false for a target with no decl_suffix data, so a // consumer can skip the whole stem branch. func (s TargetStemmer) Enabled() bool { return len(s.suffixes) > 0 } // Stem returns word with its longest registry ending removed, provided the remaining stem stays ≥ // minStemRunes; otherwise the word is returned unchanged. Case-insensitive (folded to lower), so a match is // case-blind. An empty or all-short word returns folded-as-is. func (s TargetStemmer) Stem(word string) string { w := strings.ToLower(word) rs := []rune(w) if len(rs) < minStemRunes { return w } for _, suf := range s.suffixes { sr := []rune(suf) if len(rs)-len(sr) < minStemRunes { continue // stripping this ending would leave too short a stem } if strings.HasSuffix(w, suf) { return string(rs[:len(rs)-len(sr)]) } } return w } // SameStem reports whether two words share a stem under this stemmer — the decl-aware equality a consumer // uses to accept one form of a word against another. Two identical words trivially match; the value the // stemmer adds is matching «мечом»/«меча»/«меч». Never true for an inert stemmer beyond exact equality. func (s TargetStemmer) SameStem(a, b string) bool { la, lb := strings.ToLower(a), strings.ToLower(b) if la == lb { return true } if !s.Enabled() { return false } return s.Stem(la) == s.Stem(lb) } // TokenizeWords splits a target string into its whole words (maximal letter runs), lower-cased. Script- // generic (any Unicode letter), so it needs no per-language branch. Used to match a declined multi-word term // head against the output. It delegates to text.TokenizeLetters — the ONE tokenizer, shared with the target // lexical checkers' TokenizeScript, so a stress-accented output word no longer splits mid-word here either. func TokenizeWords(s string) []string { return text.TokenizeLetters(s) } // NearStem reports whether two words are the same word under a ONE-RUNE tolerance on the stem — the // relation SameStem cannot express, and the measured reason it is needed. // // Stem strips exactly ONE ending. Two case forms of one word often need endings of DIFFERENT length // stripped, so their stems come out different lengths and SameStem — which demands equality — says no: // «путь»→«путь» against «пути»→«пут», «корифей»→«кориф» against «корифея»→«корифе», «злой»→«злой» // against «злого»→«злог». On the paid cold run of 11.09 that accounted for ELEVEN of the eighteen recorded // post-check misses: the bank's rendering was in the shipped unit, inflected, and the check could not see // it. In every one of the eleven the two stems shared a prefix of at least minStemRunes and each ran at // most one rune past it — which is exactly the predicate below. // // ⛔ IT IS DELIBERATELY NOT A DROP-IN FOR SameStem, and the hazard is documented, not hypothetical. // data/target-ru.txt refuses the bare soft sign as a decl_suffix because «Синь»/«Линь» (common zh surnames) // would collapse onto «син»/«лин», the stems of «синий»/«линия» — and this tolerance re-opens exactly that // collapse: «синь» and «синий» satisfy it, prefix rule and all. So it is safe ONLY behind an anchor — // another word of the same rendering that matched outright — which a single-word translit name can never // have. The caller owns that gate; this function is only the relation. See the NB in data/target-ru.txt. func (s TargetStemmer) NearStem(a, b string) bool { if s.SameStem(a, b) { return true } if !s.Enabled() { return false } return s.NearStems(s.Stem(strings.ToLower(a)), s.Stem(strings.ToLower(b))) } // NearStems is the relation NearStem applies, on stems ALREADY taken. It is a METHOD, not a free function, // and that is the whole point: it must be inert on a target with no decl_suffix registry. // // ⛔ AS A FREE FUNCTION IT WAS A GENERALITY BUG, found by the acceptance. An inert stemmer's Stem is the // identity, so every word is its own stem — and a one-rune-prefix relation over raw words says `cart` and // `cars` are the same word. For any pair this repo does not yet have (the project's default review // question), the counting column would have silently become fuzzy prefix matching. Enabled() is what makes // it what it claims to be: a tolerance on STEMS, which a target without stemming does not have. func (s TargetStemmer) NearStems(a, b string) bool { if a == b { return true } if !s.Enabled() { return false // no registry ⇒ no stems ⇒ nothing to be one rune away from } x, y := []rune(a), []rune(b) if len(x) > len(y) { x, y = y, x } // ⛔ ONE STEM MUST BE A STRICT PREFIX OF THE OTHER, one rune shorter. That is not a tightening for // neatness — it is what the relation MEANS. Stem TRUNCATES: two stems of one word differ only in how // much was stripped, so the shorter is necessarily a prefix of the longer. A same-length pair differing // in its final rune is two different words, and admitting it is how «глав»/«глаз» («глава рода Гуюэ» // satisfied by «глаза рода Гуюэ») got through — a false SILENCE, the report saying a rendering arrived // when it did not. // // Measured on the shipped book of the cold run (2094 distinct words): the loose form added 426 pairs // beyond SameStem, of which this rule keeps 291 and refuses 135 — and the 135 are «вред»~«время», // «ветви»~«ветром», «весны»~«весь», «будто»~«будь». It costs exactly one real recovery in the measured // set, «злой»~«злого», where the nominative has nothing strippable that leaves a credible stem, so the // two stems differ by substitution rather than by a strip. That price is named rather than hidden. if len(y)-len(x) != 1 || len(x) < minStemRunes { return false } for i := range x { if x[i] != y[i] { return false } } return true }