textmachine/backend/internal/lang/stemmer.go

package lang

import (
	"sort"
	"strings"

	"textmachine/backend/internal/text"
)

// stemmer.go: the conservative TARGET stemmer (bank-quality §3, primitive C). It strips one inflectional
// ending off a word to get its stem, so a decl-aware check accepts an oblique case («мечом») against its
// nominative («меч») WITHOUT the seed listing every form — the fix for the decl-142/142-null false-miss
// noise, and the backstop the checker omission-detector shares. The ALGORITHM is generic; the ending REGISTRY
// is target data (decl_suffix in data/target-<tgt>.txt), so a target with none gets an inert stemmer (exact
// match only). A FLAGGER aid, biased to ACCEPT declensions; it never gates.

// minStemRunes is the shortest stem a suffix strip may leave. Below it a "stem" is too short to distinguish
// two words (стripping «а» off «дома» is fine → «дом»; stripping it off «яма» to «ям» is not worth trusting),
// so the ending is not stripped and the word stands whole. Conservative on the side of NOT conflating words.
const minStemRunes = 3

// TargetStemmer strips a registry ending off a word. The zero value is a valid inert stemmer (no endings →
// Stem is the identity → SameStem is exact equality), which is what a target with no decl_suffix data gets.
type TargetStemmer struct {
	suffixes []string // registry endings, longest-first
}

// NewTargetStemmer builds the stemmer from a target's decl_suffix registry. No registry → an inert stemmer.
func NewTargetStemmer(tc TargetChecks) TargetStemmer {
	sfx := append([]string(nil), tc.List("decl_suffix")...)
	// Longest-first so Stem strips the most specific ending (е.g. «ому» before «у»); ties by bytes for a
	// stable, deterministic order.
	sort.SliceStable(sfx, func(i, j int) bool {
		if a, b := len([]rune(sfx[i])), len([]rune(sfx[j])); a != b {
			return a > b
		}
		return sfx[i] < sfx[j]
	})
	return TargetStemmer{suffixes: sfx}
}

// Enabled reports whether the stemmer carries any endings — false for a target with no decl_suffix data, so a
// consumer can skip the whole stem branch.
func (s TargetStemmer) Enabled() bool { return len(s.suffixes) > 0 }

// Stem returns word with its longest registry ending removed, provided the remaining stem stays ≥
// minStemRunes; otherwise the word is returned unchanged. Case-insensitive (folded to lower), so a match is
// case-blind. An empty or all-short word returns folded-as-is.
func (s TargetStemmer) Stem(word string) string {
	w := strings.ToLower(word)
	rs := []rune(w)
	if len(rs) < minStemRunes {
		return w
	}
	for _, suf := range s.suffixes {
		sr := []rune(suf)
		if len(rs)-len(sr) < minStemRunes {
			continue // stripping this ending would leave too short a stem
		}
		if strings.HasSuffix(w, suf) {
			return string(rs[:len(rs)-len(sr)])
		}
	}
	return w
}

// SameStem reports whether two words share a stem under this stemmer — the decl-aware equality a consumer
// uses to accept one form of a word against another. Two identical words trivially match; the value the
// stemmer adds is matching «мечом»/«меча»/«меч». Never true for an inert stemmer beyond exact equality.
func (s TargetStemmer) SameStem(a, b string) bool {
	la, lb := strings.ToLower(a), strings.ToLower(b)
	if la == lb {
		return true
	}
	if !s.Enabled() {
		return false
	}
	return s.Stem(la) == s.Stem(lb)
}

// TokenizeWords splits a target string into its whole words (maximal letter runs), lower-cased. Script-
// generic (any Unicode letter), so it needs no per-language branch. Used to match a declined multi-word term
// head against the output. It delegates to text.TokenizeLetters — the ONE tokenizer, shared with the target
// lexical checkers' TokenizeScript, so a stress-accented output word no longer splits mid-word here either.
func TokenizeWords(s string) []string { return text.TokenizeLetters(s) }

// NearStem reports whether two words are the same word under a ONE-RUNE tolerance on the stem — the
// relation SameStem cannot express, and the measured reason it is needed.
//
// Stem strips exactly ONE ending. Two case forms of one word often need endings of DIFFERENT length
// stripped, so their stems come out different lengths and SameStem — which demands equality — says no:
// «путь»→«путь» against «пути»→«пут», «корифей»→«кориф» against «корифея»→«корифе», «злой»→«злой»
// against «злого»→«злог». On the paid cold run of 11.09 that accounted for ELEVEN of the eighteen recorded
// post-check misses: the bank's rendering was in the shipped unit, inflected, and the check could not see
// it. In every one of the eleven the two stems shared a prefix of at least minStemRunes and each ran at
// most one rune past it — which is exactly the predicate below.
//
// ⛔ IT IS DELIBERATELY NOT A DROP-IN FOR SameStem, and the hazard is documented, not hypothetical.
// data/target-ru.txt refuses the bare soft sign as a decl_suffix because «Синь»/«Линь» (common zh surnames)
// would collapse onto «син»/«лин», the stems of «синий»/«линия» — and this tolerance re-opens exactly that
// collapse: «синь» and «синий» satisfy it, prefix rule and all. So it is safe ONLY behind an anchor —
// another word of the same rendering that matched outright — which a single-word translit name can never
// have. The caller owns that gate; this function is only the relation. See the NB in data/target-ru.txt.
func (s TargetStemmer) NearStem(a, b string) bool {
	if s.SameStem(a, b) {
		return true
	}
	if !s.Enabled() {
		return false
	}
	return s.NearStems(s.Stem(strings.ToLower(a)), s.Stem(strings.ToLower(b)))
}

// NearStems is the relation NearStem applies, on stems ALREADY taken. It is a METHOD, not a free function,
// and that is the whole point: it must be inert on a target with no decl_suffix registry.
//
// ⛔ AS A FREE FUNCTION IT WAS A GENERALITY BUG, found by the acceptance. An inert stemmer's Stem is the
// identity, so every word is its own stem — and a one-rune-prefix relation over raw words says `cart` and
// `cars` are the same word. For any pair this repo does not yet have (the project's default review
// question), the counting column would have silently become fuzzy prefix matching. Enabled() is what makes
// it what it claims to be: a tolerance on STEMS, which a target without stemming does not have.
func (s TargetStemmer) NearStems(a, b string) bool {
	if a == b {
		return true
	}
	if !s.Enabled() {
		return false // no registry ⇒ no stems ⇒ nothing to be one rune away from
	}
	x, y := []rune(a), []rune(b)
	if len(x) > len(y) {
		x, y = y, x
	}
	// ⛔ ONE STEM MUST BE A STRICT PREFIX OF THE OTHER, one rune shorter. That is not a tightening for
	// neatness — it is what the relation MEANS. Stem TRUNCATES: two stems of one word differ only in how
	// much was stripped, so the shorter is necessarily a prefix of the longer. A same-length pair differing
	// in its final rune is two different words, and admitting it is how «глав»/«глаз» («глава рода Гуюэ»
	// satisfied by «глаза рода Гуюэ») got through — a false SILENCE, the report saying a rendering arrived
	// when it did not.
	//
	// Measured on the shipped book of the cold run (2094 distinct words): the loose form added 426 pairs
	// beyond SameStem, of which this rule keeps 291 and refuses 135 — and the 135 are «вред»~«время»,
	// «ветви»~«ветром», «весны»~«весь», «будто»~«будь». It costs exactly one real recovery in the measured
	// set, «злой»~«злого», where the nominative has nothing strippable that leaves a credible stem, so the
	// two stems differ by substitution rather than by a strip. That price is named rather than hidden.
	if len(y)-len(x) != 1 || len(x) < minStemRunes {
		return false
	}
	for i := range x {
		if x[i] != y[i] {
			return false
		}
	}
	return true
}