package lang import "testing" func TestTargetStemmerRuDeclension(t *testing.T) { s := NewTargetStemmer(TargetChecksFor("ru")) if !s.Enabled() { t.Fatal("ru ships a decl_suffix registry, so the stemmer must be enabled") } // Common-noun oblique cases collapse to one stem — the measured decl-142/142-null case. groups := [][]string{ {"меч", "меча", "мечом", "мечи", "мечу"}, {"апертура", "апертуры", "апертуру", "апертурой", "апертуре"}, {"старейшина", "старейшину", "старейшины", "старейшиной"}, {"источник", "источника", "источнику", "источнике"}, {"гора", "горы", "гору", "горе", "горой"}, } for _, g := range groups { for _, w := range g { if !s.SameStem(g[0], w) { t.Errorf("%q and %q must share a stem (%q vs %q)", g[0], w, s.Stem(g[0]), s.Stem(w)) } } } // It must NOT conflate distinct words: a shared prefix that is not an inflection. for _, pair := range [][2]string{{"меч", "мечта"}, {"гора", "город"}, {"дом", "домна"}} { if s.SameStem(pair[0], pair[1]) { t.Errorf("%q and %q are different words and must not share a stem (%q vs %q)", pair[0], pair[1], s.Stem(pair[0]), s.Stem(pair[1])) } } } // TestSoftSignNamesDoNotCollideWithCommonWords is the fix-pack §а REGRESSION GUARD: a bare «ь» decl_suffix was // weighed to close the Palladius soft-sign case («Фан Юаню»~«Фан Юань») and REJECTED, because stripping «ь» // collapses top zh surnames onto common-adjective stems — «Синь/Линь» (辛/信/新, 林) → «син/лин» = «синий/линия». // That fires a spurious Bank.SpoilerLeaks false-positive (the noise direction). This guard fails loudly if // «ь» is ever re-added to the registry, so the collision cannot silently return. func TestSoftSignNamesDoNotCollideWithCommonWords(t *testing.T) { s := NewTargetStemmer(TargetChecksFor("ru")) for _, pair := range [][2]string{{"Синь", "синий"}, {"Синь", "синим"}, {"Линь", "линий"}, {"Тень", "теней"}} { if s.SameStem(pair[0], pair[1]) { t.Errorf("«%s» (a soft-sign name) must NOT share a stem with the common word «%s» — a bare «ь» decl_suffix was re-added and re-introduced the collision (%q vs %q)", pair[0], pair[1], s.Stem(pair[0]), s.Stem(pair[1])) } } } func TestTargetStemmerInertWithoutRegistry(t *testing.T) { var s TargetStemmer // zero value = a target with no decl_suffix data if s.Enabled() { t.Fatal("a stemmer with no registry must be inert") } if s.SameStem("меча", "меч") { t.Fatal("an inert stemmer must fall back to EXACT equality only") } if !s.SameStem("меч", "меч") { t.Fatal("an inert stemmer must still match identical words") } } func TestTokenizeWords(t *testing.T) { got := TokenizeWords("Фан Юаню, к Горе!") want := []string{"фан", "юаню", "к", "горе"} if len(got) != len(want) { t.Fatalf("tokens = %v, want %v", got, want) } for i := range want { if got[i] != want[i] { t.Fatalf("token %d = %q, want %q", i, got[i], want[i]) } } }