72 lines
3.2 KiB
Go
72 lines
3.2 KiB
Go
package lang
|
||
|
||
import "testing"
|
||
|
||
func TestTargetStemmerRuDeclension(t *testing.T) {
|
||
s := NewTargetStemmer(TargetChecksFor("ru"))
|
||
if !s.Enabled() {
|
||
t.Fatal("ru ships a decl_suffix registry, so the stemmer must be enabled")
|
||
}
|
||
// Common-noun oblique cases collapse to one stem — the measured decl-142/142-null case.
|
||
groups := [][]string{
|
||
{"меч", "меча", "мечом", "мечи", "мечу"},
|
||
{"апертура", "апертуры", "апертуру", "апертурой", "апертуре"},
|
||
{"старейшина", "старейшину", "старейшины", "старейшиной"},
|
||
{"источник", "источника", "источнику", "источнике"},
|
||
{"гора", "горы", "гору", "горе", "горой"},
|
||
}
|
||
for _, g := range groups {
|
||
for _, w := range g {
|
||
if !s.SameStem(g[0], w) {
|
||
t.Errorf("%q and %q must share a stem (%q vs %q)", g[0], w, s.Stem(g[0]), s.Stem(w))
|
||
}
|
||
}
|
||
}
|
||
// It must NOT conflate distinct words: a shared prefix that is not an inflection.
|
||
for _, pair := range [][2]string{{"меч", "мечта"}, {"гора", "город"}, {"дом", "домна"}} {
|
||
if s.SameStem(pair[0], pair[1]) {
|
||
t.Errorf("%q and %q are different words and must not share a stem (%q vs %q)", pair[0], pair[1], s.Stem(pair[0]), s.Stem(pair[1]))
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestSoftSignNamesDoNotCollideWithCommonWords is the fix-pack §а REGRESSION GUARD: a bare «ь» decl_suffix was
|
||
// weighed to close the Palladius soft-sign case («Фан Юаню»~«Фан Юань») and REJECTED, because stripping «ь»
|
||
// collapses top zh surnames onto common-adjective stems — «Синь/Линь» (辛/信/新, 林) → «син/лин» = «синий/линия».
|
||
// That fires a spurious Bank.SpoilerLeaks false-positive (the noise direction). This guard fails loudly if
|
||
// «ь» is ever re-added to the registry, so the collision cannot silently return.
|
||
func TestSoftSignNamesDoNotCollideWithCommonWords(t *testing.T) {
|
||
s := NewTargetStemmer(TargetChecksFor("ru"))
|
||
for _, pair := range [][2]string{{"Синь", "синий"}, {"Синь", "синим"}, {"Линь", "линий"}, {"Тень", "теней"}} {
|
||
if s.SameStem(pair[0], pair[1]) {
|
||
t.Errorf("«%s» (a soft-sign name) must NOT share a stem with the common word «%s» — a bare «ь» decl_suffix was re-added and re-introduced the collision (%q vs %q)",
|
||
pair[0], pair[1], s.Stem(pair[0]), s.Stem(pair[1]))
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestTargetStemmerInertWithoutRegistry(t *testing.T) {
|
||
var s TargetStemmer // zero value = a target with no decl_suffix data
|
||
if s.Enabled() {
|
||
t.Fatal("a stemmer with no registry must be inert")
|
||
}
|
||
if s.SameStem("меча", "меч") {
|
||
t.Fatal("an inert stemmer must fall back to EXACT equality only")
|
||
}
|
||
if !s.SameStem("меч", "меч") {
|
||
t.Fatal("an inert stemmer must still match identical words")
|
||
}
|
||
}
|
||
|
||
func TestTokenizeWords(t *testing.T) {
|
||
got := TokenizeWords("Фан Юаню, к Горе!")
|
||
want := []string{"фан", "юаню", "к", "горе"}
|
||
if len(got) != len(want) {
|
||
t.Fatalf("tokens = %v, want %v", got, want)
|
||
}
|
||
for i := range want {
|
||
if got[i] != want[i] {
|
||
t.Fatalf("token %d = %q, want %q", i, got[i], want[i])
|
||
}
|
||
}
|
||
}
|