57 lines
2.1 KiB
Go
57 lines
2.1 KiB
Go
package lang
|
||
|
||
import "testing"
|
||
|
||
func TestTargetStemmerRuDeclension(t *testing.T) {
|
||
s := NewTargetStemmer(TargetChecksFor("ru"))
|
||
if !s.Enabled() {
|
||
t.Fatal("ru ships a decl_suffix registry, so the stemmer must be enabled")
|
||
}
|
||
// Common-noun oblique cases collapse to one stem — the measured decl-142/142-null case.
|
||
groups := [][]string{
|
||
{"меч", "меча", "мечом", "мечи", "мечу"},
|
||
{"апертура", "апертуры", "апертуру", "апертурой", "апертуре"},
|
||
{"старейшина", "старейшину", "старейшины", "старейшиной"},
|
||
{"источник", "источника", "источнику", "источнике"},
|
||
{"гора", "горы", "гору", "горе", "горой"},
|
||
}
|
||
for _, g := range groups {
|
||
for _, w := range g {
|
||
if !s.SameStem(g[0], w) {
|
||
t.Errorf("%q and %q must share a stem (%q vs %q)", g[0], w, s.Stem(g[0]), s.Stem(w))
|
||
}
|
||
}
|
||
}
|
||
// It must NOT conflate distinct words: a shared prefix that is not an inflection.
|
||
for _, pair := range [][2]string{{"меч", "мечта"}, {"гора", "город"}, {"дом", "домна"}} {
|
||
if s.SameStem(pair[0], pair[1]) {
|
||
t.Errorf("%q and %q are different words and must not share a stem (%q vs %q)", pair[0], pair[1], s.Stem(pair[0]), s.Stem(pair[1]))
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestTargetStemmerInertWithoutRegistry(t *testing.T) {
|
||
var s TargetStemmer // zero value = a target with no decl_suffix data
|
||
if s.Enabled() {
|
||
t.Fatal("a stemmer with no registry must be inert")
|
||
}
|
||
if s.SameStem("меча", "меч") {
|
||
t.Fatal("an inert stemmer must fall back to EXACT equality only")
|
||
}
|
||
if !s.SameStem("меч", "меч") {
|
||
t.Fatal("an inert stemmer must still match identical words")
|
||
}
|
||
}
|
||
|
||
func TestTokenizeWords(t *testing.T) {
|
||
got := TokenizeWords("Фан Юаню, к Горе!")
|
||
want := []string{"фан", "юаню", "к", "горе"}
|
||
if len(got) != len(want) {
|
||
t.Fatalf("tokens = %v, want %v", got, want)
|
||
}
|
||
for i := range want {
|
||
if got[i] != want[i] {
|
||
t.Fatalf("token %d = %q, want %q", i, got[i], want[i])
|
||
}
|
||
}
|
||
}
|