31 lines
1.5 KiB
Go
31 lines
1.5 KiB
Go
package text
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
"unicode"
|
||
)
|
||
|
||
// TestTokenizeLetters pins the script-agnostic tokenizer the bank post-check uses (fix-pack §и consolidation):
|
||
// maximal any-letter runs, lower-cased, with a combining mark dropped rather than splitting the word — the
|
||
// same combining-mark handling TokenizeScript has, now shared through one core.
|
||
func TestTokenizeLetters(t *testing.T) {
|
||
got := TokenizeLetters("Фан Юаню, к Горе!")
|
||
if strings.Join(got, "|") != "фан|юаню|к|горе" {
|
||
t.Fatalf("tokens = %v", got)
|
||
}
|
||
// A stress accent (о + U+0301) must not split the word — the pack-21 #11 fix, now inherited by the
|
||
// decl post-check that lang.TokenizeWords feeds.
|
||
if got := TokenizeLetters("сло́во"); len(got) != 1 || got[0] != "слово" {
|
||
t.Fatalf("a combining mark must not split the word, got %v", got)
|
||
}
|
||
// Script-agnostic: a Latin word and a Cyrillic word both tokenize (TokenizeScript would drop one).
|
||
if got := TokenizeLetters("Гу and Gu"); strings.Join(got, "|") != "гу|and|gu" {
|
||
t.Fatalf("any-letter tokenization must keep both scripts, got %v", got)
|
||
}
|
||
// It agrees with TokenizeScript on a single-script string (the shared core), so the consolidation is
|
||
// behaviour-preserving for the target lexical checkers.
|
||
if a, b := TokenizeLetters("Меч и Щит"), TokenizeScript("Меч и Щит", unicode.Cyrillic); strings.Join(a, "|") != strings.Join(b, "|") {
|
||
t.Fatalf("TokenizeLetters and TokenizeScript diverged on Cyrillic-only text: %v vs %v", a, b)
|
||
}
|
||
}
|