50 lines
1.7 KiB
Go
50 lines
1.7 KiB
Go
package text
|
||
|
||
import (
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
// runes.go: the rune/word primitives shared by the matchers that run over already-normalized
|
||
// text — the memory bank's whole-word post-check, the miner's containment/prefix tests and the
|
||
// cheap style checkers' word scans. They are deliberately allocation-free and script-agnostic;
|
||
// script-specific policy stays in its subsystem.
|
||
|
||
// RunesEqual reports whether two rune slices of the SAME LENGTH hold the same runes.
|
||
// PRECONDITION: len(a) <= len(b) — the loop ranges over a and indexes b, so a shorter b
|
||
// panics. Every caller slices both sides to one length before calling (a fixed-width window
|
||
// scan), which is why the check is not repeated here: it is a hot inner loop of the
|
||
// post-check and of the miner's O(n·m) containment scan.
|
||
func RunesEqual(a, b []rune) bool {
|
||
for i := range a {
|
||
if a[i] != b[i] {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// TokenizeCyrillic lower-cases text and splits it into maximal runs of Cyrillic letters
|
||
// (ё kept distinct from е), dropping everything else (spaces, punctuation, digits, Latin).
|
||
// It is the word unit of the Russian-side lexical checkers: they compare whole words against
|
||
// a word list, so a run that stops at any non-Cyrillic rune is exactly the boundary those
|
||
// checks need, and the ё/е distinction is what the yofication check measures.
|
||
func TokenizeCyrillic(text string) []string {
|
||
var words []string
|
||
var b strings.Builder
|
||
flush := func() {
|
||
if b.Len() > 0 {
|
||
words = append(words, b.String())
|
||
b.Reset()
|
||
}
|
||
}
|
||
for _, r := range strings.ToLower(text) {
|
||
if unicode.Is(unicode.Cyrillic, r) {
|
||
b.WriteRune(r)
|
||
} else {
|
||
flush()
|
||
}
|
||
}
|
||
flush()
|
||
return words
|
||
}
|