102 lines
4.8 KiB
Go
102 lines
4.8 KiB
Go
package text
|
||
|
||
import (
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
// runes.go: the rune/word primitives shared by the matchers that run over already-normalized
|
||
// text — the memory bank's whole-word post-check, the miner's containment/prefix tests and the
|
||
// cheap style checkers' word scans. They are deliberately allocation-free and script-agnostic;
|
||
// script-specific policy stays in its subsystem.
|
||
|
||
// RunesEqual reports whether two rune slices of the SAME LENGTH hold the same runes.
|
||
// PRECONDITION: len(a) <= len(b) — the loop ranges over a and indexes b, so a shorter b
|
||
// panics. Every caller slices both sides to one length before calling (a fixed-width window
|
||
// scan), which is why the check is not repeated here: it is a hot inner loop of the
|
||
// post-check and of the miner's O(n·m) containment scan.
|
||
func RunesEqual(a, b []rune) bool {
|
||
for i := range a {
|
||
if a[i] != b[i] {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// TokenizeScript lower-cases text and splits it into maximal runs of letters in the given script `ws`
|
||
// (a Unicode range), dropping everything else (spaces, punctuation, digits, other scripts). A COMBINING
|
||
// MARK (unicode.Mn) inside a run does NOT break it and is dropped from the token, so a stress-accented word
|
||
// «сло́во» (о + U+0301) tokenizes as ONE word «слово», not «сло»+«во» (a leaked stress accent no longer
|
||
// splits a target word — pack-21 #11). It is the word unit of the →target lexical checkers: they compare
|
||
// whole words against a word list, so a run that stops at any non-word-script rune is exactly the boundary
|
||
// those checks need. The word SCRIPT is the target's declared alphabet (data), so the tokenizer is target-
|
||
// general; the ё/е distinction the yofication check measures is preserved (ё is a Cyrillic letter).
|
||
func TokenizeScript(text string, ws *unicode.RangeTable) []string {
|
||
return tokenizeWords(text, func(r rune) bool { return unicode.Is(ws, r) })
|
||
}
|
||
|
||
// TokenizeLetters is the SCRIPT-AGNOSTIC sibling of TokenizeScript: maximal runs of any Unicode letter,
|
||
// lower-cased, with the same combining-mark handling. For a caller that matches words regardless of the
|
||
// target's alphabet and has no declared script to pass — the decl-aware bank post-check tokenizes a target
|
||
// term head against the output without needing (or knowing) the target's word script. lang.TokenizeWords
|
||
// wraps it, so `lang` (which MAY import `text`) keeps one tokenizer instead of a second copy of this loop.
|
||
func TokenizeLetters(text string) []string {
|
||
return tokenizeWords(text, unicode.IsLetter)
|
||
}
|
||
|
||
// tokenizeWords is the shared core of TokenizeScript/TokenizeLetters: lower-case, then emit maximal runs of
|
||
// runes the predicate accepts, dropping a combining mark that sits on an open run rather than breaking it.
|
||
func tokenizeWords(text string, isWord func(rune) bool) []string {
|
||
var words []string
|
||
var b strings.Builder
|
||
flush := func() {
|
||
if b.Len() > 0 {
|
||
words = append(words, b.String())
|
||
b.Reset()
|
||
}
|
||
}
|
||
for _, r := range strings.ToLower(text) {
|
||
switch {
|
||
case isWord(r):
|
||
b.WriteRune(r)
|
||
case b.Len() > 0 && unicode.Is(unicode.Mn, r):
|
||
// a combining mark sits ON the preceding word base: keep the run open and drop the mark.
|
||
default:
|
||
flush()
|
||
}
|
||
}
|
||
flush()
|
||
return words
|
||
}
|
||
|
||
// TokenizeCyrillic is the ru-default word tokenizer (script = unicode.Cyrillic). The target-general pipeline
|
||
// path tokenizes on the target's DECLARED word script (Checkers.tokenizeWords); this wrapper serves callers
|
||
// that are unconditionally Russian (unit tests, ad-hoc tooling).
|
||
func TokenizeCyrillic(text string) []string { return TokenizeScript(text, unicode.Cyrillic) }
|
||
|
||
// DenseScript reports whether r is a "dense" writing-system rune — a CJK ideograph, kana or Hangul syllable
|
||
// that maps to roughly ONE model token per character, versus an alphabetic script (~⅓ token/char). It is the
|
||
// SINGLE home of the token-sizing script class that render.EstimateTokens, chunk.TokenClassCounts and the
|
||
// memory-bank glossary budget all read, so the sizing taxonomy can never byte-drift across the packages
|
||
// (D39.60 §3.3 C-1: the same class was inlined in five places).
|
||
func DenseScript(r rune) bool {
|
||
return unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul)
|
||
}
|
||
|
||
// DenseSparseCounts splits s into (dense, sparse) rune counts for the token estimate: dense = DenseScript
|
||
// runes (~1 token each), sparse = other letters/digits/punctuation (~⅓ token each); whitespace folds into
|
||
// neighbours and counts as neither. The shared classifier behind the reservation-sizing estimate AND the
|
||
// glossary-line budget — one taxonomy, so the "second copy of the arithmetic" can never diverge from the first.
|
||
func DenseSparseCounts(s string) (dense, sparse int) {
|
||
for _, r := range s {
|
||
switch {
|
||
case DenseScript(r):
|
||
dense++
|
||
case unicode.IsSpace(r):
|
||
default:
|
||
sparse++
|
||
}
|
||
}
|
||
return dense, sparse
|
||
}
|