84 lines
3.4 KiB
Go
84 lines
3.4 KiB
Go
package lang
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"unicode"
|
|
)
|
|
|
|
// script.go: the DECLARED writing-script data (D39.60 §5 G3). A language's script is a linguistic fact the
|
|
// engine must know WITHOUT sniffing the text (a verdict sniffed from a random input sample is not reproducible
|
|
// on a resume — "declare, don't sniff"). It feeds the source-echo detector, the repair script-guard and the
|
|
// CJK-language predicate. Script NAMES come from data; the engine resolves a name to a Unicode range here and
|
|
// never branches on a language.
|
|
|
|
// scriptRanges is the canonical map from a declared SCRIPT NAME to its Unicode letter range. It is the ONE
|
|
// registry both the target word tokenizer (checks resolves word_script through ScriptRange) and the source-
|
|
// script detectors share — "add a script" is one row here plus the data that names it.
|
|
var scriptRanges = map[string]*unicode.RangeTable{
|
|
"han": unicode.Han,
|
|
"hiragana": unicode.Hiragana,
|
|
"katakana": unicode.Katakana,
|
|
"hangul": unicode.Hangul,
|
|
"latin": unicode.Latin,
|
|
"cyrillic": unicode.Cyrillic,
|
|
}
|
|
|
|
// cjkScriptNames is the set of script NAMES that count as CJK/dense — the writing systems where the
|
|
// reasoning-off echo class was measured (D19.1). A linguistic constant (the definition of "CJK script"), not
|
|
// a pair/book branch, so it lives in Go beside the ranges it names.
|
|
var cjkScriptNames = map[string]bool{"han": true, "hiragana": true, "katakana": true, "hangul": true}
|
|
|
|
// ScriptRange resolves a declared script name to its Unicode range (ok=false for an unknown name).
|
|
func ScriptRange(name string) (*unicode.RangeTable, bool) { r, ok := scriptRanges[name]; return r, ok }
|
|
|
|
var (
|
|
langScriptOnce sync.Once
|
|
langScriptBy map[string]map[string]bool // lang → set of declared script names
|
|
)
|
|
|
|
// loadLangScripts parses the embedded lang-script table once (lang → set of script names).
|
|
func loadLangScripts() {
|
|
langScriptOnce.Do(func() {
|
|
m, err := parseSectionedSet(mustEmbed("data/lang-script.txt"))
|
|
if err != nil {
|
|
panic(fmt.Sprintf("lang: embedded lang-script.txt is corrupt: %v", err))
|
|
}
|
|
langScriptBy = m
|
|
})
|
|
}
|
|
|
|
// LangScripts returns the Unicode ranges of the scripts a language is WRITTEN in (declared data). An unknown
|
|
// language yields nil — its consumers (the echo detector, the repair guard) then measure nothing rather than
|
|
// guessing. A named script with no range in scriptRanges is a corrupt data/registry pair → panic.
|
|
func LangScripts(lng string) []*unicode.RangeTable {
|
|
loadLangScripts()
|
|
names := langScriptBy[strings.ToLower(strings.TrimSpace(lng))]
|
|
if len(names) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]*unicode.RangeTable, 0, len(names))
|
|
for name := range names {
|
|
rt, ok := scriptRanges[name]
|
|
if !ok {
|
|
panic(fmt.Sprintf("lang: lang-script names script %q with no Unicode range (add it to scriptRanges)", name))
|
|
}
|
|
out = append(out, rt)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// IsCJKScriptLang reports whether a language is written in a CJK/dense script — the data-driven replacement
|
|
// for the isCJKLang Go switch. True when any of the language's declared scripts is a CJK one. Serves BOTH the
|
|
// source query (echo-exposure fires for a CJK source) and the target query (the echo check is skipped for a
|
|
// CJK target, where source-script runes in the output ARE the translation).
|
|
func IsCJKScriptLang(lng string) bool {
|
|
loadLangScripts()
|
|
for name := range langScriptBy[strings.ToLower(strings.TrimSpace(lng))] {
|
|
if cjkScriptNames[name] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|