175 lines
7 KiB
Go
175 lines
7 KiB
Go
package lang
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"unicode"
|
|
)
|
|
|
|
// script.go: the DECLARED writing-script data (D39.60 §5 G3). A language's script is a linguistic fact the
|
|
// engine must know WITHOUT sniffing the text (a verdict sniffed from a random input sample is not reproducible
|
|
// on a resume — "declare, don't sniff"). It feeds the source-echo detector, the repair script-guard and the
|
|
// CJK-language predicate. Script NAMES come from data; the engine resolves a name to a Unicode range here and
|
|
// never branches on a language.
|
|
|
|
// scriptRanges is the canonical map from a declared SCRIPT NAME to its Unicode letter range. It is the ONE
|
|
// registry both the target word tokenizer (checks resolves word_script through ScriptRange) and the source-
|
|
// script detectors share — "add a script" is one row here plus the data that names it.
|
|
var scriptRanges = map[string]*unicode.RangeTable{
|
|
"han": unicode.Han,
|
|
"hiragana": unicode.Hiragana,
|
|
"katakana": unicode.Katakana,
|
|
"hangul": unicode.Hangul,
|
|
"latin": unicode.Latin,
|
|
"cyrillic": unicode.Cyrillic,
|
|
}
|
|
|
|
// cjkScriptNames is the set of script NAMES that count as CJK/dense — the writing systems where the
|
|
// reasoning-off echo class was measured (D19.1). A linguistic constant (the definition of "CJK script"), not
|
|
// a pair/book branch, so it lives in Go beside the ranges it names.
|
|
var cjkScriptNames = map[string]bool{"han": true, "hiragana": true, "katakana": true, "hangul": true}
|
|
|
|
// ScriptRange resolves a declared script name to its Unicode range (ok=false for an unknown name).
|
|
func ScriptRange(name string) (*unicode.RangeTable, bool) { r, ok := scriptRanges[name]; return r, ok }
|
|
|
|
// IsDenseScript reports whether a Unicode range is a dense (CJK/kana/Hangul) writing script. The one
|
|
// RangeTable-level authority over cjkScriptNames, so a consumer (the checks magnitude source-gate) never
|
|
// re-lists the dense set. Adding a dense script is one row in cjkScriptNames, not an edit here.
|
|
func IsDenseScript(rt *unicode.RangeTable) bool {
|
|
for name := range cjkScriptNames {
|
|
if scriptRanges[name] == rt {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
var (
|
|
langScriptOnce sync.Once
|
|
langScriptBy map[string]map[string]bool // lang → set of declared script names
|
|
)
|
|
|
|
// loadLangScripts parses the embedded lang-script table once (lang → set of script names).
|
|
func loadLangScripts() {
|
|
langScriptOnce.Do(func() {
|
|
m, err := parseSectionedSet(mustEmbed("data/lang-script.txt"))
|
|
if err != nil {
|
|
panic(fmt.Sprintf("lang: embedded lang-script.txt is corrupt: %v", err))
|
|
}
|
|
langScriptBy = m
|
|
})
|
|
}
|
|
|
|
// LangScripts returns the Unicode ranges of the scripts a language is WRITTEN in (declared data). An unknown
|
|
// language yields nil — its consumers (the echo detector, the repair guard) then measure nothing rather than
|
|
// guessing. A named script with no range in scriptRanges is a corrupt data/registry pair → panic.
|
|
func LangScripts(lng string) []*unicode.RangeTable {
|
|
loadLangScripts()
|
|
names := langScriptBy[strings.ToLower(strings.TrimSpace(lng))]
|
|
if len(names) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]*unicode.RangeTable, 0, len(names))
|
|
for name := range names {
|
|
rt, ok := scriptRanges[name]
|
|
if !ok {
|
|
panic(fmt.Sprintf("lang: lang-script names script %q with no Unicode range (add it to scriptRanges)", name))
|
|
}
|
|
out = append(out, rt)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// IsCJKScriptLang reports whether a language is written in a CJK/dense script — the data-driven replacement
|
|
// for the isCJKLang Go switch. True when any of the language's declared scripts is a CJK one. Serves BOTH the
|
|
// source query (echo-exposure fires for a CJK source) and the target query (the echo check is skipped for a
|
|
// CJK target, where source-script runes in the output ARE the translation).
|
|
func IsCJKScriptLang(lng string) bool {
|
|
loadLangScripts()
|
|
for name := range langScriptBy[strings.ToLower(strings.TrimSpace(lng))] {
|
|
if cjkScriptNames[name] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
var (
|
|
scriptSeriesOnce sync.Once
|
|
scriptHeadFinal map[string]bool // script name → head-final (present only for scripts with a data row)
|
|
)
|
|
|
|
// loadScriptSeries parses the embedded script-series table once (script name → head-final flag).
|
|
func loadScriptSeries() {
|
|
scriptSeriesOnce.Do(func() {
|
|
m, err := parseScriptHeadFinal(mustEmbed("data/script-series.txt"))
|
|
if err != nil {
|
|
panic(fmt.Sprintf("lang: embedded script-series.txt is corrupt: %v", err))
|
|
}
|
|
scriptHeadFinal = m
|
|
})
|
|
}
|
|
|
|
// parseScriptHeadFinal reads `head_position<TAB>scriptname<TAB>final|initial` rows into script name →
|
|
// head-final. Pure, so a test can drive it with a synthetic table and prove a new script's head direction is
|
|
// DATA, not a Go edit. An unknown category or position value is a corrupt-data error, never silent.
|
|
func parseScriptHeadFinal(b []byte) (map[string]bool, error) {
|
|
out := map[string]bool{}
|
|
for i, raw := range strings.Split(string(b), "\n") {
|
|
t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
|
|
if t == "" || strings.HasPrefix(t, "#") {
|
|
continue
|
|
}
|
|
f := strings.Split(t, "\t")
|
|
if len(f) != 3 || strings.TrimSpace(f[0]) != "head_position" {
|
|
return nil, fmt.Errorf("line %d: want `head_position<TAB>scriptname<TAB>final|initial`, got %q", i+1, t)
|
|
}
|
|
name := strings.TrimSpace(f[1])
|
|
switch strings.TrimSpace(f[2]) {
|
|
case "final":
|
|
out[name] = true
|
|
case "initial":
|
|
out[name] = false
|
|
default:
|
|
return nil, fmt.Errorf("line %d: position must be final|initial, got %q", i+1, f[2])
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// SeriesHeadFinal reports whether a dense SCRIPT forms head-FINAL series (the shared generic head is the
|
|
// trailing rune(s)). It is DECLARED data (data/script-series.txt); a dense script with no row defaults to
|
|
// head-final, the CJK modifier-head norm — so a dense script that co-batches head-INITIALLY is added by a
|
|
// data row, with no Go edit. Exported so the data-drive is testable without routing through a source language.
|
|
func SeriesHeadFinal(scriptName string) bool {
|
|
loadScriptSeries()
|
|
if hf, ok := scriptHeadFinal[scriptName]; ok {
|
|
return hf
|
|
}
|
|
return true
|
|
}
|
|
|
|
// SeriesMorphology reports whether a SOURCE language forms rune-morpheme SERIES the terminologist should
|
|
// co-batch (甲等/乙等/丙等 — grades sharing the head 等), and whether that shared head is final. Series apply
|
|
// to DENSE (CJK) scripts, where one rune ≈ one morpheme so a single-rune difference is a real minimal pair;
|
|
// in an alphabetic source care/core differ in one letter by coincidence, so the channel stays off. Head
|
|
// finality is no longer a Go constant: it is resolved from the language's dense script(s) via SeriesHeadFinal
|
|
// (data), so a dense non-head-final script needs no Go edit — headFinal is true only when EVERY dense script
|
|
// of the language is head-final (they agree for zh/ja/ko; any head-initial dense script flips it).
|
|
func SeriesMorphology(lng string) (enabled, headFinal bool) {
|
|
loadLangScripts()
|
|
headFinal = true
|
|
for name := range langScriptBy[strings.ToLower(strings.TrimSpace(lng))] {
|
|
if cjkScriptNames[name] {
|
|
enabled = true
|
|
if !SeriesHeadFinal(name) {
|
|
headFinal = false
|
|
}
|
|
}
|
|
}
|
|
if !enabled {
|
|
return false, false
|
|
}
|
|
return true, headFinal
|
|
}
|