package lang import ( "fmt" "strings" "sync" "unicode" ) // script.go: the DECLARED writing-script data (D39.60 §5 G3). A language's script is a linguistic fact the // engine must know WITHOUT sniffing the text (a verdict sniffed from a random input sample is not reproducible // on a resume — "declare, don't sniff"). It feeds the source-echo detector, the repair script-guard and the // CJK-language predicate. Script NAMES come from data; the engine resolves a name to a Unicode range here and // never branches on a language. // scriptRanges is the canonical map from a declared SCRIPT NAME to its Unicode letter range. It is the ONE // registry both the target word tokenizer (checks resolves word_script through ScriptRange) and the source- // script detectors share — "add a script" is one row here plus the data that names it. var scriptRanges = map[string]*unicode.RangeTable{ "han": unicode.Han, "hiragana": unicode.Hiragana, "katakana": unicode.Katakana, "hangul": unicode.Hangul, "latin": unicode.Latin, "cyrillic": unicode.Cyrillic, } // cjkScriptNames is the set of script NAMES that count as CJK/dense — the writing systems where the // reasoning-off echo class was measured (D19.1). A linguistic constant (the definition of "CJK script"), not // a pair/book branch, so it lives in Go beside the ranges it names. var cjkScriptNames = map[string]bool{"han": true, "hiragana": true, "katakana": true, "hangul": true} // ScriptRange resolves a declared script name to its Unicode range (ok=false for an unknown name). func ScriptRange(name string) (*unicode.RangeTable, bool) { r, ok := scriptRanges[name]; return r, ok } // IsDenseScript reports whether a Unicode range is a dense (CJK/kana/Hangul) writing script. The one // RangeTable-level authority over cjkScriptNames, so a consumer (the checks magnitude source-gate) never // re-lists the dense set. Adding a dense script is one row in cjkScriptNames, not an edit here. func IsDenseScript(rt *unicode.RangeTable) bool { for name := range cjkScriptNames { if scriptRanges[name] == rt { return true } } return false } var ( langScriptOnce sync.Once langScriptBy map[string]map[string]bool // lang → set of declared script names ) // loadLangScripts parses the embedded lang-script table once (lang → set of script names). func loadLangScripts() { langScriptOnce.Do(func() { m, err := parseSectionedSet(mustEmbed("data/lang-script.txt")) if err != nil { panic(fmt.Sprintf("lang: embedded lang-script.txt is corrupt: %v", err)) } langScriptBy = m }) } // LangScripts returns the Unicode ranges of the scripts a language is WRITTEN in (declared data). An unknown // language yields nil — its consumers (the echo detector, the repair guard) then measure nothing rather than // guessing. A named script with no range in scriptRanges is a corrupt data/registry pair → panic. func LangScripts(lng string) []*unicode.RangeTable { loadLangScripts() names := langScriptBy[strings.ToLower(strings.TrimSpace(lng))] if len(names) == 0 { return nil } out := make([]*unicode.RangeTable, 0, len(names)) for name := range names { rt, ok := scriptRanges[name] if !ok { panic(fmt.Sprintf("lang: lang-script names script %q with no Unicode range (add it to scriptRanges)", name)) } out = append(out, rt) } return out } // IsCJKScriptLang reports whether a language is written in a CJK/dense script — the data-driven replacement // for the isCJKLang Go switch. True when any of the language's declared scripts is a CJK one. Serves BOTH the // source query (echo-exposure fires for a CJK source) and the target query (the echo check is skipped for a // CJK target, where source-script runes in the output ARE the translation). func IsCJKScriptLang(lng string) bool { loadLangScripts() for name := range langScriptBy[strings.ToLower(strings.TrimSpace(lng))] { if cjkScriptNames[name] { return true } } return false } var ( scriptSeriesOnce sync.Once scriptHeadFinal map[string]bool // script name → head-final (present only for scripts with a data row) ) // loadScriptSeries parses the embedded script-series table once (script name → head-final flag). func loadScriptSeries() { scriptSeriesOnce.Do(func() { m, err := parseScriptHeadFinal(mustEmbed("data/script-series.txt")) if err != nil { panic(fmt.Sprintf("lang: embedded script-series.txt is corrupt: %v", err)) } scriptHeadFinal = m }) } // parseScriptHeadFinal reads `head_positionscriptnamefinal|initial` rows into script name → // head-final. Pure, so a test can drive it with a synthetic table and prove a new script's head direction is // DATA, not a Go edit. An unknown category or position value is a corrupt-data error, never silent. func parseScriptHeadFinal(b []byte) (map[string]bool, error) { out := map[string]bool{} for i, raw := range strings.Split(string(b), "\n") { t := strings.TrimSpace(strings.TrimRight(raw, "\r")) if t == "" || strings.HasPrefix(t, "#") { continue } f := strings.Split(t, "\t") if len(f) != 3 || strings.TrimSpace(f[0]) != "head_position" { return nil, fmt.Errorf("line %d: want `head_positionscriptnamefinal|initial`, got %q", i+1, t) } name := strings.TrimSpace(f[1]) switch strings.TrimSpace(f[2]) { case "final": out[name] = true case "initial": out[name] = false default: return nil, fmt.Errorf("line %d: position must be final|initial, got %q", i+1, f[2]) } } return out, nil } // SeriesHeadFinal reports whether a dense SCRIPT forms head-FINAL series (the shared generic head is the // trailing rune(s)). It is DECLARED data (data/script-series.txt); a dense script with no row defaults to // head-final, the CJK modifier-head norm — so a dense script that co-batches head-INITIALLY is added by a // data row, with no Go edit. Exported so the data-drive is testable without routing through a source language. func SeriesHeadFinal(scriptName string) bool { loadScriptSeries() if hf, ok := scriptHeadFinal[scriptName]; ok { return hf } return true } // SeriesMorphology reports whether a SOURCE language forms rune-morpheme SERIES the terminologist should // co-batch (甲等/乙等/丙等 — grades sharing the head 等), and whether that shared head is final. Series apply // to DENSE (CJK) scripts, where one rune ≈ one morpheme so a single-rune difference is a real minimal pair; // in an alphabetic source care/core differ in one letter by coincidence, so the channel stays off. Head // finality is no longer a Go constant: it is resolved from the language's dense script(s) via SeriesHeadFinal // (data), so a dense non-head-final script needs no Go edit — headFinal is true only when EVERY dense script // of the language is head-final (they agree for zh/ja/ko; any head-initial dense script flips it). func SeriesMorphology(lng string) (enabled, headFinal bool) { loadLangScripts() headFinal = true for name := range langScriptBy[strings.ToLower(strings.TrimSpace(lng))] { if cjkScriptNames[name] { enabled = true if !SeriesHeadFinal(name) { headFinal = false } } } if !enabled { return false, false } return true, headFinal }