467 lines
18 KiB
Go
467 lines
18 KiB
Go
package lang
|
||
|
||
import (
|
||
"embed"
|
||
"fmt"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
)
|
||
|
||
// embedded.go: language data the engine needs even for a book with NO langpack pack — engine-universal
|
||
// (a CJK numeral is a linguistic constant; a refusal phrase is engine safety behaviour) or target-generic
|
||
// (the Russian glossary-injection wire-text). It is versioned with the engine BINARY (like code), not with
|
||
// a book's pack, because a no-pack book still splits 第X章 chapters, still flags a refusal, and still injects
|
||
// a Russian glossary block (the ja→ru golden proves all three). Kept OUT of internal/pipeline so the
|
||
// data/algorithm boundary holds; embedded so it is available without a book's langpack_root. Each file is
|
||
// SECTIONED per language/target, so adding a language = add a section, removing one = delete its section.
|
||
|
||
//go:embed data/cjk-section.txt data/refusal.txt data/injection.txt data/sentence-abbrev.txt data/target-ru.txt data/terminator.txt
|
||
var embeddedFS embed.FS
|
||
|
||
// targetCheckFiles maps a TARGET language to its embedded readability-checker data file (pair-14 data-out).
|
||
// Add a target = add data/target-<tgt>.txt to the go:embed line above and a row here.
|
||
var targetCheckFiles = map[string]string{"ru": "data/target-ru.txt"}
|
||
|
||
// TargetChecks is the TARGET-language readability wordlists + DETECTION patterns (pair-14 data-out) the
|
||
// target-general checkers/sanitizer run on ANY →target output. A target with no file yields an empty value
|
||
// (HasData()==false → the consumers stay inert). Values are grouped per category in AUTHORED ORDER.
|
||
type TargetChecks struct{ byKey map[string][]string }
|
||
|
||
// List returns the authored-order values for a category ("" categories → nil).
|
||
func (t TargetChecks) List(key string) []string { return t.byKey[key] }
|
||
|
||
// HasData reports whether this target ships any checker data (used to gate the target-general checkers so a
|
||
// target without data flags nothing rather than mis-firing).
|
||
func (t TargetChecks) HasData() bool { return len(t.byKey) > 0 }
|
||
|
||
var (
|
||
targetChecksOnce sync.Once
|
||
targetChecksBy map[string]TargetChecks
|
||
)
|
||
|
||
// TargetChecksFor returns the readability-checker data for a TARGET language, parsed once from the embedded
|
||
// per-target file. Unknown/absent target → empty TargetChecks. Panics on a corrupt embed.
|
||
func TargetChecksFor(tgt string) TargetChecks {
|
||
targetChecksOnce.Do(func() {
|
||
targetChecksBy = map[string]TargetChecks{}
|
||
for lang, file := range targetCheckFiles {
|
||
byKey, err := parseOrderedByCategory(mustEmbed(file))
|
||
if err != nil {
|
||
panic(fmt.Sprintf("lang: embedded %s is corrupt: %v", file, err))
|
||
}
|
||
targetChecksBy[lang] = TargetChecks{byKey: byKey}
|
||
}
|
||
})
|
||
return targetChecksBy[tgt]
|
||
}
|
||
|
||
// parseOrderedByCategory reads `category<TAB>value` lines into category → ordered values. Value is VERBATIM
|
||
// (a regex may carry a trailing metachar), so only the whole line's \r is stripped; blank/`#`-comment lines
|
||
// (by their trimmed form) are dropped.
|
||
func parseOrderedByCategory(b []byte) (map[string][]string, error) {
|
||
out := map[string][]string{}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
line := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(line) == "" || strings.HasPrefix(strings.TrimSpace(line), "#") {
|
||
continue
|
||
}
|
||
f := strings.SplitN(line, "\t", 2)
|
||
if len(f) != 2 || strings.TrimSpace(f[0]) == "" || f[1] == "" {
|
||
return nil, fmt.Errorf("line %d: want `category<TAB>value`, got %q", i+1, line)
|
||
}
|
||
out[f[0]] = append(out[f[0]], f[1])
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// CJKSection is the universal chapter/section-numeral inventory shared by the ingest chapter-splitter and
|
||
// the chunker heading rule (pair-14 §4 + addendum-A: ONE source, no ingest↔chunker byte-duplication). The
|
||
// heading-numeral membership test derives from Digit ∪ Unit ∪ Zero — nothing is stored twice.
|
||
type CJKSection struct {
|
||
Digit map[rune]int // 一→1 … 九→9 (incl. 两/兩→2)
|
||
Unit map[rune]int // 十→10 百→100 千→1000
|
||
Zero map[rune]bool // 〇 零 (positional zero: value*10 in a numeral run)
|
||
// ChapterUnit membership + ChapterUnitOrdered (authored order) — ingest's detectChapterUnit iterates in
|
||
// order and breaks a tie by first-seen, so the ORDER is behaviour, not just a set.
|
||
ChapterUnit map[rune]bool
|
||
ChapterUnitOrdered []rune
|
||
// Magnitude: the myriad-scale big units (万/億/兆 → value) for the number-magnitude gate. int64 (10^12
|
||
// overflows int32). ONE source with Digit/Unit — the gate no longer keeps its own CJK numeral switch.
|
||
Magnitude map[rune]int64
|
||
// ChapterMarker: the rune(s) that OPEN a section header (第N章), sorted. Data, not a literal in the
|
||
// ingest regex — the same reason the numerals beside it are data.
|
||
ChapterMarker []rune
|
||
headingRunes string // sorted Zero∪Digit∪Unit, cached for the ingest chapter-numeral regex class
|
||
}
|
||
|
||
// ChapterMarkerClass returns the section-opening runes as a sorted string for use INSIDE a regex character
|
||
// class (the ingest chapter-numeral pattern). None of the runes are class metacharacters, so no escaping.
|
||
func (c *CJKSection) ChapterMarkerClass() string { return string(c.ChapterMarker) }
|
||
|
||
// IsNumeralRune reports whether r can be part of a CJK numeral EXPRESSION (Zero∪Digit∪Unit∪Magnitude) — the
|
||
// alphabet of a magnitude candidate run (the gate adds ASCII digits itself). BigUnit returns a big unit's
|
||
// value (万/億/兆). Both read the single CJKSection source (pair-14 data-out dedup).
|
||
func (c *CJKSection) IsNumeralRune(r rune) bool {
|
||
if c.IsHeadingNumeral(r) {
|
||
return true
|
||
}
|
||
_, ok := c.Magnitude[r]
|
||
return ok
|
||
}
|
||
func (c *CJKSection) BigUnit(r rune) (int64, bool) { v, ok := c.Magnitude[r]; return v, ok }
|
||
|
||
// IsHeadingNumeral reports whether r can be part of a CJK chapter-number run (Zero ∪ Digit ∪ Unit). Arabic
|
||
// digits are handled by the caller (they are not language data).
|
||
func (c *CJKSection) IsHeadingNumeral(r rune) bool {
|
||
if c.Zero[r] {
|
||
return true
|
||
}
|
||
if _, ok := c.Digit[r]; ok {
|
||
return true
|
||
}
|
||
_, ok := c.Unit[r]
|
||
return ok
|
||
}
|
||
|
||
// HeadingNumeralClass returns the CJK heading-numeral runes (Zero∪Digit∪Unit) as a sorted string, for use
|
||
// INSIDE a regex character class (the ingest chapter-numeral pattern). Sorted for a stable pattern; order
|
||
// inside a class is immaterial to matching. None of the runes are class metacharacters, so no escaping.
|
||
func (c *CJKSection) HeadingNumeralClass() string { return c.headingRunes }
|
||
|
||
// Terminators is the engine's sentence-terminator alphabet, in two classes: CJK marks split a sentence
|
||
// ZERO-WIDTH (CJK prose has no inter-sentence space), generic marks split only before whitespace. It is
|
||
// the single source both the source chunker and the coverage gate read — they used to keep a private
|
||
// copy of the same two switches each.
|
||
type Terminators struct {
|
||
cjk map[rune]bool
|
||
generic map[rune]bool
|
||
}
|
||
|
||
// IsTerminator reports whether r ends a sentence in EITHER class.
|
||
func (t *Terminators) IsTerminator(r rune) bool { return t.cjk[r] || t.generic[r] }
|
||
|
||
// IsCJK reports whether r is a zero-width-splitting CJK terminator.
|
||
func (t *Terminators) IsCJK(r rune) bool { return t.cjk[r] }
|
||
|
||
var (
|
||
termOnce sync.Once
|
||
termVal *Terminators
|
||
termErr error
|
||
)
|
||
|
||
// DefaultTerminators returns the process-wide terminator classes, parsed once from the embedded file.
|
||
// Panics on a corrupt embed (an in-repo data file — a build/test failure, never user input).
|
||
func DefaultTerminators() *Terminators {
|
||
termOnce.Do(func() { termVal, termErr = parseTerminators(mustEmbed("data/terminator.txt")) })
|
||
if termErr != nil {
|
||
panic(fmt.Sprintf("lang: embedded data/terminator.txt is corrupt: %v", termErr))
|
||
}
|
||
return termVal
|
||
}
|
||
|
||
func parseTerminators(b []byte) (*Terminators, error) {
|
||
t := &Terminators{cjk: map[rune]bool{}, generic: map[rune]bool{}}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
line := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(line) == "" || strings.HasPrefix(strings.TrimSpace(line), "#") {
|
||
continue
|
||
}
|
||
f := strings.Split(line, "\t")
|
||
if len(f) != 2 {
|
||
return nil, fmt.Errorf("line %d: want `category<TAB>rune`, got %q", i+1, line)
|
||
}
|
||
r := []rune(f[1])
|
||
if len(r) != 1 {
|
||
return nil, fmt.Errorf("line %d: key %q must be a single rune", i+1, f[1])
|
||
}
|
||
switch f[0] {
|
||
case "cjk":
|
||
t.cjk[r[0]] = true
|
||
case "generic":
|
||
t.generic[r[0]] = true
|
||
default:
|
||
return nil, fmt.Errorf("line %d: unknown category %q (want cjk|generic)", i+1, f[0])
|
||
}
|
||
}
|
||
if len(t.cjk) == 0 || len(t.generic) == 0 {
|
||
return nil, fmt.Errorf("terminator needs non-empty cjk and generic sections")
|
||
}
|
||
return t, nil
|
||
}
|
||
|
||
var (
|
||
cjkOnce sync.Once
|
||
cjkVal *CJKSection
|
||
cjkErr error
|
||
)
|
||
|
||
// DefaultCJKSection returns the process-wide CJK section-numeral data, parsed once from the embedded file.
|
||
// It panics on a corrupt embed (a build-time asset the tests exercise, so a bad edit fails the suite, never
|
||
// production silently) — the parse is deterministic and input-free.
|
||
func DefaultCJKSection() *CJKSection {
|
||
cjkOnce.Do(func() { cjkVal, cjkErr = parseCJKSection(mustEmbed("data/cjk-section.txt")) })
|
||
if cjkErr != nil {
|
||
panic(fmt.Sprintf("lang: embedded cjk-section.txt is corrupt: %v", cjkErr))
|
||
}
|
||
return cjkVal
|
||
}
|
||
|
||
func parseCJKSection(b []byte) (*CJKSection, error) {
|
||
c := &CJKSection{Digit: map[rune]int{}, Unit: map[rune]int{}, Zero: map[rune]bool{}, ChapterUnit: map[rune]bool{}, Magnitude: map[rune]int64{}}
|
||
valueRune := func(i int, f []string) (rune, int, error) {
|
||
if len(f) != 3 {
|
||
return 0, 0, fmt.Errorf("line %d: %s wants `%s<TAB>rune<TAB>value`, got %q", i+1, f[0], f[0], strings.Join(f, "\t"))
|
||
}
|
||
r := []rune(f[1])
|
||
if len(r) != 1 {
|
||
return 0, 0, fmt.Errorf("line %d: key %q must be a single rune", i+1, f[1])
|
||
}
|
||
v, err := strconv.Atoi(strings.TrimSpace(f[2]))
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("line %d: value %q: %w", i+1, f[2], err)
|
||
}
|
||
return r[0], v, nil
|
||
}
|
||
setRune := func(i int, f []string) (rune, error) {
|
||
if len(f) != 2 {
|
||
return 0, fmt.Errorf("line %d: %s wants `%s<TAB>rune`, got %q", i+1, f[0], f[0], strings.Join(f, "\t"))
|
||
}
|
||
r := []rune(f[1])
|
||
if len(r) != 1 {
|
||
return 0, fmt.Errorf("line %d: key %q must be a single rune", i+1, f[1])
|
||
}
|
||
return r[0], nil
|
||
}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
t := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
|
||
continue
|
||
}
|
||
f := strings.Split(t, "\t")
|
||
switch f[0] {
|
||
case "digit":
|
||
r, v, err := valueRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c.Digit[r] = v
|
||
case "unit":
|
||
r, v, err := valueRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c.Unit[r] = v
|
||
case "zero":
|
||
r, err := setRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c.Zero[r] = true
|
||
case "chapter_marker":
|
||
r, err := setRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c.ChapterMarker = append(c.ChapterMarker, r)
|
||
case "chapter_unit":
|
||
r, err := setRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !c.ChapterUnit[r] {
|
||
c.ChapterUnit[r] = true
|
||
c.ChapterUnitOrdered = append(c.ChapterUnitOrdered, r)
|
||
}
|
||
case "magnitude":
|
||
if len(f) != 3 {
|
||
return nil, fmt.Errorf("line %d: magnitude wants `magnitude<TAB>rune<TAB>value`, got %q", i+1, t)
|
||
}
|
||
r := []rune(f[1])
|
||
if len(r) != 1 {
|
||
return nil, fmt.Errorf("line %d: magnitude key %q must be a single rune", i+1, f[1])
|
||
}
|
||
v, err := strconv.ParseInt(strings.TrimSpace(f[2]), 10, 64)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("line %d: magnitude value %q: %w", i+1, f[2], err)
|
||
}
|
||
c.Magnitude[r[0]] = v
|
||
default:
|
||
return nil, fmt.Errorf("line %d: unknown category %q (want digit|unit|zero|chapter_marker|chapter_unit|magnitude)", i+1, f[0])
|
||
}
|
||
}
|
||
if len(c.Digit) == 0 || len(c.Unit) == 0 || len(c.Zero) == 0 || len(c.ChapterUnit) == 0 || len(c.Magnitude) == 0 || len(c.ChapterMarker) == 0 {
|
||
return nil, fmt.Errorf("cjk-section needs non-empty digit, unit, zero, chapter_marker, chapter_unit and magnitude sections")
|
||
}
|
||
sort.Slice(c.ChapterMarker, func(i, j int) bool { return c.ChapterMarker[i] < c.ChapterMarker[j] })
|
||
// Cache the sorted heading-numeral class (Zero∪Digit∪Unit) for the ingest regex.
|
||
var runes []rune
|
||
for r := range c.Zero {
|
||
runes = append(runes, r)
|
||
}
|
||
for r := range c.Digit {
|
||
runes = append(runes, r)
|
||
}
|
||
for r := range c.Unit {
|
||
runes = append(runes, r)
|
||
}
|
||
sort.Slice(runes, func(i, j int) bool { return runes[i] < runes[j] })
|
||
c.headingRunes = string(runes)
|
||
return c, nil
|
||
}
|
||
|
||
// InjectionTexts is the TARGET-language glossary/injection wire-text (pair-14 §2): the Russian headers and
|
||
// annotations the memory renderers emit into a request. TARGET data (a Russian header for a →ru book), so
|
||
// the renderers gate on HasData — a target with no injection texts (e.g. →en) injects NOTHING rather than a
|
||
// stray Russian block. The ja→ru golden renders these with NO book pack, so they are engine/target data.
|
||
type InjectionTexts struct {
|
||
GlossaryHeader string // "ГЛОССАРИЙ (…):" — introduces the draft glossary block
|
||
EditorHeader string // "КАНОНИЧЕСКИЕ ПЕРЕВОДЫ …:" — introduces the editor constraint block
|
||
UnverifiedMarker string // " ⟨проверить⟩" — tags an unverified draft candidate (leading spaces significant)
|
||
GenderMale string // " (муж. — …)" DC3 gender directive (leading space significant)
|
||
GenderFemale string // " (жен. — …)"
|
||
GenderHidden string // " (пол СКРЫТ …)"
|
||
}
|
||
|
||
// HasData reports whether the target has injection texts (its header is present). The renderers use it to
|
||
// gate: a target without texts injects nothing, so a non-ru book never gets a Russian glossary block.
|
||
func (t InjectionTexts) HasData() bool { return t.GlossaryHeader != "" }
|
||
|
||
var (
|
||
injectionOnce sync.Once
|
||
injectionBy map[string]*InjectionTexts
|
||
injectionErr error
|
||
)
|
||
|
||
// InjectionTextsFor returns the injection wire-text for a TARGET language (pair-14 §2), parsed once from the
|
||
// embedded file. A target with no rows returns a zero value (HasData()==false → no injection). Panics on a
|
||
// corrupt embed. VALUE bytes are verbatim (leading spaces in gender_*/unverified_marker are significant).
|
||
func InjectionTextsFor(targetLang string) InjectionTexts {
|
||
injectionOnce.Do(func() { injectionBy, injectionErr = parseInjection(mustEmbed("data/injection.txt")) })
|
||
if injectionErr != nil {
|
||
panic(fmt.Sprintf("lang: embedded injection.txt is corrupt: %v", injectionErr))
|
||
}
|
||
if t, ok := injectionBy[targetLang]; ok {
|
||
return *t
|
||
}
|
||
return InjectionTexts{}
|
||
}
|
||
|
||
func parseInjection(b []byte) (map[string]*InjectionTexts, error) {
|
||
out := map[string]*InjectionTexts{}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
line := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(line) == "" || strings.HasPrefix(strings.TrimSpace(line), "#") {
|
||
continue
|
||
}
|
||
f := strings.SplitN(line, "\t", 3) // value (f[2]) VERBATIM — leading spaces are significant
|
||
if len(f) != 3 || strings.TrimSpace(f[0]) == "" || strings.TrimSpace(f[1]) == "" {
|
||
return nil, fmt.Errorf("line %d: want `target<TAB>key<TAB>value`, got %q", i+1, line)
|
||
}
|
||
tgt, key, val := f[0], f[1], f[2]
|
||
if out[tgt] == nil {
|
||
out[tgt] = &InjectionTexts{}
|
||
}
|
||
t := out[tgt]
|
||
switch key {
|
||
case "glossary_header":
|
||
t.GlossaryHeader = val
|
||
case "editor_header":
|
||
t.EditorHeader = val
|
||
case "unverified_marker":
|
||
t.UnverifiedMarker = val
|
||
case "gender_male":
|
||
t.GenderMale = val
|
||
case "gender_female":
|
||
t.GenderFemale = val
|
||
case "gender_hidden":
|
||
t.GenderHidden = val
|
||
default:
|
||
return nil, fmt.Errorf("line %d: unknown injection key %q", i+1, key)
|
||
}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
var (
|
||
refusalOnce sync.Once
|
||
refusalVal []string
|
||
)
|
||
|
||
// RefusalPatterns returns the universal refusal-blacklist regex patterns (pair-14 §3), parsed once from the
|
||
// embedded sectioned file in authored order. Engine safety data: a model can refuse in any language for any
|
||
// pair, so this is not book-pack data — the caller joins the patterns into one case-insensitive regex. Each
|
||
// non-comment line is ONE pattern, taken verbatim (a regex may contain leading/trailing metacharacters, so
|
||
// it is NOT trimmed beyond the line's \r). Panics on a corrupt embed.
|
||
func RefusalPatterns() []string {
|
||
refusalOnce.Do(func() { refusalVal = embeddedLines(mustEmbed("data/refusal.txt")) })
|
||
return refusalVal
|
||
}
|
||
|
||
// embeddedLines returns the non-comment, non-blank lines of an embedded file VERBATIM (only the trailing \r
|
||
// is stripped; the content is NOT trimmed — significant for a regex pattern). A blank/`#`-comment line is
|
||
// dropped by its TRIMMED form, but a kept line keeps its own leading/trailing bytes.
|
||
func embeddedLines(b []byte) []string {
|
||
var out []string
|
||
for _, raw := range strings.Split(string(b), "\n") {
|
||
s := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(s) == "" || strings.HasPrefix(strings.TrimSpace(s), "#") {
|
||
continue
|
||
}
|
||
out = append(out, s)
|
||
}
|
||
return out
|
||
}
|
||
|
||
var (
|
||
abbrevOnce sync.Once
|
||
abbrevBy map[string]map[string]bool
|
||
)
|
||
|
||
// SentenceAbbrev returns the lower-cased sentence-splitter abbreviation SET for a SOURCE language (pair-14
|
||
// §4), parsed once from the embedded sectioned file. An unknown/absent language returns an empty set (a
|
||
// source with no ASCII-period abbreviations — e.g. a CJK source using 。). Panics on a corrupt embed.
|
||
func SentenceAbbrev(srcLang string) map[string]bool {
|
||
abbrevOnce.Do(func() {
|
||
var err error
|
||
abbrevBy, err = parseSectionedSet(mustEmbed("data/sentence-abbrev.txt"))
|
||
if err != nil {
|
||
panic(fmt.Sprintf("lang: embedded sentence-abbrev.txt is corrupt: %v", err))
|
||
}
|
||
})
|
||
if m, ok := abbrevBy[srcLang]; ok {
|
||
return m
|
||
}
|
||
return map[string]bool{}
|
||
}
|
||
|
||
// parseSectionedSet reads `key<TAB>member` lines into per-key sets (keys lower-cased on read is NOT done
|
||
// here — the key is the language tag; members are stored verbatim). Used for the sentence-abbrev table.
|
||
func parseSectionedSet(b []byte) (map[string]map[string]bool, error) {
|
||
out := map[string]map[string]bool{}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
t := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
|
||
continue
|
||
}
|
||
f := strings.Split(t, "\t")
|
||
if len(f) != 2 || strings.TrimSpace(f[0]) == "" || strings.TrimSpace(f[1]) == "" {
|
||
return nil, fmt.Errorf("line %d: want `key<TAB>member`, got %q", i+1, t)
|
||
}
|
||
key := strings.TrimSpace(f[0])
|
||
if out[key] == nil {
|
||
out[key] = map[string]bool{}
|
||
}
|
||
out[key][strings.TrimSpace(f[1])] = true
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func mustEmbed(name string) []byte {
|
||
b, err := embeddedFS.ReadFile(name)
|
||
if err != nil {
|
||
panic(fmt.Sprintf("lang: missing embedded asset %q: %v", name, err))
|
||
}
|
||
return b
|
||
}
|