621 lines
27 KiB
Go
621 lines
27 KiB
Go
package lang
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"embed"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
)
|
||
|
||
// embedded.go: language data the engine needs even for a book with NO langpack pack — engine-universal
|
||
// (a CJK numeral is a linguistic constant; a refusal phrase is engine safety behaviour) or target-generic
|
||
// (the Russian glossary-injection wire-text). It is versioned with the engine BINARY (like code), not with
|
||
// a book's pack, because a no-pack book still splits 第X章 chapters, still flags a refusal, and still injects
|
||
// a Russian glossary block (the ja→ru golden proves all three). Kept OUT of internal/pipeline so the
|
||
// data/algorithm boundary holds; embedded so it is available without a book's langpack_root. Each file is
|
||
// SECTIONED per language/target, so adding a language = add a section, removing one = delete its section.
|
||
|
||
//go:embed data/cjk-section.txt data/refusal.txt data/injection.txt data/sentence-abbrev.txt data/target-ru.txt data/terminator.txt data/lang-script.txt data/script-series.txt
|
||
var embeddedFS embed.FS
|
||
|
||
// embedAlgoVersion tags the embedded-data content-hash RECIPE (like packAlgoVersion tags the pack's). Bump
|
||
// it only when the hashing scheme changes (the framing or the file set/order); NEVER for a data edit — a
|
||
// data edit already moves the hash via the file bytes.
|
||
const embedAlgoVersion = "embed-v1"
|
||
|
||
var (
|
||
embeddedVersionOnce sync.Once
|
||
embeddedVersionVal string
|
||
)
|
||
|
||
// embeddedFile is one embedded data file's name + bytes, the unit hashEmbeddedFiles folds. Split out so a
|
||
// mutation test can recompute the version over a byte-flipped copy and prove any edit diverges the hash.
|
||
type embeddedFile struct {
|
||
name string
|
||
data []byte
|
||
}
|
||
|
||
// embeddedDataFiles returns every embedded data/* file as (name, bytes) in FIXED lexical order (independent
|
||
// of the go:embed directive's order). Panics on a corrupt embed (a build-time asset, never user input).
|
||
func embeddedDataFiles() []embeddedFile {
|
||
ents, err := embeddedFS.ReadDir("data")
|
||
if err != nil {
|
||
panic(fmt.Sprintf("lang: cannot list embedded data dir: %v", err))
|
||
}
|
||
files := make([]embeddedFile, 0, len(ents))
|
||
for _, e := range ents {
|
||
if !e.IsDir() {
|
||
files = append(files, embeddedFile{name: e.Name(), data: mustEmbed("data/" + e.Name())})
|
||
}
|
||
}
|
||
sort.Slice(files, func(i, j int) bool { return files[i].name < files[j].name })
|
||
return files
|
||
}
|
||
|
||
// hashEmbeddedFiles folds (embedAlgoVersion, then each file's path+bytes) into the content hash — the same
|
||
// framing Load() uses for the pack. Pure, so a test can call it over a mutated file list.
|
||
func hashEmbeddedFiles(files []embeddedFile) string {
|
||
h := sha256.New()
|
||
h.Write([]byte(embedAlgoVersion))
|
||
for _, f := range files {
|
||
// Fold path + bytes so a rename OR a moved byte both shift the hash (same framing as Load()).
|
||
h.Write([]byte("\x00data/" + f.name + "\x00"))
|
||
h.Write(f.data)
|
||
}
|
||
return embedAlgoVersion + "-" + hex.EncodeToString(h.Sum(nil))[:12]
|
||
}
|
||
|
||
// EmbeddedVersion is the content hash of ALL embedded language data (embedAlgoVersion + a sha256 of every
|
||
// data/* file, each folded as name+bytes in a FIXED lexical order). It mirrors Pack.Version(): the bytes ARE
|
||
// the version, drift-proof by mechanism. The pipeline folds it into the snapshot beside LangpackVersion, so a
|
||
// byte edit of ANY embedded file is a loud --resnapshot, never a silent verdict/wire change. This closes the
|
||
// D39.60 §6.1 gap: injection.txt rides the wire and target-ru.txt drives sanitizer verdicts, yet the embed
|
||
// plane sat in NO content hash — a "bytes are version" hole the pack plane never had. Computed once (the
|
||
// embed FS is immutable).
|
||
func EmbeddedVersion() string {
|
||
embeddedVersionOnce.Do(func() { embeddedVersionVal = hashEmbeddedFiles(embeddedDataFiles()) })
|
||
return embeddedVersionVal
|
||
}
|
||
|
||
// TargetChecks is the TARGET-language readability wordlists + DETECTION patterns (pair-14 data-out) the
|
||
// target-general checkers/sanitizer run on ANY →target output. A target with no file yields an empty value
|
||
// (HasData()==false → the consumers stay inert). Values are grouped per category in AUTHORED ORDER.
|
||
type TargetChecks struct{ byKey map[string][]string }
|
||
|
||
// List returns the authored-order values for a category ("" categories → nil).
|
||
func (t TargetChecks) List(key string) []string { return t.byKey[key] }
|
||
|
||
// HasData reports whether this target ships any checker data (used to gate the target-general checkers so a
|
||
// target without data flags nothing rather than mis-firing).
|
||
func (t TargetChecks) HasData() bool { return len(t.byKey) > 0 }
|
||
|
||
var (
|
||
targetChecksOnce sync.Once
|
||
targetChecksBy map[string]TargetChecks
|
||
)
|
||
|
||
// TargetChecksFor returns the readability-checker data for a TARGET language, parsed once from the embedded
|
||
// per-target file. The target REGISTRY is DERIVED from the embed plane: every `data/target-<tgt>.txt` file
|
||
// registers target `<tgt>` (no Go map to keep in sync — add a target by dropping its file into the go:embed
|
||
// manifest). Unknown/absent target → empty TargetChecks (HasData()==false → its consumers stay inert).
|
||
// Panics on a corrupt embed.
|
||
func TargetChecksFor(tgt string) TargetChecks {
|
||
targetChecksOnce.Do(func() {
|
||
targetChecksBy = map[string]TargetChecks{}
|
||
for _, f := range embeddedDataFiles() {
|
||
name, ok := strings.CutPrefix(f.name, "target-")
|
||
if !ok {
|
||
continue
|
||
}
|
||
lng, ok := strings.CutSuffix(name, ".txt")
|
||
if !ok || lng == "" {
|
||
continue
|
||
}
|
||
byKey, err := parseOrderedByCategory(f.data)
|
||
if err != nil {
|
||
panic(fmt.Sprintf("lang: embedded data/%s is corrupt: %v", f.name, err))
|
||
}
|
||
targetChecksBy[lng] = TargetChecks{byKey: byKey}
|
||
}
|
||
})
|
||
return targetChecksBy[tgt]
|
||
}
|
||
|
||
// parseOrderedByCategory reads `category<TAB>value` lines into category → ordered values. Value is VERBATIM
|
||
// (a regex may carry a trailing metachar), so only the whole line's \r is stripped; blank/`#`-comment lines
|
||
// (by their trimmed form) are dropped.
|
||
func parseOrderedByCategory(b []byte) (map[string][]string, error) {
|
||
out := map[string][]string{}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
line := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(line) == "" || strings.HasPrefix(strings.TrimSpace(line), "#") {
|
||
continue
|
||
}
|
||
f := strings.SplitN(line, "\t", 2)
|
||
if len(f) != 2 || strings.TrimSpace(f[0]) == "" || f[1] == "" {
|
||
return nil, fmt.Errorf("line %d: want `category<TAB>value`, got %q", i+1, line)
|
||
}
|
||
out[f[0]] = append(out[f[0]], f[1])
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// CJKSection is the universal chapter/section-numeral inventory shared by the ingest chapter-splitter and
|
||
// the chunker heading rule (pair-14 §4 + addendum-A: ONE source, no ingest↔chunker byte-duplication). The
|
||
// heading-numeral membership test derives from Digit ∪ Unit ∪ Zero — nothing is stored twice.
|
||
type CJKSection struct {
|
||
Digit map[rune]int // 一→1 … 九→9 (incl. 两/兩→2)
|
||
Unit map[rune]int // 十→10 百→100 千→1000
|
||
Zero map[rune]bool // 〇 零 (positional zero: value*10 in a numeral run)
|
||
// ChapterUnit membership + ChapterUnitOrdered (authored order) — ingest's detectChapterUnit iterates in
|
||
// order and breaks a tie by first-seen, so the ORDER is behaviour, not just a set.
|
||
ChapterUnit map[rune]bool
|
||
ChapterUnitOrdered []rune
|
||
// Magnitude: the myriad-scale big units (万/億/兆 → value) for the number-magnitude gate. int64 (10^12
|
||
// overflows int32). ONE source with Digit/Unit — the gate no longer keeps its own CJK numeral switch.
|
||
Magnitude map[rune]int64
|
||
// ChapterMarker: the rune(s) that OPEN a section header (第N章), sorted. Data, not a literal in the
|
||
// ingest regex — the same reason the numerals beside it are data.
|
||
ChapterMarker []rune
|
||
headingRunes string // sorted Zero∪Digit∪Unit, cached for the ingest chapter-numeral regex class
|
||
}
|
||
|
||
// ChapterMarkerClass returns the section-opening runes as a sorted string for use INSIDE a regex character
|
||
// class (the ingest chapter-numeral pattern). None of the runes are class metacharacters, so no escaping.
|
||
func (c *CJKSection) ChapterMarkerClass() string { return string(c.ChapterMarker) }
|
||
|
||
// IsNumeralRune reports whether r can be part of a CJK numeral EXPRESSION (Zero∪Digit∪Unit∪Magnitude) — the
|
||
// alphabet of a magnitude candidate run (the gate adds ASCII digits itself). BigUnit returns a big unit's
|
||
// value (万/億/兆). Both read the single CJKSection source (pair-14 data-out dedup).
|
||
func (c *CJKSection) IsNumeralRune(r rune) bool {
|
||
if c.IsHeadingNumeral(r) {
|
||
return true
|
||
}
|
||
_, ok := c.Magnitude[r]
|
||
return ok
|
||
}
|
||
func (c *CJKSection) BigUnit(r rune) (int64, bool) { v, ok := c.Magnitude[r]; return v, ok }
|
||
|
||
// IsHeadingNumeral reports whether r can be part of a CJK chapter-number run (Zero ∪ Digit ∪ Unit). Arabic
|
||
// digits are handled by the caller (they are not language data).
|
||
func (c *CJKSection) IsHeadingNumeral(r rune) bool {
|
||
if c.Zero[r] {
|
||
return true
|
||
}
|
||
if _, ok := c.Digit[r]; ok {
|
||
return true
|
||
}
|
||
_, ok := c.Unit[r]
|
||
return ok
|
||
}
|
||
|
||
// HeadingNumeralClass returns the CJK heading-numeral runes (Zero∪Digit∪Unit) as a sorted string, for use
|
||
// INSIDE a regex character class (the ingest chapter-numeral pattern). Sorted for a stable pattern; order
|
||
// inside a class is immaterial to matching. None of the runes are class metacharacters, so no escaping.
|
||
func (c *CJKSection) HeadingNumeralClass() string { return c.headingRunes }
|
||
|
||
// Terminators is the engine's sentence-terminator alphabet, in two classes: CJK marks split a sentence
|
||
// ZERO-WIDTH (CJK prose has no inter-sentence space), generic marks split only before whitespace. It is
|
||
// the single source both the source chunker and the coverage gate read — they used to keep a private
|
||
// copy of the same two switches each.
|
||
type Terminators struct {
|
||
cjk map[rune]bool
|
||
generic map[rune]bool
|
||
}
|
||
|
||
// IsTerminator reports whether r ends a sentence in EITHER class.
|
||
func (t *Terminators) IsTerminator(r rune) bool { return t.cjk[r] || t.generic[r] }
|
||
|
||
// IsCJK reports whether r is a zero-width-splitting CJK terminator.
|
||
func (t *Terminators) IsCJK(r rune) bool { return t.cjk[r] }
|
||
|
||
var (
|
||
termOnce sync.Once
|
||
termVal *Terminators
|
||
termErr error
|
||
)
|
||
|
||
// DefaultTerminators returns the process-wide terminator classes, parsed once from the embedded file.
|
||
// Panics on a corrupt embed (an in-repo data file — a build/test failure, never user input).
|
||
func DefaultTerminators() *Terminators {
|
||
termOnce.Do(func() { termVal, termErr = parseTerminators(mustEmbed("data/terminator.txt")) })
|
||
if termErr != nil {
|
||
panic(fmt.Sprintf("lang: embedded data/terminator.txt is corrupt: %v", termErr))
|
||
}
|
||
return termVal
|
||
}
|
||
|
||
func parseTerminators(b []byte) (*Terminators, error) {
|
||
t := &Terminators{cjk: map[rune]bool{}, generic: map[rune]bool{}}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
line := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(line) == "" || strings.HasPrefix(strings.TrimSpace(line), "#") {
|
||
continue
|
||
}
|
||
f := strings.Split(line, "\t")
|
||
if len(f) != 2 {
|
||
return nil, fmt.Errorf("line %d: want `category<TAB>rune`, got %q", i+1, line)
|
||
}
|
||
r := []rune(f[1])
|
||
if len(r) != 1 {
|
||
return nil, fmt.Errorf("line %d: key %q must be a single rune", i+1, f[1])
|
||
}
|
||
switch f[0] {
|
||
case "cjk":
|
||
t.cjk[r[0]] = true
|
||
case "generic":
|
||
t.generic[r[0]] = true
|
||
default:
|
||
return nil, fmt.Errorf("line %d: unknown category %q (want cjk|generic)", i+1, f[0])
|
||
}
|
||
}
|
||
if len(t.cjk) == 0 || len(t.generic) == 0 {
|
||
return nil, fmt.Errorf("terminator needs non-empty cjk and generic sections")
|
||
}
|
||
return t, nil
|
||
}
|
||
|
||
var (
|
||
cjkOnce sync.Once
|
||
cjkVal *CJKSection
|
||
cjkErr error
|
||
)
|
||
|
||
// DefaultCJKSection returns the process-wide CJK section-numeral data, parsed once from the embedded file.
|
||
// It panics on a corrupt embed (a build-time asset the tests exercise, so a bad edit fails the suite, never
|
||
// production silently) — the parse is deterministic and input-free.
|
||
func DefaultCJKSection() *CJKSection {
|
||
cjkOnce.Do(func() { cjkVal, cjkErr = parseCJKSection(mustEmbed("data/cjk-section.txt")) })
|
||
if cjkErr != nil {
|
||
panic(fmt.Sprintf("lang: embedded cjk-section.txt is corrupt: %v", cjkErr))
|
||
}
|
||
return cjkVal
|
||
}
|
||
|
||
func parseCJKSection(b []byte) (*CJKSection, error) {
|
||
c := &CJKSection{Digit: map[rune]int{}, Unit: map[rune]int{}, Zero: map[rune]bool{}, ChapterUnit: map[rune]bool{}, Magnitude: map[rune]int64{}}
|
||
valueRune := func(i int, f []string) (rune, int, error) {
|
||
if len(f) != 3 {
|
||
return 0, 0, fmt.Errorf("line %d: %s wants `%s<TAB>rune<TAB>value`, got %q", i+1, f[0], f[0], strings.Join(f, "\t"))
|
||
}
|
||
r := []rune(f[1])
|
||
if len(r) != 1 {
|
||
return 0, 0, fmt.Errorf("line %d: key %q must be a single rune", i+1, f[1])
|
||
}
|
||
v, err := strconv.Atoi(strings.TrimSpace(f[2]))
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("line %d: value %q: %w", i+1, f[2], err)
|
||
}
|
||
return r[0], v, nil
|
||
}
|
||
setRune := func(i int, f []string) (rune, error) {
|
||
if len(f) != 2 {
|
||
return 0, fmt.Errorf("line %d: %s wants `%s<TAB>rune`, got %q", i+1, f[0], f[0], strings.Join(f, "\t"))
|
||
}
|
||
r := []rune(f[1])
|
||
if len(r) != 1 {
|
||
return 0, fmt.Errorf("line %d: key %q must be a single rune", i+1, f[1])
|
||
}
|
||
return r[0], nil
|
||
}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
t := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
|
||
continue
|
||
}
|
||
f := strings.Split(t, "\t")
|
||
switch f[0] {
|
||
case "digit":
|
||
r, v, err := valueRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c.Digit[r] = v
|
||
case "unit":
|
||
r, v, err := valueRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c.Unit[r] = v
|
||
case "zero":
|
||
r, err := setRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c.Zero[r] = true
|
||
case "chapter_marker":
|
||
r, err := setRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c.ChapterMarker = append(c.ChapterMarker, r)
|
||
case "chapter_unit":
|
||
r, err := setRune(i, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !c.ChapterUnit[r] {
|
||
c.ChapterUnit[r] = true
|
||
c.ChapterUnitOrdered = append(c.ChapterUnitOrdered, r)
|
||
}
|
||
case "magnitude":
|
||
if len(f) != 3 {
|
||
return nil, fmt.Errorf("line %d: magnitude wants `magnitude<TAB>rune<TAB>value`, got %q", i+1, t)
|
||
}
|
||
r := []rune(f[1])
|
||
if len(r) != 1 {
|
||
return nil, fmt.Errorf("line %d: magnitude key %q must be a single rune", i+1, f[1])
|
||
}
|
||
v, err := strconv.ParseInt(strings.TrimSpace(f[2]), 10, 64)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("line %d: magnitude value %q: %w", i+1, f[2], err)
|
||
}
|
||
c.Magnitude[r[0]] = v
|
||
default:
|
||
return nil, fmt.Errorf("line %d: unknown category %q (want digit|unit|zero|chapter_marker|chapter_unit|magnitude)", i+1, f[0])
|
||
}
|
||
}
|
||
if len(c.Digit) == 0 || len(c.Unit) == 0 || len(c.Zero) == 0 || len(c.ChapterUnit) == 0 || len(c.Magnitude) == 0 || len(c.ChapterMarker) == 0 {
|
||
return nil, fmt.Errorf("cjk-section needs non-empty digit, unit, zero, chapter_marker, chapter_unit and magnitude sections")
|
||
}
|
||
sort.Slice(c.ChapterMarker, func(i, j int) bool { return c.ChapterMarker[i] < c.ChapterMarker[j] })
|
||
// Cache the sorted heading-numeral class (Zero∪Digit∪Unit) for the ingest regex.
|
||
var runes []rune
|
||
for r := range c.Zero {
|
||
runes = append(runes, r)
|
||
}
|
||
for r := range c.Digit {
|
||
runes = append(runes, r)
|
||
}
|
||
for r := range c.Unit {
|
||
runes = append(runes, r)
|
||
}
|
||
sort.Slice(runes, func(i, j int) bool { return runes[i] < runes[j] })
|
||
c.headingRunes = string(runes)
|
||
return c, nil
|
||
}
|
||
|
||
// InjectionTexts is the TARGET-language glossary/injection wire-text (pair-14 §2): the Russian headers and
|
||
// annotations the memory renderers emit into a request. TARGET data (a Russian header for a →ru book), so
|
||
// the renderers gate on HasData — a target with no injection texts (e.g. →en) injects NOTHING rather than a
|
||
// stray Russian block. The ja→ru golden renders these with NO book pack, so they are engine/target data.
|
||
type InjectionTexts struct {
|
||
GlossaryHeader string // "ГЛОССАРИЙ (…):" — introduces the draft glossary block
|
||
EditorHeader string // "КАНОНИЧЕСКИЕ ПЕРЕВОДЫ …:" — introduces the editor constraint block
|
||
// EditorUnverifiedHeader and UnverifiedMarker are RETIRED wire text (D39.104 п.2, backlog row 134):
|
||
// the editor's two sections are one law block now and the ⟨проверить⟩ marker has left the wire, so
|
||
// nothing renders either of these. They stay DECLARED because their rows stay in the embedded data:
|
||
// `injection.txt` is folded byte-for-byte into lang.EmbeddedVersion(), which rides inside every unit
|
||
// id through the manifest's cut tag, so deleting the two lines would re-mint the unit ids of every
|
||
// book — and parseInjection refuses an unknown key, so the fields cannot go while the rows stay. Both
|
||
// leave together at the next touch of the embedded plane, in a window that already re-cuts.
|
||
EditorUnverifiedHeader string
|
||
UnverifiedMarker string
|
||
GenderMale string // " (муж. — …)" DC3 gender directive (leading space significant)
|
||
GenderFemale string // " (жен. — …)"
|
||
GenderHidden string // " (пол СКРЫТ …)"
|
||
GenderNeuter string // " (ср. — …)" — a neuter entity/creature (row 84); "" for a target with no row
|
||
}
|
||
|
||
// HasData reports whether the target has injection texts (its header is present). The renderers use it to
|
||
// gate: a target without texts injects nothing, so a non-ru book never gets a Russian glossary block.
|
||
func (t InjectionTexts) HasData() bool { return t.GlossaryHeader != "" }
|
||
|
||
var (
|
||
injectionOnce sync.Once
|
||
injectionBy map[string]*InjectionTexts
|
||
injectionErr error
|
||
)
|
||
|
||
// InjectionTextsFor returns the injection wire-text for a TARGET language (pair-14 §2), parsed once from the
|
||
// embedded file. A target with no rows returns a zero value (HasData()==false → no injection). Panics on a
|
||
// corrupt embed. VALUE bytes are verbatim (leading spaces in gender_*/unverified_marker are significant).
|
||
func InjectionTextsFor(targetLang string) InjectionTexts {
|
||
injectionOnce.Do(func() { injectionBy, injectionErr = parseInjection(mustEmbed("data/injection.txt")) })
|
||
if injectionErr != nil {
|
||
panic(fmt.Sprintf("lang: embedded injection.txt is corrupt: %v", injectionErr))
|
||
}
|
||
if t, ok := injectionBy[targetLang]; ok {
|
||
return *t
|
||
}
|
||
return InjectionTexts{}
|
||
}
|
||
|
||
func parseInjection(b []byte) (map[string]*InjectionTexts, error) {
|
||
out := map[string]*InjectionTexts{}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
line := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(line) == "" || strings.HasPrefix(strings.TrimSpace(line), "#") {
|
||
continue
|
||
}
|
||
f := strings.SplitN(line, "\t", 3) // value (f[2]) VERBATIM — leading spaces are significant
|
||
if len(f) != 3 || strings.TrimSpace(f[0]) == "" || strings.TrimSpace(f[1]) == "" {
|
||
return nil, fmt.Errorf("line %d: want `target<TAB>key<TAB>value`, got %q", i+1, line)
|
||
}
|
||
tgt, key, val := f[0], f[1], f[2]
|
||
if out[tgt] == nil {
|
||
out[tgt] = &InjectionTexts{}
|
||
}
|
||
fld, known := injectionFields[key]
|
||
if !known {
|
||
return nil, fmt.Errorf("line %d: unknown injection key %q", i+1, key)
|
||
}
|
||
fld.set(out[tgt], val)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// The injection DATA KEYS — the first column of a row in data/injection.txt.
|
||
//
|
||
// They are EXPORTED because the write path requires some of them before it buys anything: a target whose
|
||
// rows are missing renders its blocks into nothing at all, and the refusal that stops such a run has to
|
||
// name the rows an author must add (pipeline: roleInjectionKeys, openRunner). A guard naming its own string
|
||
// literal would be a second vocabulary, free to drift from the one the parser accepts.
|
||
const (
|
||
InjectionKeyGlossaryHeader = "glossary_header"
|
||
InjectionKeyEditorHeader = "editor_header"
|
||
InjectionKeyGenderMale = "gender_male"
|
||
InjectionKeyGenderFemale = "gender_female"
|
||
InjectionKeyGenderHidden = "gender_hidden"
|
||
InjectionKeyGenderNeuter = "gender_neuter"
|
||
|
||
// RETIRED wire text (D39.104 п.2, backlog row 134): nothing renders these two, and the fields exist only
|
||
// because their ROWS may not leave the file yet (see EditorUnverifiedHeader). They are therefore keys the
|
||
// DATA may carry and a guard must never REQUIRE — demanding them would make a new target author dead text.
|
||
// Unexported for exactly that reason: a caller outside this package cannot name them.
|
||
injectionKeyEditorUnverifiedHeader = "editor_unverified_header"
|
||
injectionKeyUnverifiedMarker = "unverified_marker"
|
||
)
|
||
|
||
// injectionField binds a data key to the field it authors and to the value that field carries. ONE table,
|
||
// read by the parser and by every caller asking «does this target author that row?»: the key a file may
|
||
// carry and the key a guard may require are the same vocabulary by construction, so a row added to the
|
||
// parser is visible to the guard on the same edit.
|
||
type injectionField struct {
|
||
set func(*InjectionTexts, string)
|
||
get func(InjectionTexts) string
|
||
}
|
||
|
||
var injectionFields = map[string]injectionField{
|
||
InjectionKeyGlossaryHeader: {func(t *InjectionTexts, v string) { t.GlossaryHeader = v }, func(t InjectionTexts) string { return t.GlossaryHeader }},
|
||
InjectionKeyEditorHeader: {func(t *InjectionTexts, v string) { t.EditorHeader = v }, func(t InjectionTexts) string { return t.EditorHeader }},
|
||
InjectionKeyGenderMale: {func(t *InjectionTexts, v string) { t.GenderMale = v }, func(t InjectionTexts) string { return t.GenderMale }},
|
||
InjectionKeyGenderFemale: {func(t *InjectionTexts, v string) { t.GenderFemale = v }, func(t InjectionTexts) string { return t.GenderFemale }},
|
||
InjectionKeyGenderHidden: {func(t *InjectionTexts, v string) { t.GenderHidden = v }, func(t InjectionTexts) string { return t.GenderHidden }},
|
||
InjectionKeyGenderNeuter: {func(t *InjectionTexts, v string) { t.GenderNeuter = v }, func(t InjectionTexts) string { return t.GenderNeuter }},
|
||
injectionKeyEditorUnverifiedHeader: {func(t *InjectionTexts, v string) { t.EditorUnverifiedHeader = v },
|
||
func(t InjectionTexts) string { return t.EditorUnverifiedHeader }},
|
||
injectionKeyUnverifiedMarker: {func(t *InjectionTexts, v string) { t.UnverifiedMarker = v },
|
||
func(t InjectionTexts) string { return t.UnverifiedMarker }},
|
||
}
|
||
|
||
// Authored reports whether this target carries a non-empty value for an injection key. An unknown key is
|
||
// NOT authored: a caller asking for a row this file cannot carry is asking a question with one honest
|
||
// answer, and inventing `true` would let a guard pass on a key nobody parses.
|
||
func (t InjectionTexts) Authored(key string) bool {
|
||
fld, known := injectionFields[key]
|
||
return known && fld.get(t) != ""
|
||
}
|
||
|
||
// MissingKeys reports which of the given keys this target does not author, in the order given (so a
|
||
// refusal lists them the same way twice). nil when every key is authored.
|
||
func (t InjectionTexts) MissingKeys(keys []string) []string {
|
||
var missing []string
|
||
for _, k := range keys {
|
||
if !t.Authored(k) {
|
||
missing = append(missing, k)
|
||
}
|
||
}
|
||
return missing
|
||
}
|
||
|
||
// genderDirectiveKeys are the three directives a target with grammatical gender has to author TOGETHER:
|
||
// male and female are the two forms, and hidden is the instruction for a character whose sex the book has
|
||
// not revealed yet — the owner's own wording (D39.254 п.3), which a row silently loses if the key is absent.
|
||
//
|
||
// gender_neuter is deliberately NOT here: it is documented optional ("" for a target with no row), because
|
||
// not every target language has a neuter to name.
|
||
var genderDirectiveKeys = []string{InjectionKeyGenderMale, InjectionKeyGenderFemale, InjectionKeyGenderHidden}
|
||
|
||
// GenderKeyGaps reports a PARTIAL gender directive set — the directives this target leaves unauthored while
|
||
// authoring at least one other. Empty in both healthy cases: a target that authors all three, and a target
|
||
// that authors none of them (a language without grammatical gender legitimately says nothing here, and its
|
||
// rows render without a directive exactly as they should).
|
||
//
|
||
// ⚠ It is a WARNING's evidence and not a refusal's: the bank still reaches the model, and only the rows of
|
||
// the missing sex lose their directive. The line stops holding the moment a directive becomes load-bearing
|
||
// for something other than wording — then the gap is a mechanism that cannot work, and it belongs with the
|
||
// keys in roleInjectionKeys instead.
|
||
func (t InjectionTexts) GenderKeyGaps() []string {
|
||
gaps := t.MissingKeys(genderDirectiveKeys)
|
||
if len(gaps) == len(genderDirectiveKeys) {
|
||
return nil // authors none: a gender-free target, not a half-authored one
|
||
}
|
||
return gaps
|
||
}
|
||
|
||
var (
|
||
refusalOnce sync.Once
|
||
refusalVal []string
|
||
)
|
||
|
||
// RefusalPatterns returns the universal refusal-blacklist regex patterns (pair-14 §3), parsed once from the
|
||
// embedded sectioned file in authored order. Engine safety data: a model can refuse in any language for any
|
||
// pair, so this is not book-pack data — the caller joins the patterns into one case-insensitive regex. Each
|
||
// non-comment line is ONE pattern, taken verbatim (a regex may contain leading/trailing metacharacters, so
|
||
// it is NOT trimmed beyond the line's \r). Panics on a corrupt embed.
|
||
func RefusalPatterns() []string {
|
||
refusalOnce.Do(func() { refusalVal = embeddedLines(mustEmbed("data/refusal.txt")) })
|
||
return refusalVal
|
||
}
|
||
|
||
// embeddedLines returns the non-comment, non-blank lines of an embedded file VERBATIM (only the trailing \r
|
||
// is stripped; the content is NOT trimmed — significant for a regex pattern). A blank/`#`-comment line is
|
||
// dropped by its TRIMMED form, but a kept line keeps its own leading/trailing bytes.
|
||
func embeddedLines(b []byte) []string {
|
||
var out []string
|
||
for _, raw := range strings.Split(string(b), "\n") {
|
||
s := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(s) == "" || strings.HasPrefix(strings.TrimSpace(s), "#") {
|
||
continue
|
||
}
|
||
out = append(out, s)
|
||
}
|
||
return out
|
||
}
|
||
|
||
var (
|
||
abbrevOnce sync.Once
|
||
abbrevBy map[string]map[string]bool
|
||
)
|
||
|
||
// SentenceAbbrev returns the lower-cased sentence-splitter abbreviation SET for a SOURCE language (pair-14
|
||
// §4), parsed once from the embedded sectioned file. An unknown/absent language returns an empty set (a
|
||
// source with no ASCII-period abbreviations — e.g. a CJK source using 。). Panics on a corrupt embed.
|
||
func SentenceAbbrev(srcLang string) map[string]bool {
|
||
abbrevOnce.Do(func() {
|
||
var err error
|
||
abbrevBy, err = parseSectionedSet(mustEmbed("data/sentence-abbrev.txt"))
|
||
if err != nil {
|
||
panic(fmt.Sprintf("lang: embedded sentence-abbrev.txt is corrupt: %v", err))
|
||
}
|
||
})
|
||
if m, ok := abbrevBy[srcLang]; ok {
|
||
return m
|
||
}
|
||
return map[string]bool{}
|
||
}
|
||
|
||
// parseSectionedSet reads `key<TAB>member` lines into per-key sets (keys lower-cased on read is NOT done
|
||
// here — the key is the language tag; members are stored verbatim). Used for the sentence-abbrev table.
|
||
func parseSectionedSet(b []byte) (map[string]map[string]bool, error) {
|
||
out := map[string]map[string]bool{}
|
||
for i, raw := range strings.Split(string(b), "\n") {
|
||
t := strings.TrimRight(raw, "\r")
|
||
if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
|
||
continue
|
||
}
|
||
f := strings.Split(t, "\t")
|
||
if len(f) != 2 || strings.TrimSpace(f[0]) == "" || strings.TrimSpace(f[1]) == "" {
|
||
return nil, fmt.Errorf("line %d: want `key<TAB>member`, got %q", i+1, t)
|
||
}
|
||
key := strings.TrimSpace(f[0])
|
||
if out[key] == nil {
|
||
out[key] = map[string]bool{}
|
||
}
|
||
out[key][strings.TrimSpace(f[1])] = true
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func mustEmbed(name string) []byte {
|
||
b, err := embeddedFS.ReadFile(name)
|
||
if err != nil {
|
||
panic(fmt.Sprintf("lang: missing embedded asset %q: %v", name, err))
|
||
}
|
||
return b
|
||
}
|