263 lines
13 KiB
Go
263 lines
13 KiB
Go
// Package text is the pipeline's text substrate: the canonical forms a source or target string is
|
||
// reduced to before anything compares, matches or hashes it.
|
||
//
|
||
// It exists because three subsystems must agree on those forms EXACTLY — the memory bank matches and
|
||
// post-checks with them, the miner normalizes its candidate space with them, the cheap checkers tokenize
|
||
// with them — and a private copy in any of the three would be a silent match failure, not a compile
|
||
// error. Everything here is pure, deterministic and free of language policy: no pair data, no book
|
||
// canon, no configuration. The normalization ARTIFACT is versioned as a whole (NormVersion) and folded
|
||
// into the job snapshot, so changing a rule or the embedded trad→simp table is a loud --resnapshot
|
||
// rather than a quiet re-verdict on already-paid chunks.
|
||
package text
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
_ "embed"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"golang.org/x/text/unicode/norm"
|
||
)
|
||
|
||
// norm.go: the NORMALIZATION artifact behind every source/target match in the pipeline
|
||
// (registry A4 — the most likely "silently empty" bug: a glossary key is present in the
|
||
// chunk but in a different orthographic form, so the record silently fails to match).
|
||
// It is shared substrate: the memory bank matches and post-checks with it, the miner
|
||
// normalizes its candidate space with it, and the seed loader keys with it. Two pure,
|
||
// deterministic, VERSIONED normalizers, applied SYMMETRICALLY to both sides of every
|
||
// comparison:
|
||
//
|
||
// NormalizeSourceKey — zh/ja source surfaces (glossary keys/aliases AND chunk text):
|
||
// NFKC (fold half/full-width, e.g. ﾃ→テ, Ｑ→Q) → trad→simp (per-char, the
|
||
// embedded table) → katakana→hiragana fold → Unicode lower. So a key stored in
|
||
// Simplified still matches a Traditional source surface (魯鎮 ↔ 鲁镇), a full-width
|
||
// Ｑ matches a half-width Q, and スズキ matches すずき.
|
||
// NormalizeTargetForm — Russian dst forms (stored decl forms AND model output) for
|
||
// the post-check substring test: NFC → Unicode lower → ё→е fold.
|
||
//
|
||
// The whole artifact is versioned by NormVersion(), which is FOLDED INTO the
|
||
// snapshot via memoryVersion() (pipeline/snapshot.go): a change to the algorithm OR to the
|
||
// embedded trad→simp table is a loud --resnapshot, never a silent match-behaviour
|
||
// change on already-checkpointed chunks. Determinism is load-bearing — the injected
|
||
// glossary block is a pure function of (frozen glossary, normalized chunk), and it
|
||
// enters request_hash via the rendered messages.
|
||
|
||
//go:embed data/trad2simp.txt
|
||
var trad2simpRaw string
|
||
|
||
// memoryNormAlgoVersion tags the normalization ALGORITHM (the pipeline of steps and
|
||
// their order). Bump it on an algorithm change. The trad→simp TABLE content is
|
||
// versioned SEPARATELY by hashing trad2simpRaw into memoryNormVersion below, so
|
||
// editing/completing the data file also invalidates — drift-proof (D8): you cannot
|
||
// forget to bump a version when you change the table, because the table's bytes ARE
|
||
// the version.
|
||
// Ш-2 (D39.16, folded here in R1): unicode.Version is woven into the algorithm tag so a Unicode-table
|
||
// bump (a go toolchain upgrade — NFKC/NFC via x/text, unicode.Is(Cf), the kana range, SignificantLen's
|
||
// unicode.Han/Cyrillic predicates) changes this version → the snapshot moves → a LOUD --resnapshot,
|
||
// never a silent match-behaviour change on already-checkpointed chunks. TRIPWIRE: do NOT bump the
|
||
// toolchain past the pinned Unicode edition without a --resnapshot (we are on go1.26.4 / Unicode 15.0.0).
|
||
//
|
||
// R1-FL-C: golang.org/x/text/norm carries its OWN Unicode edition INDEPENDENT of stdlib unicode.Version
|
||
// (its NFKC/NFC tables ship in the module, not the toolchain), so a standalone `go get -u golang.org/x/text`
|
||
// could silently shift NormalizeSourceKey's NFKC output → a re-verdict on the post-check WITHOUT moving the
|
||
// snapshot. Weaving xTextVersion in makes an x/text bump a loud --resnapshot too. It is a PINNED constant
|
||
// (debug.ReadBuildInfo returns an empty x/text version under `go test`, where the golden is captured, so a
|
||
// build-info fold would be non-deterministic) DRIFT-CHECKED against go.mod by TestMemnormXTextVersionPin:
|
||
// a go.mod bump that forgets this constant fails that test loudly — the tripwire enforced by mechanism.
|
||
const xTextVersion = "v0.40.0" // MUST equal go.mod's golang.org/x/text (asserted by TestMemnormXTextVersionPin); a bump = --resnapshot
|
||
const memoryNormAlgoVersion = "memnorm-v3-nfkc+apos+stripignorable+trad+kana+lower/nfc+dash+apos+stripignorable+lower+yofold+u" + unicode.Version + "+xtext" + xTextVersion
|
||
|
||
var (
|
||
// trad2simp is the parsed Traditional→Simplified single-char map (data/trad2simp.txt).
|
||
trad2simp map[rune]rune
|
||
// memoryNormVersion is the snapshot-load-bearing version of the entire
|
||
// normalization artifact: the algorithm tag + a content hash of the embedded
|
||
// trad→simp table. memoryVersion() (pipeline/snapshot.go) folds it into the job snapshot.
|
||
memoryNormVersion string
|
||
)
|
||
|
||
func init() {
|
||
trad2simp = parseTradTable(trad2simpRaw)
|
||
h := sha256.Sum256([]byte(memoryNormAlgoVersion + "\x00" + trad2simpRaw))
|
||
memoryNormVersion = memoryNormAlgoVersion + "-" + hex.EncodeToString(h[:])[:12]
|
||
}
|
||
|
||
// NormVersion returns the version of the whole normalization artifact (algorithm tag +
|
||
// content hash of the embedded trad→simp table). It is a function, not an exported var,
|
||
// so no caller can reassign the value the job snapshot folds.
|
||
func NormVersion() string { return memoryNormVersion }
|
||
|
||
// parseTradTable parses the embedded "<trad> <simp>" table. Comment ('#') and blank
|
||
// lines are ignored; a non-empty content line that is NOT exactly two
|
||
// single-rune whitespace-separated fields is a corrupt table → panic at init (a
|
||
// loud build/test failure, caught the moment any test in the package runs, rather
|
||
// than a silently truncated map that would re-open the A4 hole). The table is
|
||
// authored in-repo and asserted by memnorm_test.go, so a panic here means the data
|
||
// file was corrupted, not bad user input.
|
||
func parseTradTable(raw string) map[rune]rune {
|
||
m := make(map[rune]rune, 1024)
|
||
for i, line := range strings.Split(raw, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" || strings.HasPrefix(line, "#") {
|
||
continue
|
||
}
|
||
fields := strings.Fields(line)
|
||
if len(fields) != 2 {
|
||
panic(fmt.Sprintf("text: trad2simp table line %d: want 2 fields, got %d (%q)", i+1, len(fields), line))
|
||
}
|
||
tr := []rune(fields[0])
|
||
si := []rune(fields[1])
|
||
if len(tr) != 1 || len(si) != 1 {
|
||
panic(fmt.Sprintf("text: trad2simp table line %d: each side must be a single rune (%q → %q)", i+1, fields[0], fields[1]))
|
||
}
|
||
// A later duplicate wins deterministically (the file may list a char under
|
||
// two families); assert the mapping is consistent to catch a typo where the
|
||
// same trad char is mapped to two different simp chars.
|
||
if prev, ok := m[tr[0]]; ok && prev != si[0] {
|
||
panic(fmt.Sprintf("text: trad2simp table line %d: %q maps to both %q and %q", i+1, fields[0], string(prev), fields[1]))
|
||
}
|
||
m[tr[0]] = si[0]
|
||
}
|
||
return m
|
||
}
|
||
|
||
// NormalizeSourceKey normalizes a zh/ja SOURCE surface (a glossary key/alias OR the
|
||
// chunk text) for exact matching. Pure and deterministic; the pipeline is
|
||
// NFKC → drop default-ignorables → trad→simp (per rune) → katakana→hiragana → Unicode
|
||
// lower. Applied to BOTH keys and text so orthographic variants match (A4). Dropping
|
||
// default-ignorable/format code points (zero-width space/joiner, BOM, variation
|
||
// selectors) closes the re-opened A4 hole where a key 渡邊 silently misses a chunk
|
||
// surface 渡<ZWSP>邊 (common in scraped/EPUB CJK). Kana folding mirrors the eval spec
|
||
// (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to hiragana;
|
||
// the prolonged-sound mark ー (U+30FC) and half-width forms are already handled (the
|
||
// latter by NFKC).
|
||
func NormalizeSourceKey(s string) string {
|
||
s = norm.NFKC.String(s)
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
for _, r := range s {
|
||
if isApostropheVariant(r) {
|
||
r = '\'' // typographic apostrophe (O’Brien) folds to ASCII so a key matches either form
|
||
}
|
||
if isIgnorableForMatch(r) {
|
||
continue // zero-width / format / variation selector: invisible, never part of the key
|
||
}
|
||
if m, ok := trad2simp[r]; ok {
|
||
r = m
|
||
}
|
||
if r >= 0x30A1 && r <= 0x30F6 { // katakana → hiragana
|
||
r -= 0x60
|
||
}
|
||
b.WriteRune(unicode.ToLower(r))
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// NormalizeTargetForm normalizes a Russian TARGET form (a stored decl form OR the
|
||
// model output) for the post-check whole-word test. Applied SYMMETRICALLY to both
|
||
// sides so a correctly-rendered term is NEVER a false MISS (over-flagging is the
|
||
// failure mode research/14 §2 warns about). Steps, each closing a self-review finding:
|
||
// - NFC;
|
||
// - dash/hyphen variants (en-dash, NB-hyphen, minus, soft hyphen, …) → ASCII '-', so
|
||
// «А–кью»/«А‑кью» match a stored «А-кью» (self-review #8);
|
||
// - drop default-ignorable/format code points (zero-width space/joiner, word joiner,
|
||
// BOM, variation selectors) that survive NFC, so a stored «вана» is not a false MISS
|
||
// against model output «ва<WJ>на» (over-flag; the soft hyphen is folded above, kept);
|
||
// - runs of ANY Unicode whitespace (newline, NBSP U+00A0, ideographic space, …) →
|
||
// one ASCII space, then trim, so a multi-word name wrapped across a line break
|
||
// («Бородатого\nВана») matches a stored «Бородатого Вана» (self-review #2);
|
||
// - Unicode lower + ё→е fold (the ё-ification е/ё variation).
|
||
//
|
||
// Pure and deterministic.
|
||
func NormalizeTargetForm(s string) string {
|
||
s = norm.NFC.String(s)
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
prevSpace := false
|
||
for _, r := range s {
|
||
if isDashVariant(r) {
|
||
r = '-'
|
||
}
|
||
if isApostropheVariant(r) {
|
||
r = '\'' // symmetric with the source side (д’Артаньян) — no false post-check MISS on a typographic apostrophe
|
||
}
|
||
if isIgnorableForMatch(r) {
|
||
continue // drop zero-width/format/VS (soft hyphen already folded to '-' above)
|
||
}
|
||
if unicode.IsSpace(r) {
|
||
if !prevSpace {
|
||
b.WriteByte(' ')
|
||
prevSpace = true
|
||
}
|
||
continue
|
||
}
|
||
prevSpace = false
|
||
r = unicode.ToLower(r)
|
||
if r == 'ё' {
|
||
r = 'е'
|
||
}
|
||
b.WriteRune(r)
|
||
}
|
||
return strings.TrimSpace(b.String())
|
||
}
|
||
|
||
// isDashVariant reports whether r is a hyphen/dash/minus variant that should fold to
|
||
// ASCII '-' for the post-check (Unicode dash punctuation + the math minus + soft hyphen).
|
||
func isDashVariant(r rune) bool {
|
||
switch r {
|
||
case '‐', '‑', '‒', '–', '—', '―', '−', '':
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// isApostropheVariant reports whether r is a typographic apostrophe/single-quote variant that
|
||
// should fold to ASCII '\” (Task-6 re-audit): U+2018/U+2019 curly single quotes, U+02BC
|
||
// modifier-letter apostrophe, U+FF07 fullwidth apostrophe. Smart-quote editors turn a source
|
||
// key O'Brien's apostrophe into U+2019, which NFKC/NFC do NOT fold to U+0027 — leaving the key
|
||
// silently un-matchable (silently empty) on en names. Folded symmetrically on both sides.
|
||
func isApostropheVariant(r rune) bool {
|
||
switch r {
|
||
case '‘', '’', 'ʼ', '＇':
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// isIgnorableForMatch reports whether r is a default-ignorable / format code point that
|
||
// must be DROPPED before matching. These survive NFKC/NFC unchanged yet are invisible or
|
||
// carry no character identity, so leaving them in silently breaks a match — on the source
|
||
// side the A4 "silently empty" hole (a key 渡邊 misses 渡<ZWSP>邊), on the target side a false
|
||
// post-check MISS (a stored «вана» misses «ва<WJ>на»). Covers unicode.Cf (U+200B ZWSP,
|
||
// U+200C/D ZWNJ/ZWJ, U+2060 WJ, U+FEFF BOM, bidi marks, …) plus the variation selectors
|
||
// (VS1–16 U+FE00..FE0F and the ideographic supplement U+E0100..E01EF), which decorate a
|
||
// base glyph without changing it. The soft hyphen U+00AD is also unicode.Cf, but the
|
||
// target normalizer folds it to '-' via isDashVariant BEFORE consulting this predicate
|
||
// (self-review #8); on the source side (no dash semantics) dropping it is correct.
|
||
func isIgnorableForMatch(r rune) bool {
|
||
if unicode.Is(unicode.Cf, r) {
|
||
return true
|
||
}
|
||
return (r >= 0xFE00 && r <= 0xFE0F) || (r >= 0xE0100 && r <= 0xE01EF)
|
||
}
|
||
|
||
// SignificantLen counts the "significant" characters of a normalized key — CJK
|
||
// ideographs/kana plus letters (Latin/Cyrillic) — for the single-key ban (registry
|
||
// A3): a key whose significant length is below minKeyLen may not fire on its own
|
||
// (a lone Han/kana/letter like 炎/气/钱 matches unrelated text on the wrong sense).
|
||
// Digits and punctuation do not count (so "阿Q" = 2, but a bare "Q" = 1). Mirrors the
|
||
// eval spec's cjk_len.
|
||
func SignificantLen(normalized string) int {
|
||
n := 0
|
||
for _, r := range normalized {
|
||
switch {
|
||
case DenseScript(r):
|
||
n++
|
||
case unicode.IsLetter(r):
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|