231 lines
10 KiB
Go
231 lines
10 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
_ "embed"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"golang.org/x/text/unicode/norm"
|
||
)
|
||
|
||
// memnorm.go: the memory-bank NORMALIZATION artifact (registry A4 — the most likely
|
||
// "тихо пусто" bug: a glossary key is present in the chunk but in a different
|
||
// orthographic form, so the record silently fails to match). Two pure, deterministic,
|
||
// VERSIONED normalizers, applied SYMMETRICALLY to both sides of every comparison:
|
||
//
|
||
// normalizeSourceKey — zh/ja source surfaces (glossary keys/aliases AND chunk text):
|
||
// NFKC (fold half/full-width, e.g. ﾃ→テ, Ｑ→Q) → trad→simp (per-char, the
|
||
// embedded table) → katakana→hiragana fold → Unicode lower. So a key stored in
|
||
// Simplified still matches a Traditional source surface (魯鎮 ↔ 鲁镇), a full-width
|
||
// Ｑ matches a half-width Q, and スズキ matches すずき.
|
||
// normalizeTargetForm — Russian dst forms (stored decl forms AND model output) for
|
||
// the post-check substring test: NFC → Unicode lower → ё→е fold.
|
||
//
|
||
// The whole artifact is versioned by memoryNormVersion, which is FOLDED INTO the
|
||
// snapshot via memoryVersion() (snapshot.go): a change to the algorithm OR to the
|
||
// embedded trad→simp table is a loud --resnapshot, never a silent match-behaviour
|
||
// change on already-checkpointed chunks. Determinism is load-bearing — the injected
|
||
// glossary block is a pure function of (frozen glossary, normalized chunk), and it
|
||
// enters request_hash via the rendered messages.
|
||
|
||
//go:embed data/trad2simp.txt
|
||
var trad2simpRaw string
|
||
|
||
// memoryNormAlgoVersion tags the normalization ALGORITHM (the pipeline of steps and
|
||
// their order). Bump it on an algorithm change. The trad→simp TABLE content is
|
||
// versioned SEPARATELY by hashing trad2simpRaw into memoryNormVersion below, so
|
||
// editing/completing the data file also invalidates — drift-proof (D8): you cannot
|
||
// forget to bump a version when you change the table, because the table's bytes ARE
|
||
// the version.
|
||
const memoryNormAlgoVersion = "memnorm-v3-nfkc+apos+stripignorable+trad+kana+lower/nfc+dash+apos+stripignorable+lower+yofold"
|
||
|
||
var (
|
||
// trad2simp is the parsed Traditional→Simplified single-char map (data/trad2simp.txt).
|
||
trad2simp map[rune]rune
|
||
// memoryNormVersion is the snapshot-load-bearing version of the entire
|
||
// normalization artifact: the algorithm tag + a content hash of the embedded
|
||
// trad→simp table. memoryVersion() (snapshot.go) folds it into the job snapshot.
|
||
memoryNormVersion string
|
||
)
|
||
|
||
func init() {
|
||
trad2simp = parseTradTable(trad2simpRaw)
|
||
h := sha256.Sum256([]byte(memoryNormAlgoVersion + "\x00" + trad2simpRaw))
|
||
memoryNormVersion = memoryNormAlgoVersion + "-" + hex.EncodeToString(h[:])[:12]
|
||
}
|
||
|
||
// parseTradTable parses the embedded "<trad> <simp>" table. Comment ('#') and blank
|
||
// lines are ignored; a non-empty content line that is NOT exactly two
|
||
// single-rune whitespace-separated fields is a corrupt table → panic at init (a
|
||
// loud build/test failure, caught the moment any test in the package runs, rather
|
||
// than a silently truncated map that would re-open the A4 hole). The table is
|
||
// authored in-repo and asserted by memnorm_test.go, so a panic here means the data
|
||
// file was corrupted, not bad user input.
|
||
func parseTradTable(raw string) map[rune]rune {
|
||
m := make(map[rune]rune, 1024)
|
||
for i, line := range strings.Split(raw, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" || strings.HasPrefix(line, "#") {
|
||
continue
|
||
}
|
||
fields := strings.Fields(line)
|
||
if len(fields) != 2 {
|
||
panic(fmt.Sprintf("pipeline: trad2simp table line %d: want 2 fields, got %d (%q)", i+1, len(fields), line))
|
||
}
|
||
tr := []rune(fields[0])
|
||
si := []rune(fields[1])
|
||
if len(tr) != 1 || len(si) != 1 {
|
||
panic(fmt.Sprintf("pipeline: trad2simp table line %d: each side must be a single rune (%q → %q)", i+1, fields[0], fields[1]))
|
||
}
|
||
// A later duplicate wins deterministically (the file may list a char under
|
||
// two families); assert the mapping is consistent to catch a typo where the
|
||
// same trad char is mapped to two different simp chars.
|
||
if prev, ok := m[tr[0]]; ok && prev != si[0] {
|
||
panic(fmt.Sprintf("pipeline: trad2simp table line %d: %q maps to both %q and %q", i+1, fields[0], string(prev), fields[1]))
|
||
}
|
||
m[tr[0]] = si[0]
|
||
}
|
||
return m
|
||
}
|
||
|
||
// normalizeSourceKey normalizes a zh/ja SOURCE surface (a glossary key/alias OR the
|
||
// chunk text) for exact matching. Pure and deterministic; the pipeline is
|
||
// NFKC → drop default-ignorables → trad→simp (per rune) → katakana→hiragana → Unicode
|
||
// lower. Applied to BOTH keys and text so orthographic variants match (A4). Dropping
|
||
// default-ignorable/format code points (zero-width space/joiner, BOM, variation
|
||
// selectors) closes the re-opened A4 hole where a key 渡邊 silently misses a chunk
|
||
// surface 渡<ZWSP>邊 (common in scraped/EPUB CJK). Kana folding mirrors the eval spec
|
||
// (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to hiragana;
|
||
// the prolonged-sound mark ー (U+30FC) and half-width forms are already handled (the
|
||
// latter by NFKC).
|
||
func normalizeSourceKey(s string) string {
|
||
s = norm.NFKC.String(s)
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
for _, r := range s {
|
||
if isApostropheVariant(r) {
|
||
r = '\'' // typographic apostrophe (O’Brien) folds to ASCII so a key matches either form
|
||
}
|
||
if isIgnorableForMatch(r) {
|
||
continue // zero-width / format / variation selector: invisible, never part of the key
|
||
}
|
||
if m, ok := trad2simp[r]; ok {
|
||
r = m
|
||
}
|
||
if r >= 0x30A1 && r <= 0x30F6 { // katakana → hiragana
|
||
r -= 0x60
|
||
}
|
||
b.WriteRune(unicode.ToLower(r))
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// normalizeTargetForm normalizes a Russian TARGET form (a stored decl form OR the
|
||
// model output) for the post-check whole-word test. Applied SYMMETRICALLY to both
|
||
// sides so a correctly-rendered term is NEVER a false MISS (over-flagging is the
|
||
// failure mode research/14 §2 warns about). Steps, each closing a self-review finding:
|
||
// - NFC;
|
||
// - dash/hyphen variants (en-dash, NB-hyphen, minus, soft hyphen, …) → ASCII '-', so
|
||
// «А–кью»/«А‑кью» match a stored «А-кью» (self-review #8);
|
||
// - drop default-ignorable/format code points (zero-width space/joiner, word joiner,
|
||
// BOM, variation selectors) that survive NFC, so a stored «вана» is not a false MISS
|
||
// against model output «ва<WJ>на» (over-flag; the soft hyphen is folded above, kept);
|
||
// - runs of ANY Unicode whitespace (newline, NBSP U+00A0, ideographic space, …) →
|
||
// one ASCII space, then trim, so a multi-word name wrapped across a line break
|
||
// («Бородатого\nВана») matches a stored «Бородатого Вана» (self-review #2);
|
||
// - Unicode lower + ё→е fold (the ёфикатор's е/ё variation).
|
||
//
|
||
// Pure and deterministic.
|
||
func normalizeTargetForm(s string) string {
|
||
s = norm.NFC.String(s)
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
prevSpace := false
|
||
for _, r := range s {
|
||
if isDashVariant(r) {
|
||
r = '-'
|
||
}
|
||
if isApostropheVariant(r) {
|
||
r = '\'' // symmetric with the source side (д’Артаньян) — no false post-check MISS on a typographic apostrophe
|
||
}
|
||
if isIgnorableForMatch(r) {
|
||
continue // drop zero-width/format/VS (soft hyphen already folded to '-' above)
|
||
}
|
||
if unicode.IsSpace(r) {
|
||
if !prevSpace {
|
||
b.WriteByte(' ')
|
||
prevSpace = true
|
||
}
|
||
continue
|
||
}
|
||
prevSpace = false
|
||
r = unicode.ToLower(r)
|
||
if r == 'ё' {
|
||
r = 'е'
|
||
}
|
||
b.WriteRune(r)
|
||
}
|
||
return strings.TrimSpace(b.String())
|
||
}
|
||
|
||
// isDashVariant reports whether r is a hyphen/dash/minus variant that should fold to
|
||
// ASCII '-' for the post-check (Unicode dash punctuation + the math minus + soft hyphen).
|
||
func isDashVariant(r rune) bool {
|
||
switch r {
|
||
case '‐', '‑', '‒', '–', '—', '―', '−', '':
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// isApostropheVariant reports whether r is a typographic apostrophe/single-quote variant that
|
||
// should fold to ASCII '\” (Task-6 re-audit): U+2018/U+2019 curly single quotes, U+02BC
|
||
// modifier-letter apostrophe, U+FF07 fullwidth apostrophe. Smart-quote editors turn a source
|
||
// key O'Brien's apostrophe into U+2019, which NFKC/NFC do NOT fold to U+0027 — leaving the key
|
||
// silently un-matchable (тихо пусто) on en names. Folded symmetrically on both sides.
|
||
func isApostropheVariant(r rune) bool {
|
||
switch r {
|
||
case '‘', '’', 'ʼ', '＇':
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// isIgnorableForMatch reports whether r is a default-ignorable / format code point that
|
||
// must be DROPPED before matching. These survive NFKC/NFC unchanged yet are invisible or
|
||
// carry no character identity, so leaving them in silently breaks a match — on the source
|
||
// side the A4 "тихо пусто" hole (a key 渡邊 misses 渡<ZWSP>邊), on the target side a false
|
||
// post-check MISS (a stored «вана» misses «ва<WJ>на»). Covers unicode.Cf (U+200B ZWSP,
|
||
// U+200C/D ZWNJ/ZWJ, U+2060 WJ, U+FEFF BOM, bidi marks, …) plus the variation selectors
|
||
// (VS1–16 U+FE00..FE0F and the ideographic supplement U+E0100..E01EF), which decorate a
|
||
// base glyph without changing it. The soft hyphen U+00AD is also unicode.Cf, but the
|
||
// target normalizer folds it to '-' via isDashVariant BEFORE consulting this predicate
|
||
// (self-review #8); on the source side (no dash semantics) dropping it is correct.
|
||
func isIgnorableForMatch(r rune) bool {
|
||
if unicode.Is(unicode.Cf, r) {
|
||
return true
|
||
}
|
||
return (r >= 0xFE00 && r <= 0xFE0F) || (r >= 0xE0100 && r <= 0xE01EF)
|
||
}
|
||
|
||
// significantLen counts the "significant" characters of a normalized key — CJK
|
||
// ideographs/kana plus letters (Latin/Cyrillic) — for the single-key ban (registry
|
||
// A3): a key whose significant length is below minKeyLen may not fire on its own
|
||
// (a lone Han/kana/letter like 炎/气/钱 matches unrelated text on the wrong sense).
|
||
// Digits and punctuation do not count (so "阿Q" = 2, but a bare "Q" = 1). Mirrors the
|
||
// eval spec's cjk_len.
|
||
func significantLen(normalized string) int {
|
||
n := 0
|
||
for _, r := range normalized {
|
||
switch {
|
||
case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul):
|
||
n++
|
||
case unicode.IsLetter(r):
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|