182 lines
7.4 KiB
Go
182 lines
7.4 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
_ "embed"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"golang.org/x/text/unicode/norm"
|
||
)
|
||
|
||
// memnorm.go: the memory-bank NORMALIZATION artifact (registry A4 — the most likely
|
||
// "тихо пусто" bug: a glossary key is present in the chunk but in a different
|
||
// orthographic form, so the record silently fails to match). Two pure, deterministic,
|
||
// VERSIONED normalizers, applied SYMMETRICALLY to both sides of every comparison:
|
||
//
|
||
// normalizeSourceKey — zh/ja source surfaces (glossary keys/aliases AND chunk text):
|
||
// NFKC (fold half/full-width, e.g. ﾃ→テ, Ｑ→Q) → trad→simp (per-char, the
|
||
// embedded table) → katakana→hiragana fold → Unicode lower. So a key stored in
|
||
// Simplified still matches a Traditional source surface (魯鎮 ↔ 鲁镇), a full-width
|
||
// Ｑ matches a half-width Q, and スズキ matches すずき.
|
||
// normalizeTargetForm — Russian dst forms (stored decl forms AND model output) for
|
||
// the post-check substring test: NFC → Unicode lower → ё→е fold.
|
||
//
|
||
// The whole artifact is versioned by memoryNormVersion, which is FOLDED INTO the
|
||
// snapshot via memoryVersion() (runner.go): a change to the algorithm OR to the
|
||
// embedded trad→simp table is a loud --resnapshot, never a silent match-behaviour
|
||
// change on already-checkpointed chunks. Determinism is load-bearing — the injected
|
||
// glossary block is a pure function of (frozen glossary, normalized chunk), and it
|
||
// enters request_hash via the rendered messages.
|
||
|
||
//go:embed data/trad2simp.txt
|
||
var trad2simpRaw string
|
||
|
||
// memoryNormAlgoVersion tags the normalization ALGORITHM (the pipeline of steps and
|
||
// their order). Bump it on an algorithm change. The trad→simp TABLE content is
|
||
// versioned SEPARATELY by hashing trad2simpRaw into memoryNormVersion below, so
|
||
// editing/completing the data file also invalidates — drift-proof (D8): you cannot
|
||
// forget to bump a version when you change the table, because the table's bytes ARE
|
||
// the version.
|
||
const memoryNormAlgoVersion = "memnorm-v1-nfkc+trad+kana+lower/nfc+lower+yofold"
|
||
|
||
var (
|
||
// trad2simp is the parsed Traditional→Simplified single-char map (data/trad2simp.txt).
|
||
trad2simp map[rune]rune
|
||
// memoryNormVersion is the snapshot-load-bearing version of the entire
|
||
// normalization artifact: the algorithm tag + a content hash of the embedded
|
||
// trad→simp table. memoryVersion() (runner.go) folds it into the job snapshot.
|
||
memoryNormVersion string
|
||
)
|
||
|
||
func init() {
|
||
trad2simp = parseTradTable(trad2simpRaw)
|
||
h := sha256.Sum256([]byte(memoryNormAlgoVersion + "\x00" + trad2simpRaw))
|
||
memoryNormVersion = memoryNormAlgoVersion + "-" + hex.EncodeToString(h[:])[:12]
|
||
}
|
||
|
||
// parseTradTable parses the embedded "<trad> <simp>" table. Comment ('#') and blank
|
||
// lines are ignored; a non-empty content line that is NOT exactly two
|
||
// single-rune whitespace-separated fields is a corrupt table → panic at init (a
|
||
// loud build/test failure, caught the moment any test in the package runs, rather
|
||
// than a silently truncated map that would re-open the A4 hole). The table is
|
||
// authored in-repo and asserted by memnorm_test.go, so a panic here means the data
|
||
// file was corrupted, not bad user input.
|
||
func parseTradTable(raw string) map[rune]rune {
|
||
m := make(map[rune]rune, 1024)
|
||
for i, line := range strings.Split(raw, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" || strings.HasPrefix(line, "#") {
|
||
continue
|
||
}
|
||
fields := strings.Fields(line)
|
||
if len(fields) != 2 {
|
||
panic(fmt.Sprintf("pipeline: trad2simp table line %d: want 2 fields, got %d (%q)", i+1, len(fields), line))
|
||
}
|
||
tr := []rune(fields[0])
|
||
si := []rune(fields[1])
|
||
if len(tr) != 1 || len(si) != 1 {
|
||
panic(fmt.Sprintf("pipeline: trad2simp table line %d: each side must be a single rune (%q → %q)", i+1, fields[0], fields[1]))
|
||
}
|
||
// A later duplicate wins deterministically (the file may list a char under
|
||
// two families); assert the mapping is consistent to catch a typo where the
|
||
// same trad char is mapped to two different simp chars.
|
||
if prev, ok := m[tr[0]]; ok && prev != si[0] {
|
||
panic(fmt.Sprintf("pipeline: trad2simp table line %d: %q maps to both %q and %q", i+1, fields[0], string(prev), fields[1]))
|
||
}
|
||
m[tr[0]] = si[0]
|
||
}
|
||
return m
|
||
}
|
||
|
||
// normalizeSourceKey normalizes a zh/ja SOURCE surface (a glossary key/alias OR the
|
||
// chunk text) for exact matching. Pure and deterministic; the pipeline is
|
||
// NFKC → trad→simp (per rune) → katakana→hiragana → Unicode lower. Applied to BOTH
|
||
// keys and text so orthographic variants match (A4). Kana folding mirrors the eval
|
||
// spec (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to
|
||
// hiragana; the prolonged-sound mark ー (U+30FC) and half-width forms are already
|
||
// handled (the latter by NFKC).
|
||
func normalizeSourceKey(s string) string {
|
||
s = norm.NFKC.String(s)
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
for _, r := range s {
|
||
if m, ok := trad2simp[r]; ok {
|
||
r = m
|
||
}
|
||
if r >= 0x30A1 && r <= 0x30F6 { // katakana → hiragana
|
||
r -= 0x60
|
||
}
|
||
b.WriteRune(unicode.ToLower(r))
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// normalizeTargetForm normalizes a Russian TARGET form (a stored decl form OR the
|
||
// model output) for the post-check whole-word test. Applied SYMMETRICALLY to both
|
||
// sides so a correctly-rendered term is NEVER a false MISS (over-flagging is the
|
||
// failure mode research/14 §2 warns about). Steps, each closing a self-review finding:
|
||
// - NFC;
|
||
// - dash/hyphen variants (en-dash, NB-hyphen, minus, soft hyphen, …) → ASCII '-', so
|
||
// «А–кью»/«А‑кью» match a stored «А-кью» (self-review #8);
|
||
// - runs of ANY Unicode whitespace (newline, NBSP U+00A0, ideographic space, …) →
|
||
// one ASCII space, then trim, so a multi-word name wrapped across a line break
|
||
// («Бородатого\nВана») matches a stored «Бородатого Вана» (self-review #2);
|
||
// - Unicode lower + ё→е fold (the ёфикатор's е/ё variation).
|
||
// Pure and deterministic.
|
||
func normalizeTargetForm(s string) string {
|
||
s = norm.NFC.String(s)
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
prevSpace := false
|
||
for _, r := range s {
|
||
if isDashVariant(r) {
|
||
r = '-'
|
||
}
|
||
if unicode.IsSpace(r) {
|
||
if !prevSpace {
|
||
b.WriteByte(' ')
|
||
prevSpace = true
|
||
}
|
||
continue
|
||
}
|
||
prevSpace = false
|
||
r = unicode.ToLower(r)
|
||
if r == 'ё' {
|
||
r = 'е'
|
||
}
|
||
b.WriteRune(r)
|
||
}
|
||
return strings.TrimSpace(b.String())
|
||
}
|
||
|
||
// isDashVariant reports whether r is a hyphen/dash/minus variant that should fold to
|
||
// ASCII '-' for the post-check (Unicode dash punctuation + the math minus + soft hyphen).
|
||
func isDashVariant(r rune) bool {
|
||
switch r {
|
||
case '‐', '‑', '‒', '–', '—', '―', '−', '':
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// significantLen counts the "significant" characters of a normalized key — CJK
|
||
// ideographs/kana plus letters (Latin/Cyrillic) — for the single-key ban (registry
|
||
// A3): a key whose significant length is below minKeyLen may not fire on its own
|
||
// (a lone Han/kana/letter like 炎/气/钱 matches unrelated text on the wrong sense).
|
||
// Digits and punctuation do not count (so "阿Q" = 2, but a bare "Q" = 1). Mirrors the
|
||
// eval spec's cjk_len.
|
||
func significantLen(normalized string) int {
|
||
n := 0
|
||
for _, r := range normalized {
|
||
switch {
|
||
case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul):
|
||
n++
|
||
case unicode.IsLetter(r):
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|