From 38c7f3a04a85ed6cb33cad79f4d3e7f63d1af918 Mon Sep 17 00:00:00 2001 From: "Claude (backend session)" Date: Sun, 5 Jul 2026 14:51:36 +0300 Subject: [PATCH] Close the two adversarial-review majors: strip default-ignorable code points in both memory normalizers and trim surrounding whitespace on seed keying surfaces, bumping the normalizer version --- backend/internal/pipeline/memnorm.go | 42 ++++++++++++--- backend/internal/pipeline/memnorm_test.go | 65 ++++++++++++++++++++--- backend/internal/pipeline/memseed.go | 19 +++++-- backend/internal/pipeline/memseed_test.go | 39 ++++++++++++-- docs/PROGRESS.md | 9 ++++ 5 files changed, 152 insertions(+), 22 deletions(-) diff --git a/backend/internal/pipeline/memnorm.go b/backend/internal/pipeline/memnorm.go index f72b834..44c6e24 100644 --- a/backend/internal/pipeline/memnorm.go +++ b/backend/internal/pipeline/memnorm.go @@ -40,7 +40,7 @@ var trad2simpRaw string // editing/completing the data file also invalidates — drift-proof (D8): you cannot // forget to bump a version when you change the table, because the table's bytes ARE // the version. -const memoryNormAlgoVersion = "memnorm-v1-nfkc+trad+kana+lower/nfc+lower+yofold" +const memoryNormAlgoVersion = "memnorm-v2-nfkc+stripignorable+trad+kana+lower/nfc+dash+stripignorable+lower+yofold" var ( // trad2simp is the parsed Traditional→Simplified single-char map (data/trad2simp.txt). @@ -93,16 +93,22 @@ func parseTradTable(raw string) map[rune]rune { // normalizeSourceKey normalizes a zh/ja SOURCE surface (a glossary key/alias OR the // chunk text) for exact matching. Pure and deterministic; the pipeline is -// NFKC → trad→simp (per rune) → katakana→hiragana → Unicode lower. Applied to BOTH -// keys and text so orthographic variants match (A4). Kana folding mirrors the eval -// spec (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to -// hiragana; the prolonged-sound mark ー (U+30FC) and half-width forms are already -// handled (the latter by NFKC). +// NFKC → drop default-ignorables → trad→simp (per rune) → katakana→hiragana → Unicode +// lower. Applied to BOTH keys and text so orthographic variants match (A4). Dropping +// default-ignorable/format code points (zero-width space/joiner, BOM, variation +// selectors) closes the re-opened A4 hole where a key 渡邊 silently misses a chunk +// surface 渡邊 (common in scraped/EPUB CJK). Kana folding mirrors the eval spec +// (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to hiragana; +// the prolonged-sound mark ー (U+30FC) and half-width forms are already handled (the +// latter by NFKC). func normalizeSourceKey(s string) string { s = norm.NFKC.String(s) var b strings.Builder b.Grow(len(s)) for _, r := range s { + if isIgnorableForMatch(r) { + continue // zero-width / format / variation selector: invisible, never part of the key + } if m, ok := trad2simp[r]; ok { r = m } @@ -121,10 +127,14 @@ func normalizeSourceKey(s string) string { // - NFC; // - dash/hyphen variants (en-dash, NB-hyphen, minus, soft hyphen, …) → ASCII '-', so // «А–кью»/«А‑кью» match a stored «А-кью» (self-review #8); +// - drop default-ignorable/format code points (zero-width space/joiner, word joiner, +// BOM, variation selectors) that survive NFC, so a stored «вана» is not a false MISS +// against model output «вана» (over-flag; the soft hyphen is folded above, kept); // - runs of ANY Unicode whitespace (newline, NBSP U+00A0, ideographic space, …) → // one ASCII space, then trim, so a multi-word name wrapped across a line break // («Бородатого\nВана») matches a stored «Бородатого Вана» (self-review #2); // - Unicode lower + ё→е fold (the ёфикатор's е/ё variation). +// // Pure and deterministic. func normalizeTargetForm(s string) string { s = norm.NFC.String(s) @@ -135,6 +145,9 @@ func normalizeTargetForm(s string) string { if isDashVariant(r) { r = '-' } + if isIgnorableForMatch(r) { + continue // drop zero-width/format/VS (soft hyphen already folded to '-' above) + } if unicode.IsSpace(r) { if !prevSpace { b.WriteByte(' ') @@ -162,6 +175,23 @@ func isDashVariant(r rune) bool { return false } +// isIgnorableForMatch reports whether r is a default-ignorable / format code point that +// must be DROPPED before matching. These survive NFKC/NFC unchanged yet are invisible or +// carry no character identity, so leaving them in silently breaks a match — on the source +// side the A4 "тихо пусто" hole (a key 渡邊 misses 渡邊), on the target side a false +// post-check MISS (a stored «вана» misses «вана»). Covers unicode.Cf (U+200B ZWSP, +// U+200C/D ZWNJ/ZWJ, U+2060 WJ, U+FEFF BOM, bidi marks, …) plus the variation selectors +// (VS1–16 U+FE00..FE0F and the ideographic supplement U+E0100..E01EF), which decorate a +// base glyph without changing it. The soft hyphen U+00AD is also unicode.Cf, but the +// target normalizer folds it to '-' via isDashVariant BEFORE consulting this predicate +// (self-review #8); on the source side (no dash semantics) dropping it is correct. +func isIgnorableForMatch(r rune) bool { + if unicode.Is(unicode.Cf, r) { + return true + } + return (r >= 0xFE00 && r <= 0xFE0F) || (r >= 0xE0100 && r <= 0xE01EF) +} + // significantLen counts the "significant" characters of a normalized key — CJK // ideographs/kana plus letters (Latin/Cyrillic) — for the single-key ban (registry // A3): a key whose significant length is below minKeyLen may not fire on its own diff --git a/backend/internal/pipeline/memnorm_test.go b/backend/internal/pipeline/memnorm_test.go index 777634a..f812f98 100644 --- a/backend/internal/pipeline/memnorm_test.go +++ b/backend/internal/pipeline/memnorm_test.go @@ -52,7 +52,7 @@ func TestNormalizeSourceKey(t *testing.T) { // matches a chunk written in another, because normalization is applied to BOTH. func TestNormalizeSymmetry(t *testing.T) { // glossary key in Simplified, chunk text in Traditional - key := normalizeSourceKey("鲁镇") // simplified + key := normalizeSourceKey("鲁镇") // simplified text := normalizeSourceKey("魯鎮的冬天很冷。") // traditional if !strings.Contains(text, key) { t.Errorf("normalized simplified key %q not found in normalized traditional text %q", key, text) @@ -99,13 +99,13 @@ func TestSignificantLen(t *testing.T) { in string want int }{ - {"阿q", 2}, // han + latin letter - {"炎", 1}, // single han — banned as a standalone key - {"气", 1}, // single han - {"钱", 1}, // single han (research/14 homograph) - {"送灶", 2}, // bigram — allowed - {"すずき", 3}, // three kana - {"q", 1}, // bare latin letter + {"阿q", 2}, // han + latin letter + {"炎", 1}, // single han — banned as a standalone key + {"气", 1}, // single han + {"钱", 1}, // single han (research/14 homograph) + {"送灶", 2}, // bigram — allowed + {"すずき", 3}, // three kana + {"q", 1}, // bare latin letter {"а-кью", 4}, // cyrillic letters, hyphen not counted: а к ь ю = 4 } for _, c := range cases { @@ -114,3 +114,52 @@ func TestSignificantLen(t *testing.T) { } } } + +// TestNormalizeStripsDefaultIgnorable covers the workflow-confirmed A4 major: zero-width / +// format / variation-selector code points survive NFKC/NFC and silently break a match. A +// source key with an embedded ZWSP/joiner/BOM/VS must normalize identically to the clean +// key (else the A4 "тихо пусто" hole re-opens); a target form with a mid-word joiner must +// normalize identically to the clean form (else a correct rendering is a false MISS). +// Reverting the isIgnorableForMatch strip in either normalizer fails this test. +func TestNormalizeStripsDefaultIgnorable(t *testing.T) { + // Explicit escapes via code points (the chars are invisible by nature — the bug being tested). + zwsp := string(rune(0x200B)) // ZERO WIDTH SPACE + zwj := string(rune(0x200D)) // ZERO WIDTH JOINER + wj := string(rune(0x2060)) // WORD JOINER + bom := string(rune(0xFEFF)) // BOM / ZWNBSP + vs16 := string(rune(0xFE0F)) // VARIATION SELECTOR-16 + shy := string(rune(0x00AD)) // SOFT HYPHEN (unicode.Cf) + + cleanSrc := normalizeSourceKey("渡邊") + for _, in := range []string{ + "渡" + zwsp + "邊", + "渡" + zwj + "邊", + "渡" + wj + "邊", + "渡" + bom + "邊", + "渡邊" + vs16, // VS decorating the final key char + "渡" + shy + "邊", // soft hyphen dropped on the source side (no dash semantics) + } { + if got := normalizeSourceKey(in); got != cleanSrc { + t.Errorf("source key %+q → %q, want %q (default-ignorable not stripped)", in, got, cleanSrc) + } + } + + cleanTgt := normalizeTargetForm("вана") + for _, in := range []string{ + "ва" + wj + "на", + "ва" + zwsp + "на", + "ва" + bom + "на", + } { + if got := normalizeTargetForm(in); got != cleanTgt { + t.Errorf("target form %+q → %q, want %q (default-ignorable not stripped)", in, got, cleanTgt) + } + } + + // Regression guard for self-review #8 AND for the strip's ORDERING: the soft hyphen must + // STILL fold to a real '-' on the target side (a hyphenation point), NOT be stripped — so + // «А­кью» keeps matching a stored «А-кью». Placing the ignorable-strip before isDashVariant + // would break this. + if got, want := normalizeTargetForm("А"+shy+"кью"), normalizeTargetForm("А-кью"); got != want { + t.Errorf("soft hyphen on the target side = %q, want %q (must fold to '-', not be stripped)", got, want) + } +} diff --git a/backend/internal/pipeline/memseed.go b/backend/internal/pipeline/memseed.go index e1fa6d8..e52eae4 100644 --- a/backend/internal/pipeline/memseed.go +++ b/backend/internal/pipeline/memseed.go @@ -72,7 +72,17 @@ func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) { var problems []string termKeys := map[[4]string]bool{} // (src,sense,since,until) → detect a duplicate term key for i, t := range sf.Terms { - if strings.TrimSpace(t.Src) == "" { + // Trim surrounding whitespace on the KEYING surfaces before anything reads them. + // normalizeSourceKey does not trim, so a raw " 強敵" would key as " 強敵" and never + // match "強敵" — a silently-inert approved term (the A-class hole this bank closes). + // A trailing space on sense also makes two identical senses look distinct, slipping + // past BOTH the duplicate-key and the same-sense contradiction guards below. + // Surrounding whitespace is never intentional; trim it (a forgiving fix, like the + // alias dedup). Internal spacing of a multi-word surface is preserved. + t.Src = strings.TrimSpace(t.Src) + t.Sense = strings.TrimSpace(t.Sense) + t.Dst = strings.TrimSpace(t.Dst) + if t.Src == "" { problems = append(problems, fmt.Sprintf("term %d: src is required", i)) continue } @@ -98,7 +108,7 @@ func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) { // An approved OR draft term with no dst is silently inert (not matchable — it renders // nothing) yet reads as an intended rendering: fail loud (external-review minor #5). // Only status=auto (a raw candidate — e.g. a ruby reading awaiting a dst) may be empty. - if (status == "approved" || status == "draft") && strings.TrimSpace(t.Dst) == "" { + if (status == "approved" || status == "draft") && t.Dst == "" { problems = append(problems, fmt.Sprintf("term %q: a %s term must have a non-empty dst (an empty one is silently inert; only status=auto may lack a dst)", t.Src, status)) continue } @@ -125,7 +135,8 @@ func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) { // what matters for matching, and a redundant alias is a harmless authoring slip. seenAlias := map[string]bool{} for _, a := range t.Aliases { - if strings.TrimSpace(a.Alias) == "" { + a.Alias = strings.TrimSpace(a.Alias) // same silent-inert risk as src (an alias is a match key) + if a.Alias == "" { problems = append(problems, fmt.Sprintf("term %q: an alias is empty", t.Src)) continue } @@ -243,7 +254,7 @@ func rubyToCandidates(readings []store.RubyReading, manualSrcs map[string]bool) // ruby classifier classes (deterministic HINTS for human promotion). const ( - rubyClassName = "name_candidate" // all-Han base + all-kana reading: the furigana-name SHAPE + rubyClassName = "name_candidate" // all-Han base + all-kana reading: the furigana-name SHAPE rubyClassGloss = "gloss_candidate" // reading carries Han: likely an author double-reading → footnote, NOT a lock rubyClassAmbiguous = "ambiguous" // anything else → flag to a human ) diff --git a/backend/internal/pipeline/memseed_test.go b/backend/internal/pipeline/memseed_test.go index 15a4c16..19ab03c 100644 --- a/backend/internal/pipeline/memseed_test.go +++ b/backend/internal/pipeline/memseed_test.go @@ -141,10 +141,10 @@ func TestClassifyRubyReading(t *testing.T) { cases := []struct { base, reading, want string }{ - {"鈴木", "すずき", rubyClassName}, // all-Han base + all-kana reading → name candidate - {"強敵", "とも", rubyClassName}, // SAME shape (a double-reading!) — deliberately NOT distinguishable offline → still a CANDIDATE, never auto-locked - {"強敵", "きょうてき", rubyClassName}, // real reading, same class - {"泣蟲", "泣き虫", rubyClassGloss}, // reading carries kanji → a semantic gloss/double-reading → footnote, not a lock + {"鈴木", "すずき", rubyClassName}, // all-Han base + all-kana reading → name candidate + {"強敵", "とも", rubyClassName}, // SAME shape (a double-reading!) — deliberately NOT distinguishable offline → still a CANDIDATE, never auto-locked + {"強敵", "きょうてき", rubyClassName}, // real reading, same class + {"泣蟲", "泣き虫", rubyClassGloss}, // reading carries kanji → a semantic gloss/double-reading → footnote, not a lock {"ゲーム", "gēmu", rubyClassAmbiguous}, // base is kana → not a plain furigana-name } for _, c := range cases { @@ -193,3 +193,34 @@ func TestRubyToCandidates(t *testing.T) { t.Error("図書館 should have been skipped (owned by the manual seed)") } } + +// TestSeedSurroundingWhitespaceTrimmed covers the workflow-confirmed seed major: surrounding +// whitespace on a keying surface passed the emptiness check but was stored raw, so the term +// keyed WITH the space and could never match (silently inert), and a trailing-space sense +// disguised a same-sense contradiction. Reverting any of the three trims in loadGlossarySeed +// fails a branch here. +func TestSeedSurroundingWhitespaceTrimmed(t *testing.T) { + // (1) A leading-space src on an approved term must be trimmed, not stored raw — else the + // key " 強敵" can never match "強敵" and the approved term renders nothing for the whole book. + entries, err := loadGlossarySeed(writeSeed(t, "terms:\n - { src: \" 強敵\", dst: Соперник }\n")) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Src != "強敵" { + t.Fatalf("src surrounding whitespace not trimmed: %+v", entries) + } + // (2) A trailing-space sense must be trimmed so a same-sense contradiction (same src, same + // sense, same default window, DIFFERENT dst) is caught by the duplicate-key guard — not + // stored as two distinct-looking rows that both inject a contradictory rendering. + if _, err := loadGlossarySeed(writeSeed(t, "terms:\n - { src: 先生, dst: учитель, sense: teacher }\n - { src: 先生, dst: доктор, sense: \"teacher \" }\n")); err == nil { + t.Error("trailing-space sense must be trimmed so the duplicate (src,sense,window) fails loud") + } + // (3) An alias with surrounding whitespace must be trimmed (an alias is itself a match key). + al, err := loadGlossarySeed(writeSeed(t, "terms:\n - src: 甲\n dst: A\n aliases:\n - { alias: \" 乙 \", type: x }\n")) + if err != nil { + t.Fatal(err) + } + if len(al) != 1 || len(al[0].Aliases) != 1 || al[0].Aliases[0].Alias != "乙" { + t.Fatalf("alias surrounding whitespace not trimmed: %+v", al) + } +} diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 71036da..48670d9 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -483,6 +483,15 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор **eval low:** #a — тест `TestOverlappingNonNestedKeysBothFire` фиксирует принятое поведение (обе срабатывают); #b/#c/#d — задокументированы (редактор-инъекция = приоритет след. вехи; gate-агрессивность — `FlagGlossaryMiss` не-retryable/не-escalatable, флагает для человека; live-wire кейс — follow-up на `live_conformance`-харнесс). Регресс-тесты: per-lang-min+collision-disposition, gate-ignores-ambiguous (e2e), overlap-windows/empty-dst fail-loud, overlapping-non-nested. Оба вердикта: **ПРИНЯТО**; дефолтный конфиг чист, флип гейта и прод ja-kana разблокированы. +### 2026-07-05 — Шаг 4: глубокое адверсариальное ревью полной поверхности (44 агента) + +По запросу владельца — многоагентный воркфлоу поверх `cc57c7b`: 8 finder-дименсий (нормализация / матчер+disposition / F1 / post-check / seed+ruby / store / инъекция+resume / мутационная-стойкость тестов), каждая не-минорная находка → 3 адверсариальных верификатора refute-by-default, подтверждение большинством ≥2/3. **24 кандидата → 2 подтверждённых major, 10 отсеяно (все 0/3–1/3), 12 миноров.** Отсев здоровый: напр. «sticky протаскивает collision-prone AMBIGUOUS в CONFIRMED» зарублен 0/3 верно — sticky не матчит исходник заново, а несёт установленный в сцене термин, collision-риск (срабатывание внутри длинного слова) к нему неприменим. Оба major — класс «тихо пусто», ровно та зона, ради которой банк существует; оба закрыты, каждый фикс **мутационно-проверен** (реверт → падает именно его тест), полный `-race` зелёный. + +- **major #1 — default-ignorables в A4-нормализаторах (`memnorm.go`).** Zero-width (U+200B/200C/200D/2060/FEFF) и вариационные селекторы (U+FE0F, IVS) переживали NFKC/NFC → ключ `渡邊` молча не матчил `渡邊` (частый артефакт EPUB/скрейпа CJK — «тихо пусто» переоткрыт), симметрично на target `` внутри слова давал ложный post-check-miss. Фикс: `isIgnorableForMatch` (unicode.Cf + VS-диапазоны) дропает их в ОБОИХ нормализаторах; soft-hyphen U+00AD сохранён как fold→'-' (self-review #8 — порядок проверок гарантирует). `memoryNormAlgoVersion` бампнут **v1→v2** → громкий `--resnapshot`. +- **major #2 — окружающие пробелы на keying-поверхностях (`memseed.go`).** `loadGlossarySeed` проверял пустоту через TrimSpace, но хранил СЫРОЕ значение: `src: " 強敵"` (approved, непустой dst) проходил валидацию, ключился как `" 強敵"` и НИКОГДА не матчил `強敵` — молча-инертный approved-термин на всю книгу; плюс trailing-space на `sense` маскировал same-sense-противоречие мимо обоих гардов (#4/#5). Фикс: тримминг `src`/`sense`/`dst`/`alias` до построения ключей и хранения (прощающий фикс, как alias-дедуп). + +Отсев и 12 миноров оставлены как задокументированные размены/fast-follow (per-lang boundary-тесты post-check, sticky-decay coverage, apostrophe-fold — не гарят на дефолте). Вердикт сессии: **ПРИНЯТО**, шаг 4 закрыт. + ## Полигон (секция параллельной сессии — записи добавлять сюда)