Close the two adversarial-review majors: strip default-ignorable code points in both memory normalizers and trim surrounding whitespace on seed keying surfaces, bumping the normalizer version

backend/internal/pipeline/memnorm.go

@@ -40,7 +40,7 @@ var trad2simpRaw string
 // editing/completing the data file also invalidates — drift-proof (D8): you cannot
 // forget to bump a version when you change the table, because the table's bytes ARE
 // the version.
-const memoryNormAlgoVersion = "memnorm-v1-nfkc+trad+kana+lower/nfc+lower+yofold"
+const memoryNormAlgoVersion = "memnorm-v2-nfkc+stripignorable+trad+kana+lower/nfc+dash+stripignorable+lower+yofold"
 
 var (
 	// trad2simp is the parsed Traditional→Simplified single-char map (data/trad2simp.txt).
@@ -93,16 +93,22 @@ func parseTradTable(raw string) map[rune]rune {
 
 // normalizeSourceKey normalizes a zh/ja SOURCE surface (a glossary key/alias OR the
 // chunk text) for exact matching. Pure and deterministic; the pipeline is
-// NFKC → trad→simp (per rune) → katakana→hiragana → Unicode lower. Applied to BOTH
+// NFKC → drop default-ignorables → trad→simp (per rune) → katakana→hiragana → Unicode
-// keys and text so orthographic variants match (A4). Kana folding mirrors the eval
+// lower. Applied to BOTH keys and text so orthographic variants match (A4). Dropping
-// spec (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to
+// default-ignorable/format code points (zero-width space/joiner, BOM, variation
-// hiragana; the prolonged-sound mark ー (U+30FC) and half-width forms are already
+// selectors) closes the re-opened A4 hole where a key 渡邊 silently misses a chunk
-// handled (the latter by NFKC).
+// surface 渡<ZWSP>邊 (common in scraped/EPUB CJK). Kana folding mirrors the eval spec
+// (memory_hotpath.py): full-width katakana U+30A1..U+30F6 shift down 0x60 to hiragana;
+// the prolonged-sound mark ー (U+30FC) and half-width forms are already handled (the
+// latter by NFKC).
 func normalizeSourceKey(s string) string {
 	s = norm.NFKC.String(s)
 	var b strings.Builder
 	b.Grow(len(s))
 	for _, r := range s {
+		if isIgnorableForMatch(r) {
+			continue // zero-width / format / variation selector: invisible, never part of the key
+		}
 		if m, ok := trad2simp[r]; ok {
 			r = m
 		}
@@ -121,10 +127,14 @@ func normalizeSourceKey(s string) string {
 //   - NFC;
 //   - dash/hyphen variants (en-dash, NB-hyphen, minus, soft hyphen, …) → ASCII '-', so
 //     «А–кью»/«А‑кью» match a stored «А-кью» (self-review #8);
+//   - drop default-ignorable/format code points (zero-width space/joiner, word joiner,
+//     BOM, variation selectors) that survive NFC, so a stored «вана» is not a false MISS
+//     against model output «ва<WJ>на» (over-flag; the soft hyphen is folded above, kept);
 //   - runs of ANY Unicode whitespace (newline, NBSP U+00A0, ideographic space, …) →
 //     one ASCII space, then trim, so a multi-word name wrapped across a line break
 //     («Бородатого\nВана») matches a stored «Бородатого Вана» (self-review #2);
 //   - Unicode lower + ё→е fold (the ёфикатор's е/ё variation).
+//
 // Pure and deterministic.
 func normalizeTargetForm(s string) string {
 	s = norm.NFC.String(s)
@@ -135,6 +145,9 @@ func normalizeTargetForm(s string) string {
 		if isDashVariant(r) {
 			r = '-'
 		}
+		if isIgnorableForMatch(r) {
+			continue // drop zero-width/format/VS (soft hyphen already folded to '-' above)
+		}
 		if unicode.IsSpace(r) {
 			if !prevSpace {
 				b.WriteByte(' ')
@@ -162,6 +175,23 @@ func isDashVariant(r rune) bool {
 	return false
 }
 
+// isIgnorableForMatch reports whether r is a default-ignorable / format code point that
+// must be DROPPED before matching. These survive NFKC/NFC unchanged yet are invisible or
+// carry no character identity, so leaving them in silently breaks a match — on the source
+// side the A4 "тихо пусто" hole (a key 渡邊 misses 渡<ZWSP>邊), on the target side a false
+// post-check MISS (a stored «вана» misses «ва<WJ>на»). Covers unicode.Cf (U+200B ZWSP,
+// U+200C/D ZWNJ/ZWJ, U+2060 WJ, U+FEFF BOM, bidi marks, …) plus the variation selectors
+// (VS1–16 U+FE00..FE0F and the ideographic supplement U+E0100..E01EF), which decorate a
+// base glyph without changing it. The soft hyphen U+00AD is also unicode.Cf, but the
+// target normalizer folds it to '-' via isDashVariant BEFORE consulting this predicate
+// (self-review #8); on the source side (no dash semantics) dropping it is correct.
+func isIgnorableForMatch(r rune) bool {
+	if unicode.Is(unicode.Cf, r) {
+		return true
+	}
+	return (r >= 0xFE00 && r <= 0xFE0F) || (r >= 0xE0100 && r <= 0xE01EF)
+}
+
 // significantLen counts the "significant" characters of a normalized key — CJK
 // ideographs/kana plus letters (Latin/Cyrillic) — for the single-key ban (registry
 // A3): a key whose significant length is below minKeyLen may not fire on its own

backend/internal/pipeline/memnorm_test.go

@@ -114,3 +114,52 @@ func TestSignificantLen(t *testing.T) {
 		}
 	}
 }
+
+// TestNormalizeStripsDefaultIgnorable covers the workflow-confirmed A4 major: zero-width /
+// format / variation-selector code points survive NFKC/NFC and silently break a match. A
+// source key with an embedded ZWSP/joiner/BOM/VS must normalize identically to the clean
+// key (else the A4 "тихо пусто" hole re-opens); a target form with a mid-word joiner must
+// normalize identically to the clean form (else a correct rendering is a false MISS).
+// Reverting the isIgnorableForMatch strip in either normalizer fails this test.
+func TestNormalizeStripsDefaultIgnorable(t *testing.T) {
+	// Explicit escapes via code points (the chars are invisible by nature — the bug being tested).
+	zwsp := string(rune(0x200B)) // ZERO WIDTH SPACE
+	zwj := string(rune(0x200D))  // ZERO WIDTH JOINER
+	wj := string(rune(0x2060))   // WORD JOINER
+	bom := string(rune(0xFEFF))  // BOM / ZWNBSP
+	vs16 := string(rune(0xFE0F)) // VARIATION SELECTOR-16
+	shy := string(rune(0x00AD))  // SOFT HYPHEN (unicode.Cf)
+
+	cleanSrc := normalizeSourceKey("渡邊")
+	for _, in := range []string{
+		"渡" + zwsp + "邊",
+		"渡" + zwj + "邊",
+		"渡" + wj + "邊",
+		"渡" + bom + "邊",
+		"渡邊" + vs16,     // VS decorating the final key char
+		"渡" + shy + "邊", // soft hyphen dropped on the source side (no dash semantics)
+	} {
+		if got := normalizeSourceKey(in); got != cleanSrc {
+			t.Errorf("source key %+q → %q, want %q (default-ignorable not stripped)", in, got, cleanSrc)
+		}
+	}
+
+	cleanTgt := normalizeTargetForm("вана")
+	for _, in := range []string{
+		"ва" + wj + "на",
+		"ва" + zwsp + "на",
+		"ва" + bom + "на",
+	} {
+		if got := normalizeTargetForm(in); got != cleanTgt {
+			t.Errorf("target form %+q → %q, want %q (default-ignorable not stripped)", in, got, cleanTgt)
+		}
+	}
+
+	// Regression guard for self-review #8 AND for the strip's ORDERING: the soft hyphen must
+	// STILL fold to a real '-' on the target side (a hyphenation point), NOT be stripped — so
+	// «А­кью» keeps matching a stored «А-кью». Placing the ignorable-strip before isDashVariant
+	// would break this.
+	if got, want := normalizeTargetForm("А"+shy+"кью"), normalizeTargetForm("А-кью"); got != want {
+		t.Errorf("soft hyphen on the target side = %q, want %q (must fold to '-', not be stripped)", got, want)
+	}
+}

backend/internal/pipeline/memseed.go

@@ -72,7 +72,17 @@ func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) {
 	var problems []string
 	termKeys := map[[4]string]bool{} // (src,sense,since,until) → detect a duplicate term key
 	for i, t := range sf.Terms {
-		if strings.TrimSpace(t.Src) == "" {
+		// Trim surrounding whitespace on the KEYING surfaces before anything reads them.
+		// normalizeSourceKey does not trim, so a raw " 強敵" would key as " 強敵" and never
+		// match "強敵" — a silently-inert approved term (the A-class hole this bank closes).
+		// A trailing space on sense also makes two identical senses look distinct, slipping
+		// past BOTH the duplicate-key and the same-sense contradiction guards below.
+		// Surrounding whitespace is never intentional; trim it (a forgiving fix, like the
+		// alias dedup). Internal spacing of a multi-word surface is preserved.
+		t.Src = strings.TrimSpace(t.Src)
+		t.Sense = strings.TrimSpace(t.Sense)
+		t.Dst = strings.TrimSpace(t.Dst)
+		if t.Src == "" {
 			problems = append(problems, fmt.Sprintf("term %d: src is required", i))
 			continue
 		}
@@ -98,7 +108,7 @@ func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) {
 		// An approved OR draft term with no dst is silently inert (not matchable — it renders
 		// nothing) yet reads as an intended rendering: fail loud (external-review minor #5).
 		// Only status=auto (a raw candidate — e.g. a ruby reading awaiting a dst) may be empty.
-		if (status == "approved" || status == "draft") && strings.TrimSpace(t.Dst) == "" {
+		if (status == "approved" || status == "draft") && t.Dst == "" {
 			problems = append(problems, fmt.Sprintf("term %q: a %s term must have a non-empty dst (an empty one is silently inert; only status=auto may lack a dst)", t.Src, status))
 			continue
 		}
@@ -125,7 +135,8 @@ func loadGlossarySeed(path string) ([]store.GlossaryEntry, error) {
 		// what matters for matching, and a redundant alias is a harmless authoring slip.
 		seenAlias := map[string]bool{}
 		for _, a := range t.Aliases {
-			if strings.TrimSpace(a.Alias) == "" {
+			a.Alias = strings.TrimSpace(a.Alias) // same silent-inert risk as src (an alias is a match key)
+			if a.Alias == "" {
 				problems = append(problems, fmt.Sprintf("term %q: an alias is empty", t.Src))
 				continue
 			}

backend/internal/pipeline/memseed_test.go

@@ -193,3 +193,34 @@ func TestRubyToCandidates(t *testing.T) {
 		t.Error("図書館 should have been skipped (owned by the manual seed)")
 	}
 }
+
+// TestSeedSurroundingWhitespaceTrimmed covers the workflow-confirmed seed major: surrounding
+// whitespace on a keying surface passed the emptiness check but was stored raw, so the term
+// keyed WITH the space and could never match (silently inert), and a trailing-space sense
+// disguised a same-sense contradiction. Reverting any of the three trims in loadGlossarySeed
+// fails a branch here.
+func TestSeedSurroundingWhitespaceTrimmed(t *testing.T) {
+	// (1) A leading-space src on an approved term must be trimmed, not stored raw — else the
+	// key " 強敵" can never match "強敵" and the approved term renders nothing for the whole book.
+	entries, err := loadGlossarySeed(writeSeed(t, "terms:\n  - { src: \" 強敵\", dst: Соперник }\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 1 || entries[0].Src != "強敵" {
+		t.Fatalf("src surrounding whitespace not trimmed: %+v", entries)
+	}
+	// (2) A trailing-space sense must be trimmed so a same-sense contradiction (same src, same
+	// sense, same default window, DIFFERENT dst) is caught by the duplicate-key guard — not
+	// stored as two distinct-looking rows that both inject a contradictory rendering.
+	if _, err := loadGlossarySeed(writeSeed(t, "terms:\n  - { src: 先生, dst: учитель, sense: teacher }\n  - { src: 先生, dst: доктор, sense: \"teacher \" }\n")); err == nil {
+		t.Error("trailing-space sense must be trimmed so the duplicate (src,sense,window) fails loud")
+	}
+	// (3) An alias with surrounding whitespace must be trimmed (an alias is itself a match key).
+	al, err := loadGlossarySeed(writeSeed(t, "terms:\n  - src: 甲\n    dst: A\n    aliases:\n      - { alias: \" 乙 \", type: x }\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(al) != 1 || len(al[0].Aliases) != 1 || al[0].Aliases[0].Alias != "乙" {
+		t.Fatalf("alias surrounding whitespace not trimmed: %+v", al)
+	}
+}

docs/PROGRESS.md

@@ -483,6 +483,15 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор
 
 **eval low:** #a — тест `TestOverlappingNonNestedKeysBothFire` фиксирует принятое поведение (обе срабатывают); #b/#c/#d — задокументированы (редактор-инъекция = приоритет след. вехи; gate-агрессивность — `FlagGlossaryMiss` не-retryable/не-escalatable, флагает для человека; live-wire кейс — follow-up на `live_conformance`-харнесс). Регресс-тесты: per-lang-min+collision-disposition, gate-ignores-ambiguous (e2e), overlap-windows/empty-dst fail-loud, overlapping-non-nested. Оба вердикта: **ПРИНЯТО**; дефолтный конфиг чист, флип гейта и прод ja-kana разблокированы.
 
+### 2026-07-05 — Шаг 4: глубокое адверсариальное ревью полной поверхности (44 агента)
+
+По запросу владельца — многоагентный воркфлоу поверх `cc57c7b`: 8 finder-дименсий (нормализация / матчер+disposition / F1 / post-check / seed+ruby / store / инъекция+resume / мутационная-стойкость тестов), каждая не-минорная находка → 3 адверсариальных верификатора refute-by-default, подтверждение большинством ≥2/3. **24 кандидата → 2 подтверждённых major, 10 отсеяно (все 0/3–1/3), 12 миноров.** Отсев здоровый: напр. «sticky протаскивает collision-prone AMBIGUOUS в CONFIRMED» зарублен 0/3 верно — sticky не матчит исходник заново, а несёт установленный в сцене термин, collision-риск (срабатывание внутри длинного слова) к нему неприменим. Оба major — класс «тихо пусто», ровно та зона, ради которой банк существует; оба закрыты, каждый фикс **мутационно-проверен** (реверт → падает именно его тест), полный `-race` зелёный.
+
+- **major #1 — default-ignorables в A4-нормализаторах (`memnorm.go`).** Zero-width (U+200B/200C/200D/2060/FEFF) и вариационные селекторы (U+FE0F, IVS) переживали NFKC/NFC → ключ `渡邊` молча не матчил `渡<ZWSP>邊` (частый артефакт EPUB/скрейпа CJK — «тихо пусто» переоткрыт), симметрично на target `<WJ>` внутри слова давал ложный post-check-miss. Фикс: `isIgnorableForMatch` (unicode.Cf + VS-диапазоны) дропает их в ОБОИХ нормализаторах; soft-hyphen U+00AD сохранён как fold→'-' (self-review #8 — порядок проверок гарантирует). `memoryNormAlgoVersion` бампнут **v1→v2** → громкий `--resnapshot`.
+- **major #2 — окружающие пробелы на keying-поверхностях (`memseed.go`).** `loadGlossarySeed` проверял пустоту через TrimSpace, но хранил СЫРОЕ значение: `src: " 強敵"` (approved, непустой dst) проходил валидацию, ключился как `" 強敵"` и НИКОГДА не матчил `強敵` — молча-инертный approved-термин на всю книгу; плюс trailing-space на `sense` маскировал same-sense-противоречие мимо обоих гардов (#4/#5). Фикс: тримминг `src`/`sense`/`dst`/`alias` до построения ключей и хранения (прощающий фикс, как alias-дедуп).
+
+Отсев и 12 миноров оставлены как задокументированные размены/fast-follow (per-lang boundary-тесты post-check, sticky-decay coverage, apostrophe-fold — не гарят на дефолте). Вердикт сессии: **ПРИНЯТО**, шаг 4 закрыт.
+
 ## Полигон
 (секция параллельной сессии — записи добавлять сюда)