209 lines
11 KiB
Go
209 lines
11 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// memory_e1_test.go: the E1 MEASUREMENT the onboarding contract requires BEFORE the
|
||
// post-check may be trusted as a hard gate (research/13 §2 "post-check в русском —
|
||
// несущий И невалидированный"; research/14 §2 quantified it at 18–36% false flags for a
|
||
// naive matcher). It measures the decl-aware post-check's false-flag rate on REAL
|
||
// inflected Russian (the adaptive_levers/adaptive_reask canon: 阿Q ch.5–9, Rogov),
|
||
// empirically — not "примерно" (the agent's explicit ask).
|
||
//
|
||
// It answers two things the agent flagged:
|
||
// 1. Stored-decl WORKS when the forms are complete → 0 false flags on a correctly
|
||
// rendered, inflected chunk (the arm that lets the post-check be a real signal).
|
||
// 2. Stored-decl is NOT free-reliable: with only the base form (an under-filled decl,
|
||
// the OOV-translit risk — one cheap LLM call may miss cases) the SAME chunk yields
|
||
// false flags. This is why v1 keeps the post-check a FLAGGER, not a hard gate: if
|
||
// the seed under-fills decl, it must degrade to observability, not block chunks.
|
||
//
|
||
// The inflection cases are the exact ones research/14 measured a naive matcher losing:
|
||
// «Вэйчжуане» (loc, a SUFFIX inflection substring-matching handles), «Бородатого Вана»
|
||
// (a modifier that inflects INTERNALLY — substring of the base fails), «сюцая»
|
||
// (stem-final й→я — the base is not a substring), «Маленького Дэна» (internal).
|
||
|
||
// e1Entity is one canon entity with its full decl set.
|
||
type e1Entity struct {
|
||
src string
|
||
baseDst string
|
||
fullDecl []string
|
||
}
|
||
|
||
var e1Canon = []e1Entity{
|
||
{"阿Q", "А-кью", []string{"А-кью"}}, // hyphenated translit, invariant
|
||
{"未庄", "Вэйчжуан", []string{"Вэйчжуан", "Вэйчжуане", "Вэйчжуана", "Вэйчжуану", "Вэйчжуаном"}},
|
||
{"王胡", "Бородатый Ван", []string{"Бородатый Ван", "Бородатого Вана", "Бородатому Вану", "Бородатым Ваном", "Бородатом Ване"}},
|
||
{"秀才", "сюцай", []string{"сюцай", "сюцая", "сюцаю", "сюцае", "сюцаем"}},
|
||
{"小D", "Маленький Дэн", []string{"Маленький Дэн", "Маленького Дэна", "Маленькому Дэну", "Маленьким Дэном", "Маленьком Дэне"}},
|
||
{"邹七嫂", "тётушка Цзоу Седьмая", []string{"тётушка Цзоу Седьмая", "тётушки Цзоу Седьмой", "тётушку Цзоу Седьмую"}},
|
||
}
|
||
|
||
// e1Source contains every src key so all six entities are injected.
|
||
const e1Source = "阿Q在未庄遇见王胡和秀才，又碰到小D，邹七嫂在一旁看着。"
|
||
|
||
// e1CorrectOutput is a publisher-quality rendering: every entity present but INFLECTED
|
||
// (the case a boundary regexp / a base-only matcher false-flags).
|
||
const e1CorrectOutput = "В Вэйчжуане Бородатого Вана и сюцая знали все. " +
|
||
"А-кью повстречал Маленького Дэна у реки; тётушка Цзоу Седьмая молча прошла мимо."
|
||
|
||
func e1Glossary(useFullDecl bool) []store.GlossaryEntry {
|
||
var out []store.GlossaryEntry
|
||
for _, e := range e1Canon {
|
||
g := gl(e.src, e.baseDst, "", "approved")
|
||
if useFullDecl {
|
||
g.Decl = declJSON(false, e.fullDecl...)
|
||
} else {
|
||
g.Decl = declJSON(true, e.baseDst) // naive: only the base form
|
||
}
|
||
out = append(out, g)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func TestE1PostcheckFalseFlagRate(t *testing.T) {
|
||
// Arm A — full decl: the post-check must NOT false-flag a correct inflected output.
|
||
bFull := bankFrom(e1Glossary(true))
|
||
selFull := bFull.Select(e1Source, 1, nil, 0)
|
||
if len(selFull.injected) != len(e1Canon) {
|
||
t.Fatalf("expected all %d canon entities injected, got %d", len(e1Canon), len(selFull.injected))
|
||
}
|
||
fullMiss := bFull.postcheck(selFull.injected, e1CorrectOutput)
|
||
if len(fullMiss) != 0 {
|
||
t.Errorf("STORED-DECL FALSE FLAGS on correct output: %d/%d — %v", len(fullMiss), len(selFull.injected), fullMiss)
|
||
}
|
||
|
||
// Arm B — base only (under-filled decl): the SAME correct output now false-flags.
|
||
bBase := bankFrom(e1Glossary(false))
|
||
selBase := bBase.Select(e1Source, 1, nil, 0)
|
||
baseMiss := bBase.postcheck(selBase.injected, e1CorrectOutput)
|
||
|
||
fullRate := float64(len(fullMiss)) / float64(len(selFull.injected))
|
||
baseRate := float64(len(baseMiss)) / float64(len(selBase.injected))
|
||
t.Logf("E1 false-flag rate on real inflected Russian: full-decl %.0f%% (%d/%d), base-only %.0f%% (%d/%d)",
|
||
fullRate*100, len(fullMiss), len(selFull.injected),
|
||
baseRate*100, len(baseMiss), len(selBase.injected))
|
||
|
||
// The measurement's verdict: decl-awareness is essential (full ≪ base), and the
|
||
// stored-decl matcher is trustworthy ONLY when complete. Base-only reproduces the
|
||
// research/14 failure — the reason the post-check stays a flagger until validated.
|
||
if len(baseMiss) == 0 {
|
||
t.Error("base-only decl produced 0 false flags — the E1 measurement lost its hard inflection cases; check the fixture")
|
||
}
|
||
if len(fullMiss) >= len(baseMiss) {
|
||
t.Errorf("full decl (%d) did not reduce false flags vs base (%d)", len(fullMiss), len(baseMiss))
|
||
}
|
||
// The specific research/14 internal-inflection cases MUST be the base-only misses.
|
||
for _, src := range []string{"王胡", "秀才", "小D"} {
|
||
if !hasMiss(baseMiss, src) {
|
||
t.Errorf("base-only decl did not false-flag %s (expected an internal-inflection false flag)", src)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestPostcheckWhitespaceAndDashRobust covers self-review #2 (multi-word name wrapped
|
||
// across a newline / doubled space / NBSP) and #8 (hyphen/dash variants): a correctly
|
||
// rendered term in ANY of these forms must NOT be false-flagged.
|
||
func TestPostcheckWhitespaceAndDashRobust(t *testing.T) {
|
||
wang := gl("王胡", "Бородатый Ван", "", "approved")
|
||
wang.Decl = declJSON(false, "Бородатый Ван", "Бородатого Вана")
|
||
ahq := gl("阿Q", "А-кью", "", "approved")
|
||
ahq.Decl = declJSON(true, "А-кью")
|
||
b := bankFrom([]store.GlossaryEntry{wang, ahq})
|
||
sel := b.Select("王胡和阿Q在场。", 1, nil, 0)
|
||
|
||
robust := []string{
|
||
"Бородатого Вана и А-кью видели.", // baseline
|
||
"Бородатого\nВана и А-кью видели.", // #2 newline between the name's words
|
||
"Бородатого Вана и А-кью видели.", // #2 doubled space
|
||
"Бородатого Вана и А-кью видели.", // #2 NBSP
|
||
"Бородатого Вана и А–кью видели.", // #8 en-dash in А-кью
|
||
"Бородатого Вана и А‑кью видели.", // #8 non-breaking hyphen
|
||
}
|
||
for _, out := range robust {
|
||
if m := b.postcheck(sel.injected, out); len(m) != 0 {
|
||
t.Errorf("false flag on a correctly-rendered variant %q: %v", out, m)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestPostcheckWholeWordNoMasking covers self-review #3: a DROPPED short translit name
|
||
// must not be masked by an unrelated common word that merely contains its letters.
|
||
func TestPostcheckWholeWordNoMasking(t *testing.T) {
|
||
van := gl("王", "Ван", "", "approved")
|
||
van.AllowShort = true // single-char src, allowed here to exercise the target-side check
|
||
van.Decl = declJSON(false, "Ван", "Вана", "Вану", "Ваном", "Ване")
|
||
b := bankFrom([]store.GlossaryEntry{van})
|
||
sel := b.Select("王走了。", 1, nil, 0)
|
||
if len(sel.injected) != 1 {
|
||
t.Fatalf("expected 王 injected, got %d", len(sel.injected))
|
||
}
|
||
// 王 is ABSENT from each output, yet a naive substring would find «ван» inside these.
|
||
for _, masked := range []string{"Мимо прошёл караван верблюдов.", "Он сел на диван и молчал.", "Иван Петрович кивнул."} {
|
||
if m := b.postcheck(sel.injected, masked); !hasMiss(m, "王") {
|
||
t.Errorf("post-check masked a dropped name by an unrelated word: %q → %v", masked, m)
|
||
}
|
||
}
|
||
// A genuine rendering (inflected) is found as a whole word.
|
||
if m := b.postcheck(sel.injected, "К нему подошёл Ван."); len(m) != 0 {
|
||
t.Errorf("false flag on a correct whole-word rendering: %v", m)
|
||
}
|
||
if m := b.postcheck(sel.injected, "Он говорил с Ваном о делах."); len(m) != 0 {
|
||
t.Errorf("false flag on a correct inflected rendering: %v", m)
|
||
}
|
||
}
|
||
|
||
// TestPostcheckBaseDstAlwaysChecked is the D24.4 repro for the acceptance-stage-A
|
||
// nominative_gap (37/55 false misses): an entry whose decl.forms lists ONLY oblique
|
||
// cases while the approved base dst appears in the NOMINATIVE in the output. The base
|
||
// dst is itself an approved form, so it must ALWAYS be in the accepted set — not only as
|
||
// a fallback when decl is empty. Mutation: revert dstFormPresent to the empty-decl
|
||
// fallback (`forms := e.declForms; if len(forms)==0 {forms=[base]}`) and the positive
|
||
// case below false-flags again → the test goes red.
|
||
func TestPostcheckBaseDstAlwaysChecked(t *testing.T) {
|
||
// 方源→Фан Юань, decl carries only oblique cases (the under-seeded reality of stage A).
|
||
fang := gl("方源", "Фан Юань", "", "approved")
|
||
fang.Decl = declJSON(false, "Фан Юаня", "Фан Юаню", "Фан Юанем") // oblique-only, NO nominative base
|
||
b := bankFrom([]store.GlossaryEntry{fang})
|
||
sel := b.Select("方源走了。", 1, nil, 0)
|
||
if len(sel.injected) != 1 {
|
||
t.Fatalf("expected 方源 injected, got %d", len(sel.injected))
|
||
}
|
||
// Positive (the fix): the nominative base present but ABSENT from the oblique-only
|
||
// decl → must NOT be a miss now that the base is always accepted.
|
||
if m := b.postcheck(sel.injected, "На поляну вышел Фан Юань, окружённый гу."); len(m) != 0 {
|
||
t.Fatalf("nominative base dst present yet flagged — the base form must always be accepted (D24.4 nominative_gap): %v", m)
|
||
}
|
||
// The oblique decl forms still work (decl-awareness retained, not replaced).
|
||
if m := b.postcheck(sel.injected, "Все боялись Фан Юаня."); len(m) != 0 {
|
||
t.Fatalf("oblique decl form present yet flagged: %v", m)
|
||
}
|
||
// Negative (precision preserved): neither the base nor any decl form present → a real
|
||
// miss. The union only ADDS an accepted form; it must not blind a genuine drop.
|
||
if m := b.postcheck(sel.injected, "На поляну никто не вышел."); !hasMiss(m, "方源") {
|
||
t.Fatalf("dst and all forms absent must still flag a miss (union must not blind precision): %v", m)
|
||
}
|
||
}
|
||
|
||
// TestE1PostcheckCatchesPoisoning: with complete decl, a WRONG rendering (poisoning /
|
||
// the model ignoring the glossary) is still a true-positive flag — the post-check's
|
||
// actual job. Precision does not come for free: the decl forms are inflections of the
|
||
// FULL dst phrase, so a foreign name that merely shares the head noun does NOT slip past.
|
||
func TestE1PostcheckCatchesPoisoning(t *testing.T) {
|
||
b := bankFrom(e1Glossary(true))
|
||
sel := b.Select("王胡走过来。", 1, nil, 0) // only 王胡 fires
|
||
if len(sel.injected) != 1 {
|
||
t.Fatalf("expected 1 injected, got %d", len(sel.injected))
|
||
}
|
||
// Correct inflected rendering → no flag.
|
||
if m := b.postcheck(sel.injected, "К нему подошёл Бородатый Ван."); len(m) != 0 {
|
||
t.Errorf("false flag on correct rendering: %v", m)
|
||
}
|
||
// Poisoned rendering (a different name) → flagged (true positive).
|
||
if m := b.postcheck(sel.injected, "К нему подошёл Ван Ху."); !hasMiss(m, "王胡") {
|
||
t.Errorf("post-check missed the poisoning (Ван Ху ≠ Бородатый Ван): %v", m)
|
||
}
|
||
}
|