textmachine/backend/internal/pipeline/memory_test.go

package pipeline

import (
	"encoding/json"
	"strings"
	"testing"

	"textmachine/backend/internal/store"
)

// memory_test.go ports the executable spec eval/memory_hotpath.py (T1–T10) to Go
// unit tests, per the onboarding contract — but NOT the accept-REGEXPS verbatim
// (research/14: a \b-regexp is the 18–36% false-flag source). The selection tests
// (T1–T8) are the same cases; the post-check tests (T9/T10) use the decl-aware
// matcher (memory_e1_test.go measures its false-flag rate on real Russian).

func gl(src, dst, sense, status string) store.GlossaryEntry {
	return store.GlossaryEntry{BookID: "b", Src: src, Dst: dst, Sense: sense, Status: status, Source: "seed"}
}

func declJSON(invariant bool, forms ...string) string {
	b, _ := json.Marshal(declInfo{Invariant: invariant, Forms: forms})
	return string(b)
}

func alias(a string) store.GlossaryAlias { return store.GlossaryAlias{Alias: a, AliasType: "прозвище"} }

// specGlossary mirrors eval/memory_hotpath.py's G (with decl forms replacing the
// toy accept-regexps).
func specGlossary() []store.GlossaryEntry {
	ahq := gl("阿Q", "А-кью", "name", "approved")
	ahq.Decl = declJSON(true, "А-кью", "А кью")

	sishu := gl("四叔", "Четвёртый дядюшка", "char", "approved")
	sishu.Aliases = []store.GlossaryAlias{alias("鲁四老爷")}
	sishu.Decl = declJSON(false, "дядюшка", "дядюшки", "дядюшке", "дядюшку", "дядюшкой")

	luzhen := gl("鲁镇", "Лучжэнь", "", "approved")
	luzhen.Decl = declJSON(false, "Лучжэнь", "Лучжэня", "Лучжэне")

	songzao := gl("送灶", "проводы бога очага", "festival", "approved")
	songzao.Decl = declJSON(true, "бога очага")

	xiaoyan := gl("萧炎", "Сяо Янь", "name", "approved")
	xiaoyan.Aliases = []store.GlossaryAlias{alias("炎哥")}
	xiaoyan.Decl = declJSON(true, "Сяо Янь")

	xiuwei := gl("修为", "уровень совершенствования", "cultivation", "approved")
	xiuwei.Decl = declJSON(false, "совершенствования", "совершенствование")

	shadow := gl("影卫", "Тёмный страж", "spoiler", "approved")
	shadow.SinceCh = 200
	shadow.Decl = declJSON(false, "Тёмный страж", "Тёмного стража")

	newname := gl("小D", "Малыш Дэ", "", "auto")
	newname.Decl = declJSON(false, "Малыш Дэ", "Малыша Дэ")

	return []store.GlossaryEntry{ahq, sishu, luzhen, songzao, xiaoyan, xiuwei, shadow, newname}
}

func bankFrom(entries []store.GlossaryEntry) *MemoryBank { return materializeMemory(entries, false) }

// injMap maps injected src → disposition for assertions.
func injMap(sel memorySelection) map[string]injectionDisposition {
	m := map[string]injectionDisposition{}
	for _, p := range sel.injected {
		m[p.entry.src] = p.disp
	}
	return m
}

func TestHotPathSpecT1toT10(t *testing.T) {
	b := bankFrom(specGlossary())
	const budget = 0

	// T1: exact + alias select; alias 鲁四老爷 pulls 四叔.
	sel := b.Select("阿Q走进鲁镇，遇见鲁四老爷。", 1, nil, budget)
	m := injMap(sel)
	if m["阿Q"] != memConfirmed || m["鲁镇"] != memConfirmed || m["四叔"] != memConfirmed {
		t.Errorf("T1 exact+alias: %v", m)
	}

	// T2: 送到灶 (literal) — 送灶 must NOT inject (bigram absent, 灶 single-key banned).
	sel = b.Select("厨房里，老妈子把饭菜送到灶间去热一热。", 1, nil, budget)
	if _, ok := injMap(sel)["送灶"]; ok {
		t.Error("T2 wrong-sense 送灶 injected (A3 homograph)")
	}

	// T3: 炎势 (flame) — 萧炎 must NOT inject.
	sel = b.Select("山火蔓延，炎势冲天，村民连夜逃走。", 1, nil, budget)
	if _, ok := injMap(sel)["萧炎"]; ok {
		t.Error("T3 wrong-sense 炎 injected")
	}

	// T4: 修好 (repair) — 修为 must NOT inject.
	sel = b.Select("工匠把石桥修好了。", 1, nil, budget)
	if _, ok := injMap(sel)["修为"]; ok {
		t.Error("T4 wrong-sense 修 injected")
	}

	// T5: trad→simp normalization — 魯鎮 matches key 鲁镇 (A4).
	sel = b.Select("魯鎮的冬天很冷。", 1, nil, budget)
	if _, ok := injMap(sel)["鲁镇"]; !ok {
		t.Error("T5 trad→simp: 魯鎮 did not match 鲁镇")
	}

	// T6: pronominal chunk — sticky carries 四叔 from the previous chunk.
	sishuID := "四叔\x1fchar\x1f0\x1f0"
	sel = b.Select("他慢慢站起来，叹了口气。", 2, map[string]bool{sishuID: true}, budget)
	var stickySishu *pickedEntry
	for i := range sel.injected {
		if sel.injected[i].entry.src == "四叔" {
			stickySishu = &sel.injected[i]
		}
	}
	if stickySishu == nil || stickySishu.via != "sticky" {
		t.Errorf("T6 sticky scene-inertia failed: %+v", injMap(sel))
	}

	// T7: spoiler — 影卫 (since_ch=200) in chapter 5 → hard reject.
	sel = b.Select("影卫出现了。", 5, nil, budget)
	if _, ok := injMap(sel)["影卫"]; ok {
		t.Error("T7 spoiler 影卫 injected in ch5")
	}
	if len(sel.rejected) != 1 || sel.rejected[0].entry.src != "影卫" {
		t.Errorf("T7 spoiler not rejected+logged: %+v", sel.rejected)
	}
	// T7b: same entry passes after the reveal chapter.
	sel = b.Select("影卫出现了。", 250, nil, budget)
	if _, ok := injMap(sel)["影卫"]; !ok {
		t.Error("T7b spoiler entry did not pass at ch250")
	}

	// T8: auto term 小D → ambiguous disposition.
	sel = b.Select("小D也来了。", 1, nil, budget)
	if injMap(sel)["小D"] != memAmbiguous {
		t.Errorf("T8 auto-term not ambiguous: %v", injMap(sel))
	}

	// T9: post-check catches a missing dst (leak) and passes a present one.
	sel = b.Select("阿Q走进鲁镇。", 1, nil, budget)
	bad := b.postcheck(sel.injected, "Некто вошёл в Лучжэнь.") // А-кью dropped
	good := b.postcheck(sel.injected, "А-кью вошёл в Лучжэнь.")
	if !hasMiss(bad, "阿Q") {
		t.Errorf("T9 post-check missed the leak: %+v", bad)
	}
	if len(good) != 0 {
		t.Errorf("T9 post-check false-flagged a correct output: %+v", good)
	}

	// T10: post-check catches poisoning — injected dst absent, output used a foreign name.
	sel = b.Select("阿Q走进鲁镇。", 1, nil, budget)
	poison := b.postcheck(sel.injected, "Линь Чун вошёл в деревню Вэйчжуан.")
	if !hasMiss(poison, "阿Q") {
		t.Errorf("T10 post-check missed the poisoning: %+v", poison)
	}
}

func hasMiss(misses []postcheckMiss, src string) bool {
	for _, m := range misses {
		if m.Src == src {
			return true
		}
	}
	return false
}

// TestTrapCorpus ports the retrieval_bench wrong-sense traps: exact multi-char match
// must fire 0/N on homograph chunks that contain the KEY CHARACTER but not the term.
func TestTrapCorpus(t *testing.T) {
	entries := []store.GlossaryEntry{
		gl("送灶", "проводы бога очага", "festival", "approved"),
		gl("修为", "уровень совершенствования", "cultivation", "approved"),
		gl("萧炎", "Сяо Янь", "name", "approved"),
		gl("斗气", "боевая ци", "term", "approved"),
		gl("钱老爷", "господин Цянь", "name", "approved"), // multi-char; the bare 钱 homograph must not fire
	}
	b := bankFrom(entries)
	traps := []struct{ name, text string }{
		{"送到灶 literal", "厨房里冷了，老妈子把剩下的饭菜送到灶间去重新热一热。"},
		{"修好 repair", "工匠们花了三天，终于把那条被山洪冲垮的石桥修好了。"},
		{"炎势 flame", "山火蔓延，炎势冲天，浓烟滚滚，村民连夜收拾细软。"},
		{"雾气 mist", "清晨的山谷里雾气很重，空气格外清新，露水打湿了草叶。"},
		{"钱 money homograph", "他摸出几个铜钱，买了一碗热汤。"}, // 钱 present but 钱老爷 absent
		{"Chekhov negative", "Дмитрий Гуров сидел в ялтинском павильоне и лениво пил чай."},
	}
	for _, tr := range traps {
		t.Run(tr.name, func(t *testing.T) {
			sel := b.Select(tr.text, 1, nil, 0)
			if len(sel.injected) != 0 {
				t.Errorf("trap fired %d injection(s), want 0: %v", len(sel.injected), injMap(sel))
			}
		})
	}
}

// TestLongestMatchContainment: a shorter key fully inside a longer key's span is
// suppressed (longest-match / whole-entity replacement, A3), but the shorter entity
// still fires when it stands alone.
func TestLongestMatchContainment(t *testing.T) {
	entries := []store.GlossaryEntry{
		gl("阿Q", "А-кью", "name", "approved"),
		gl("阿Q正传", "«Подлинная история А-кью»", "title", "approved"),
	}
	b := bankFrom(entries)
	// Inside the title: only the title fires.
	sel := b.Select("我读了《阿Q正传》。", 1, nil, 0)
	m := injMap(sel)
	if _, ok := m["阿Q正传"]; !ok {
		t.Errorf("title did not fire: %v", m)
	}
	if _, ok := m["阿Q"]; ok {
		t.Errorf("short key 阿Q fired inside the longer title span: %v", m)
	}
	// Standalone: the character name fires.
	sel = b.Select("阿Q走进村子。", 1, nil, 0)
	if _, ok := injMap(sel)["阿Q"]; !ok {
		t.Error("阿Q did not fire standalone")
	}
}

func TestSingleKeyBanAndAllowShort(t *testing.T) {
	banned := gl("炎", "пламя-как-имя", "", "approved") // single Han → banned by default
	b := bankFrom([]store.GlossaryEntry{banned})
	if sel := b.Select("炎势冲天。", 1, nil, 0); len(sel.injected) != 0 {
		t.Error("single-char key fired despite the A3 ban")
	}
	// allow_short escape hatch lets it fire.
	banned.AllowShort = true
	b = bankFrom([]store.GlossaryEntry{banned})
	if sel := b.Select("炎势冲天。", 1, nil, 0); len(sel.injected) != 1 {
		t.Error("allow_short did not re-enable the single-char key")
	}
}

func TestBudgetEviction(t *testing.T) {
	entries := []store.GlossaryEntry{
		gl("甲甲", "Альфа", "", "approved"),
		gl("乙乙", "Бета", "", "auto"),   // ambiguous → lower priority
		gl("丙丙", "Гамма", "", "approved"),
	}
	b := bankFrom(entries)
	chunk := "甲甲、乙乙、丙丙都在。"
	// Unbounded: all three fire, in priority order (the two approved before the auto).
	all := b.Select(chunk, 1, nil, 0)
	if len(all.injected) != 3 {
		t.Fatalf("unbounded: want 3 injected, got %d", len(all.injected))
	}
	// A token budget that fits exactly the top TWO priority records (computed with the
	// same cost helper Select uses, so the test never drifts from the implementation).
	budget := glossaryLineTokens(all.injected[0].entry) + glossaryLineTokens(all.injected[1].entry)
	sel := b.Select(chunk, 1, nil, budget)
	if len(sel.injected) != 2 || len(sel.evicted) != 1 {
		t.Fatalf("budget: injected=%d evicted=%d (budget=%d)", len(sel.injected), len(sel.evicted), budget)
	}
	// The auto (ambiguous) term is the one evicted (confirmed>ambiguous priority, F2).
	if sel.evicted[0].entry.src != "乙乙" {
		t.Errorf("eviction priority wrong: evicted %s (want the auto term 乙乙)", sel.evicted[0].entry.src)
	}
}

// TestSelectDeterminism runs Select many times over a multi-hit chunk and asserts the
// injected order is byte-identical — guards against a map-iteration leak (the
// determinism invariant: the injected bytes enter request_hash).
func TestSelectDeterminism(t *testing.T) {
	b := bankFrom(specGlossary())
	chunk := "阿Q走进鲁镇，遇见鲁四老爷，谈起修为。"
	first := selKey(b.Select(chunk, 1, nil, 0))
	for i := 0; i < 200; i++ {
		if got := selKey(b.Select(chunk, 1, nil, 0)); got != first {
			t.Fatalf("non-deterministic Select: run %d gave %q, first was %q", i, got, first)
		}
	}
}

func selKey(sel memorySelection) string {
	s := ""
	for _, p := range sel.injected {
		s += p.entry.src + ":" + string(p.disp) + "|"
	}
	return s
}

// TestEmptyDstNotMatchable covers self-review #1: a ruby candidate (dst="") must not be
// matchable — it renders nothing, so admitting it would waste budget, evict a real line,
// and inflate the exact-hit count.
func TestEmptyDstNotMatchable(t *testing.T) {
	phantom := gl("甲甲", "", "", "auto") // ruby-style candidate, no dst
	real := gl("乙乙", "Настоящий", "", "auto")
	b := bankFrom([]store.GlossaryEntry{phantom, real})
	// Tiny budget that would fit exactly one line: the phantom must not steal it.
	budget := glossaryLineTokens(&memoryEntry{src: "乙乙", dst: "Настоящий"})
	sel := b.Select("甲甲和乙乙都在场。", 1, nil, budget)
	if len(sel.injected) != 1 || sel.injected[0].entry.src != "乙乙" {
		t.Fatalf("empty-dst phantom polluted selection: injected=%v", injMap(sel))
	}
	if renderGlossaryBlock(sel.injected) == "" {
		t.Error("a renderable line was evicted by an empty-dst phantom")
	}
	// Unbounded: the phantom never appears as an exact hit.
	sel = b.Select("甲甲和乙乙都在场。", 1, nil, 0)
	if _, ok := injMap(sel)["甲甲"]; ok {
		t.Error("empty-dst phantom counted as an exact hit")
	}
}

// TestSpoilerBeforeSuppression covers self-review #6: a spoiler-blocked LONGER key must
// not suppress a valid SHORTER name nested inside it.
func TestSpoilerBeforeSuppression(t *testing.T) {
	short := gl("林动", "Линь Дун", "", "approved")
	long := gl("林动的父亲", "отец Линь Дуна", "", "approved")
	long.UntilCh = 3 // a spoiler: valid only through chapter 3
	b := bankFrom([]store.GlossaryEntry{short, long})

	// Chapter 10: the longer key is spoiler-blocked; the shorter valid name must survive.
	sel := b.Select("林动的父亲缓缓转身。", 10, nil, 0)
	if _, ok := injMap(sel)["林动"]; !ok {
		t.Errorf("valid shorter name dropped by a spoiler-blocked longer key: %v", injMap(sel))
	}
	if _, ok := injMap(sel)["林动的父亲"]; ok {
		t.Error("spoiler-blocked longer key was injected")
	}
	if len(sel.rejected) != 1 || sel.rejected[0].entry.src != "林动的父亲" {
		t.Errorf("spoiler-blocked longer key not logged as rejected: %+v", sel.rejected)
	}
	// Chapter 2 (longer key in window): longest-match still prefers the longer entity.
	sel = b.Select("林动的父亲缓缓转身。", 2, nil, 0)
	if _, ok := injMap(sel)["林动"]; ok {
		t.Error("longest-match failed when the longer key is spoiler-valid")
	}
	if _, ok := injMap(sel)["林动的父亲"]; !ok {
		t.Error("spoiler-valid longer key was not injected at ch2")
	}
}

// TestBudgetFloorFree covers self-review #7: the per-line budget cost is floor-free, so a
// block of short name lines is not ~1.85×-overcounted and needlessly evicted.
func TestBudgetFloorFree(t *testing.T) {
	var entries []store.GlossaryEntry
	srcs := []string{"甲甲", "乙乙", "丙丙", "丁丁", "戊戊", "己己", "庚庚", "辛辛"}
	dsts := []string{"Ко", "Оцу", "Хэй", "Тэй", "Бо", "Ки", "Ко2", "Син"}
	for i := range srcs {
		entries = append(entries, gl(srcs[i], dsts[i], "", "approved"))
	}
	b := bankFrom(entries)
	chunk := strings.Join(srcs, "、") + "都在场。"
	all := b.Select(chunk, 1, nil, 0)
	if len(all.injected) != 8 {
		t.Fatalf("want 8 injected, got %d", len(all.injected))
	}
	// A budget equal to the EXACT sum of the (floor-free) line costs fits all 8.
	budget := 0
	for _, p := range all.injected {
		budget += glossaryLineTokens(p.entry)
	}
	sel := b.Select(chunk, 1, nil, budget)
	if len(sel.injected) != 8 || len(sel.evicted) != 0 {
		t.Errorf("floor-free budget=%d should fit all 8: injected=%d evicted=%d", budget, len(sel.injected), len(sel.evicted))
	}
}

// TestPerLanguageMinKeyAndCollisionDisposition covers external-review major #2/#3: a
// purely-phonetic key needs ≥3 chars (a 2-char kana/latin key collides inside ordinary
// words), and a short (≤3) phonetic key injects as AMBIGUOUS even when approved.
func TestPerLanguageMinKeyAndCollisionDisposition(t *testing.T) {
	entries := []store.GlossaryEntry{
		gl("鈴木", "Судзуки", "", "approved"),         // 2 Han → fires, CONFIRMED (ideographic anchor)
		gl("すずき", "Судзуки-х", "s2", "approved"),    // 3 kana → fires but AMBIGUOUS (collision-prone)
		gl("ながいなまえ", "Длинное имя", "", "approved"), // 6 kana → fires, CONFIRMED (long enough)
		gl("リン", "Рин", "", "approved"),              // 2 kana → BANNED (phonetic min 3)
		gl("ai", "ИИ", "", "approved"),                 // 2 latin → BANNED (phonetic min 3)
	}
	b := bankFrom(entries)
	sel := b.Select("鈴木とすずきとながいなまえが会った。", 1, nil, 0)
	m := injMap(sel)
	if m["鈴木"] != memConfirmed {
		t.Errorf("Han key must be CONFIRMED: %v", m)
	}
	if m["すずき"] != memAmbiguous {
		t.Errorf("short phonetic key must be AMBIGUOUS (A2 collision): %v", m)
	}
	if m["ながいなまえ"] != memConfirmed {
		t.Errorf("long phonetic key must be CONFIRMED: %v", m)
	}
	// The collision cases external-review named: a short phonetic key must NOT fire inside
	// an ordinary word (リン in リンゴ/apple, ai in raid).
	if s := b.Select("リンゴを食べた。", 1, nil, 0); len(s.injected) != 0 {
		t.Errorf("2-kana key fired inside リンゴ (collision): %v", injMap(s))
	}
	if s := b.Select("raidアレイを組んだ。", 1, nil, 0); len(s.injected) != 0 {
		t.Errorf("2-latin key fired inside raid (collision): %v", injMap(s))
	}
	// allow_short is the author's explicit override: the short key fires AND as CONFIRMED.
	rin := gl("リン", "Рин", "", "approved")
	rin.AllowShort = true
	b2 := bankFrom([]store.GlossaryEntry{rin})
	if s := b2.Select("リンさんが来た。", 1, nil, 0); injMap(s)["リン"] != memConfirmed {
		t.Errorf("allow_short must let リン fire as CONFIRMED: %v", injMap(s))
	}
}

// TestOverlappingNonNestedKeysBothFire documents the eval-review low finding: two keys
// that OVERLAP without nesting both fire (both surfaces are genuinely present in the
// text). Accepted behavior (precision-side), asserted so a future change is deliberate.
func TestOverlappingNonNestedKeysBothFire(t *testing.T) {
	entries := []store.GlossaryEntry{
		gl("甲乙", "Пара-А", "", "approved"),
		gl("乙丙", "Пара-Б", "", "approved"),
	}
	b := bankFrom(entries)
	sel := b.Select("甲乙丙都在。", 1, nil, 0) // 甲乙 at [0,2), 乙丙 at [1,3) — overlap, neither contains the other
	m := injMap(sel)
	if m["甲乙"] != memConfirmed || m["乙丙"] != memConfirmed {
		t.Errorf("overlapping non-nested keys should both fire: %v", m)
	}
}

func TestInjectivityCollision(t *testing.T) {
	entries := []store.GlossaryEntry{
		{BookID: "b", Src: "甲", Dst: "Мастер", Status: "approved"},
		{BookID: "b", Src: "乙", Dst: "Мастер", Status: "approved"}, // same dst → collision (B2)
		{BookID: "b", Src: "丙", Dst: "Ученик", Status: "approved"},
	}
	cols := injectivityCollisions(entries)
	if len(cols) != 1 {
		t.Fatalf("want 1 dst collision, got %d: %v", len(cols), cols)
	}
}