package pipeline import ( "reflect" "strings" "testing" "textmachine/backend/internal/store" ) // miner_test.go: synthetic, in-repo unit fixtures for the WS3 default-B miner (no external book data — // the full-book Go↔Python parity is the data-gated TestMinerFullBookParity). Each fixture pins one // load-bearing algorithm piece: the c-value length-multiplier (蛊/转 non-degenerate), subsumption α, the // pattern channels, is_palladius_token, the alias tier-1 rules (the plan's required §3(д) fixtures), the // emission filters, determinism, and the Source:"mined" write path. // mchunks builds miner chunks from raw zh strings (one chunk per chapter, normalized like the runner). func mchunks(sources ...string) []MinerChunk { out := make([]MinerChunk, len(sources)) for i, s := range sources { out[i] = MinerChunk{Chapter: i + 1, ChunkIdx: 0, NSource: normalizeSourceKey(s)} } return out } // smallContrast is a tiny general-zh reference: common function/name chars carry high freq; the domain // formant 蛊 is deliberately ABSENT so its book over-representation is high (the formant signal). func smallContrast(t *testing.T) *Contrast { t.Helper() const data = "的 100000 uj\n是 80000 v\n人 50000 n\n出 9000 v\n现 9000 v\n" + "月 4000 n\n光 3800 n\n希 900 n\n望 3000 v\n影 800 n\n在 20000 p\n此 1200 r\n" + "古 400 nr\n方 900 nr\n源 300 n\n族 500 n\n长 6000 a\n青 700 n\n茅 20 n\n山 5000 n\n" + "四 8000 m\n代 3000 n\n甲 300 n\n等 12000 u\n正 4000 a\n" c, err := LoadContrast(strings.NewReader(data)) if err != nil { t.Fatalf("LoadContrast: %v", err) } return c } func TestMinerLengthMultNonDegenerate(t *testing.T) { // §B1: g(L)=log2(L+1), NOT log2(L) — a single-char candidate (蛊/转) must keep a NON-ZERO length // multiplier, else the catastrophe screen zeros out and 蛊 sinks to rank ~4753. if got := lengthMult(1); got != 1.0 { t.Fatalf("lengthMult(1) = %v, want 1.0 (log2(2)); log2(1)=0 would degenerate |a|=1 candidates", got) } if lengthMult(2) <= lengthMult(1) { t.Fatalf("length multiplier must be monotone in L") } } func TestMinerCValueSingleCharNested(t *testing.T) { // 蛊 (len 1) nested in 蛊师 (len 2): its c-value is g(1)·(f(蛊) − f(蛊师)) — computed, finite, and its // magnitude is non-degenerate because g(1)=1 (not 0). Pins the length-mult fix at the c-value layer. cf := map[string]int{"蛊": 10, "蛊师": 4} cv := computeCValue(cf) want := 1.0 * (10.0 - 4.0/1.0) // g(1)·(f(蛊) − avg over its 1 container) if got := cv["蛊"]; got != want { t.Fatalf("cval[蛊] = %v, want %v (g(1)=1 non-degenerate)", got, want) } if cv["蛊师"] != lengthMult(2)*4 { // non-nested → g(2)·f t.Fatalf("cval[蛊师] = %v, want %v", cv["蛊师"], lengthMult(2)*4) } } func TestMinerSubsumption(t *testing.T) { // 方源(559) survives 方源的(85) [ratio 0.15 < 0.80]; 花酒行(65) is dropped by 花酒行者(65) [ratio 1.0]. cf := map[string]int{"方源": 559, "方源的": 85, "花酒行": 65, "花酒行者": 65} drop := subsumedCandidates(cf, minerSubsumeAlpha) if drop["方源"] { t.Fatalf("方源 must NOT be subsumed by 方源的 (85 < 0.8·559)") } if !drop["花酒行"] { t.Fatalf("花酒行 must be subsumed by 花酒行者 (65 >= 0.8·65)") } } func TestMinerPatternChannels(t *testing.T) { chunks := mchunks( "古月方源走进青茅山。四代族长甲等弟子。", // surname+name, topo, ordinal_title, rank_grade ) // candFreq is only needed for the formant channel; the structural channels scan the source directly. cf := map[string]int{} pats, _ := patternCandidates(chunks, cf, smallContrast(t), 3, 15.0) cases := []struct { src, typ string }{ {"古月方源", "name"}, // surname anchor (compound 古月 + 方源) {"青茅山", "place"}, // topo suffix 山 {"四代族长", "title"}, // ordinal_title 四代+族长 {"甲等", "title"}, // rank_grade 甲+等 {"族长", "title"}, // title_suffix / title_bare } for _, c := range cases { info, ok := pats[normalizeSourceKey(c.src)] if !ok { t.Errorf("pattern channel missed %q", c.src) continue } if !containsStr(info.types, c.typ) { t.Errorf("%q types = %v, want to include %q", c.src, info.types, c.typ) } } } func TestMinerFormantDetection(t *testing.T) { // 蛊 binds ≥3 distinct morphemes among freq≥3 candidates AND is over-represented in-book vs the // contrast (absent there) → auto-detected as a suffix formant; a common char (的) is NOT. chunks := mchunks(strings.Repeat("月光蛊出现。希望蛊出现。月影蛊出现。", 3)) va := runVA(chunks, smallContrast(t), minerFreqFloor, minerSubsumeAlpha) formants := detectFormants(chunks, va.candFreq, smallContrast(t), 3, 15.0) if _, ok := formants['蛊']; !ok { t.Fatalf("蛊 must be auto-detected as a productive formant, got %v", formantKeys(formants)) } if _, ok := formants['的']; ok { t.Fatalf("的 (common char) must NOT be a formant") } } func TestMinerPalladiusToken(t *testing.T) { // Positives: transliterated Chinese-name shapes segment cleanly. for _, tok := range []string{"юань", "фан", "гу", "цзюй"} { if !isPalladiusToken(tok, 1) { t.Errorf("isPalladiusToken(%q) = false, want true (Palladius name shape)", tok) } } // Negative control (palladius.py): common Russian words are NOT Palladius tokens. for _, tok := range []string{"человек", "который", "деревня", "камень"} { if isPalladiusToken(tok, 1) { t.Errorf("isPalladiusToken(%q) = true, want false (common ru word)", tok) } } // The «найти»=най+ти FALSE-POSITIVE the pymorphy3 is_name_lemma gate exists to block: it IS // Palladius-shaped, which is exactly why default B drops the whole ru-side name confirmation channel // (no morphology backend to run the gate) rather than trust the token shape alone. if !isPalladiusToken("найти", 2) { t.Fatalf("найти must be Palladius-shaped (documents why default-B drops the ru name channel)") } } // aliasSurf builds an aliasSurface for the rule tests. func aliasSurf(src, typ, gender, approvedDst string) aliasSurface { return aliasSurface{src: normalizeSourceKey(src), typ: typ, gender: gender, approvedDst: approvedDst} } func TestMinerAliasRules(t *testing.T) { seedSurf := func(ss ...string) map[string]bool { m := map[string]bool{} for _, s := range ss { m[normalizeSourceKey(s)] = true } return m } hasIdent := func(edges []aliasEdge, a, b string) bool { an, bn := normalizeSourceKey(a), normalizeSourceKey(b) for _, e := range edges { if (e.a == an && e.b == bn) || (e.a == bn && e.b == an) { return true } } return false } // R1 extension: 方源 (seed entity) ⊂ 古月方源 → identity (same person, fuller surface). t.Run("R1_extension", func(t *testing.T) { surf := map[string]aliasSurface{ normalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""), normalizeSourceKey("古月方源"): aliasSurf("古月方源", "name", "", ""), } ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("方源")) if !hasIdent(ident, "方源", "古月方源") { t.Fatalf("方源 ⊂ 古月方源 must be an identity extension, got %+v", ident) } }) // Composite guard: 古月 (seed clan) + 族长 (seed title) = 古月族长 is a PHRASE, not an alias (blocks // the clan↔title chaining). But 古月 + 方源 (name) stays a fullname alias. t.Run("composite_guard", func(t *testing.T) { surf := map[string]aliasSurface{ normalizeSourceKey("古月"): aliasSurf("古月", "name", "", ""), normalizeSourceKey("族长"): aliasSurf("族长", "title", "", ""), normalizeSourceKey("古月族长"): aliasSurf("古月族长", "title", "", ""), normalizeSourceKey("古月方源"): aliasSurf("古月方源", "name", "", ""), normalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""), } seeds := seedSurf("古月", "族长", "方源") ident, _, weak := proposeAliasEdges(surf, nil, seeds) if hasIdent(ident, "古月", "古月族长") { t.Fatalf("古月+族长 (clan+title) must be a phrase, NOT an identity alias") } phraseFlagged := false for _, e := range weak { if e.kind == "phrase_not_alias" && e.a == normalizeSourceKey("古月") && e.b == normalizeSourceKey("古月族长") { phraseFlagged = true } } if !phraseFlagged { t.Fatalf("古月+族长 must be flagged phrase_not_alias, got weak=%+v", weak) } if !hasIdent(ident, "古月", "古月方源") { t.Fatalf("古月+方源 (clan+name) must stay a fullname identity alias, got %+v", ident) } }) // R4-iii/v: 族长 (approved «глава клана») ⊂ 四代族长 (approved «четвёртый глава клана») — DIFFERENT // approved dst → different entities EVEN under containment (D38 §3). No identity edge. t.Run("R4_different_approved_dst", func(t *testing.T) { surf := map[string]aliasSurface{ normalizeSourceKey("族长"): aliasSurf("族长", "title", "", "глава клана"), normalizeSourceKey("四代族长"): aliasSurf("四代族长", "title", "", "четвёртый глава клана"), } ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("族长", "四代族长")) if hasIdent(ident, "族长", "四代族长") { t.Fatalf("族长 vs 四代族长 with different approved dst must NOT be identity, got %+v", ident) } }) // R4-ii: different confirmed gender blocks identity; hidden never blocks. t.Run("R4_gender", func(t *testing.T) { surf := map[string]aliasSurface{ normalizeSourceKey("方源"): aliasSurf("方源", "name", "male", ""), normalizeSourceKey("方源儿"): aliasSurf("方源儿", "name", "female", ""), } ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("方源")) if hasIdent(ident, "方源", "方源儿") { t.Fatalf("different confirmed gender must block identity, got %+v", ident) } }) // R4-i / R2: same surname, DIFFERENT given names → family, not identity (方源 vs 方正, surname 方). t.Run("R2_family_R4i", func(t *testing.T) { surf := map[string]aliasSurface{ normalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""), normalizeSourceKey("方正"): aliasSurf("方正", "name", "", ""), } ident, family, _ := proposeAliasEdges(surf, nil, seedSurf("方源", "方正")) if hasIdent(ident, "方源", "方正") { t.Fatalf("same surname + different given names must be family, NOT identity") } if len(family) == 0 { t.Fatalf("方源 ~ 方正 must be a family edge, got none") } }) } func TestMinerEmissionFilters(t *testing.T) { subsumed := map[string]bool{"花酒行": true} seeds := map[string]bool{normalizeSourceKey("方源"): true} cases := []struct { name string c ScoredCand want bool }{ {"typed name freq5", ScoredCand{Src: "龙公", Types: []string{"name"}, Freq: 5}, true}, {"below freq floor", ScoredCand{Src: "龙公", Types: []string{"name"}, Freq: 4}, false}, {"untyped", ScoredCand{Src: "什么", Types: []string{"term"}, Freq: 20}, false}, {"subsumed", ScoredCand{Src: "花酒行", Types: []string{"place"}, Freq: 9}, false}, {"single char", ScoredCand{Src: "蛊", Types: []string{"term", "name"}, Freq: 99}, false}, {"fragment", ScoredCand{Src: normalizeSourceKey("方源心"), Types: []string{"name"}, Freq: 9}, false}, } for _, tc := range cases { if got := emissionEligible(tc.c, subsumed, seeds); got != tc.want { t.Errorf("%s: emissionEligible = %v, want %v", tc.name, got, tc.want) } } } func TestMineBankDeterministicAndNonSeed(t *testing.T) { chunks := mchunks(strings.Repeat("龙公来到青茅山。龙公很强。青茅山很高。", 4)) seed := []store.GlossaryEntry{{Src: "青茅山", Dst: "гора Цинмао", Type: "place", Status: "approved", Source: "seed"}} cfg := frozenMinerConfig() a := MineBank(chunks, smallContrast(t), seed, cfg) b := MineBank(chunks, smallContrast(t), seed, cfg) if !reflect.DeepEqual(a, b) { t.Fatalf("MineBank must be deterministic byte-for-byte:\n a=%+v\n b=%+v", a, b) } // The seed surface 青茅山 must NOT be re-emitted as a new mined term (the curated entry owns it). for _, m := range a { if m.Src == normalizeSourceKey("青茅山") { t.Fatalf("MineBank re-emitted a seed surface as a new term: %+v", m) } } } func TestMinedToCandidatesStampsMinedSource(t *testing.T) { mined := []MinedTerm{ {Src: "龙公", Type: "name", SinceCh: 3, Freq: 7, Aliases: []string{"龙公子"}}, {Src: "青茅山", Type: "place", SinceCh: 1, Freq: 9}, // covered by manual seed → skipped } manual := map[string]bool{"青茅山": true} got := minedToCandidates(mined, manual) if len(got) != 1 { t.Fatalf("want 1 mined candidate (青茅山 skipped as manual), got %d", len(got)) } e := got[0] if e.Source != "mined" { t.Fatalf("mined candidate Source = %q, want \"mined\" (NOT seed — else it moves base-bank-version)", e.Source) } if e.Status != "auto" || e.Dst != "" { t.Fatalf("mined candidate must be status=auto with no dst, got status=%q dst=%q", e.Status, e.Dst) } if e.SinceCh != 3 || e.Confidence != 7 { t.Fatalf("mined candidate metadata wrong: %+v", e) } if len(e.Aliases) != 1 || e.Aliases[0].Alias != "龙公子" || e.Aliases[0].AliasType != "mined" { t.Fatalf("mined alias wrong: %+v", e.Aliases) } } func TestMinedDeltaStaysBaseBankStable(t *testing.T) { // The mined delta (Source:"mined") must NOT move BaseVersion — the load-bearing «переоплата ОДНА» // invariant (§1в): a W1.5 mined-row addition moves only the enriched version. (Sibling of the // Block-A TestBaseEnrichedMemoryVersionSplit, here through the real mined-write path.) seed := []store.GlossaryEntry{{Src: "方源", Dst: "Фан Юань", Status: "approved", Source: "seed"}} base0 := materializeMemory(seed, false) mined := minedToCandidates([]MinedTerm{{Src: "龙公", Type: "name", SinceCh: 1, Freq: 5}}, map[string]bool{"方源": true}) // A mined candidate is status=auto (not folded even into enriched when gate off) — promote it to // approved to exercise the split, as the owner sign would at W1.5. mined[0].Status = "approved" mined[0].Dst = "Лун Гун" enriched := append(append([]store.GlossaryEntry{}, seed...), mined...) bank1 := materializeMemory(enriched, false) if base0.BaseVersion() != bank1.BaseVersion() { t.Fatalf("base-bank-version moved on a Source:mined addition — snapshot_W1 would re-bill W1") } if base0.Version() == bank1.Version() { t.Fatalf("enriched version must move on a mined approved addition") } } func formantKeys(m map[rune]formantInfo) []string { out := make([]string, 0, len(m)) for r := range m { out = append(out, string(r)) } return out }