358 lines
16 KiB
Go
358 lines
16 KiB
Go
package miner
|
||
|
||
import (
|
||
"reflect"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/lang"
|
||
"textmachine/backend/internal/store"
|
||
"textmachine/backend/internal/text"
|
||
)
|
||
|
||
// testLangPack loads the real in-repo zh→ru language pack (configs/langpacks/) for the miner fixtures.
|
||
// The pack files are byte-derived from the former constants (langpack move, D39.15), so the miner is
|
||
// exercised over the exact same data.
|
||
func testLangPack(t *testing.T) *lang.Pack {
|
||
t.Helper()
|
||
// Load the shared zh-ru pack PLUS the 蛊真人 book-scoped overlay (pair-14 §1): 古月 is the book's clan,
|
||
// carried in the overlay, not the shared pair langpack. Every miner fixture (and the stand parity test)
|
||
// runs on this book's effective pack, so the surname-anchor cases (古月方源) and {古月:22} stay EXACT.
|
||
p, err := lang.LoadWithOverlay("../../configs/langpacks", "zh", "ru", "testdata/langpack-overlay-guzhenren")
|
||
if err != nil {
|
||
t.Fatalf("load langpack: %v", err)
|
||
}
|
||
return p
|
||
}
|
||
|
||
// miner_test.go: synthetic, in-repo unit fixtures for the WS3 default-B miner (no external book data —
|
||
// the full-book Go↔Python parity is the data-gated TestMinerFullBookParity). Each fixture pins one
|
||
// load-bearing algorithm piece: the c-value length-multiplier (蛊/转 non-degenerate), subsumption α, the
|
||
// pattern channels, is_palladius_token, the alias tier-1 rules (the plan's required §3(д) fixtures), the
|
||
// emission filters, determinism, and the Source:"mined" write path.
|
||
|
||
// mchunks builds miner chunks from raw zh strings (one chunk per chapter, normalized like the runner).
|
||
func mchunks(sources ...string) []Chunk {
|
||
out := make([]Chunk, len(sources))
|
||
for i, s := range sources {
|
||
out[i] = Chunk{Chapter: i + 1, ChunkIdx: 0, NSource: text.NormalizeSourceKey(s)}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// smallContrast is a tiny general-zh reference: common function/name chars carry high freq; the domain
|
||
// formant 蛊 is deliberately ABSENT so its book over-representation is high (the formant signal).
|
||
func smallContrast(t *testing.T) *Contrast {
|
||
t.Helper()
|
||
const data = "的 100000 uj\n是 80000 v\n人 50000 n\n出 9000 v\n现 9000 v\n" +
|
||
"月 4000 n\n光 3800 n\n希 900 n\n望 3000 v\n影 800 n\n在 20000 p\n此 1200 r\n" +
|
||
"古 400 nr\n方 900 nr\n源 300 n\n族 500 n\n长 6000 a\n青 700 n\n茅 20 n\n山 5000 n\n" +
|
||
"四 8000 m\n代 3000 n\n甲 300 n\n等 12000 u\n正 4000 a\n"
|
||
c, err := LoadContrast(strings.NewReader(data))
|
||
if err != nil {
|
||
t.Fatalf("LoadContrast: %v", err)
|
||
}
|
||
return c
|
||
}
|
||
|
||
func TestMinerLengthMultNonDegenerate(t *testing.T) {
|
||
// §B1: g(L)=log2(L+1), NOT log2(L) — a single-char candidate (蛊/转) must keep a NON-ZERO length
|
||
// multiplier, else the catastrophe screen zeros out and 蛊 sinks to rank ~4753.
|
||
if got := lengthMult(1); got != 1.0 {
|
||
t.Fatalf("lengthMult(1) = %v, want 1.0 (log2(2)); log2(1)=0 would degenerate |a|=1 candidates", got)
|
||
}
|
||
if lengthMult(2) <= lengthMult(1) {
|
||
t.Fatalf("length multiplier must be monotone in L")
|
||
}
|
||
}
|
||
|
||
func TestMinerCValueSingleCharNested(t *testing.T) {
|
||
// 蛊 (len 1) nested in 蛊师 (len 2): its c-value is g(1)·(f(蛊) − f(蛊师)) — computed, finite, and its
|
||
// magnitude is non-degenerate because g(1)=1 (not 0). Pins the length-mult fix at the c-value layer.
|
||
cf := map[string]int{"蛊": 10, "蛊师": 4}
|
||
cv := computeCValue(cf)
|
||
want := 1.0 * (10.0 - 4.0/1.0) // g(1)·(f(蛊) − avg over its 1 container)
|
||
if got := cv["蛊"]; got != want {
|
||
t.Fatalf("cval[蛊] = %v, want %v (g(1)=1 non-degenerate)", got, want)
|
||
}
|
||
if cv["蛊师"] != lengthMult(2)*4 { // non-nested → g(2)·f
|
||
t.Fatalf("cval[蛊师] = %v, want %v", cv["蛊师"], lengthMult(2)*4)
|
||
}
|
||
}
|
||
|
||
func TestMinerSubsumption(t *testing.T) {
|
||
// 方源(559) survives 方源的(85) [ratio 0.15 < 0.80]; 花酒行(65) is dropped by 花酒行者(65) [ratio 1.0].
|
||
cf := map[string]int{"方源": 559, "方源的": 85, "花酒行": 65, "花酒行者": 65}
|
||
drop := subsumedCandidates(cf, minerSubsumeAlpha)
|
||
if drop["方源"] {
|
||
t.Fatalf("方源 must NOT be subsumed by 方源的 (85 < 0.8·559)")
|
||
}
|
||
if !drop["花酒行"] {
|
||
t.Fatalf("花酒行 must be subsumed by 花酒行者 (65 >= 0.8·65)")
|
||
}
|
||
}
|
||
|
||
func TestMinerPatternChannels(t *testing.T) {
|
||
chunks := mchunks(
|
||
"古月方源走进青茅山。四代族长甲等弟子。", // surname+name, topo, ordinal_title, rank_grade
|
||
)
|
||
// candFreq is only needed for the formant channel; the structural channels scan the source directly.
|
||
cf := map[string]int{}
|
||
pats, _ := patternCandidates(chunks, cf, smallContrast(t), 3, 15.0, testLangPack(t))
|
||
cases := []struct {
|
||
src, typ string
|
||
}{
|
||
{"古月方源", "name"}, // surname anchor (compound 古月 + 方源)
|
||
{"青茅山", "place"}, // topo suffix 山
|
||
{"四代族长", "title"}, // ordinal_title 四代+族长
|
||
{"甲等", "title"}, // rank_grade 甲+等
|
||
{"族长", "title"}, // title_suffix / title_bare
|
||
}
|
||
for _, c := range cases {
|
||
info, ok := pats[text.NormalizeSourceKey(c.src)]
|
||
if !ok {
|
||
t.Errorf("pattern channel missed %q", c.src)
|
||
continue
|
||
}
|
||
if !containsStr(info.types, c.typ) {
|
||
t.Errorf("%q types = %v, want to include %q", c.src, info.types, c.typ)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestMinerFormantDetection(t *testing.T) {
|
||
// 蛊 binds ≥3 distinct morphemes among freq≥3 candidates AND is over-represented in-book vs the
|
||
// contrast (absent there) → auto-detected as a suffix formant; a common char (的) is NOT.
|
||
chunks := mchunks(strings.Repeat("月光蛊出现。希望蛊出现。月影蛊出现。", 3))
|
||
va := runVA(chunks, smallContrast(t), minerFreqFloor, minerSubsumeAlpha)
|
||
formants := detectFormants(chunks, va.candFreq, smallContrast(t), 3, 15.0)
|
||
if _, ok := formants['蛊']; !ok {
|
||
t.Fatalf("蛊 must be auto-detected as a productive formant, got %v", formantKeys(formants))
|
||
}
|
||
if _, ok := formants['的']; ok {
|
||
t.Fatalf("的 (common char) must NOT be a formant")
|
||
}
|
||
}
|
||
|
||
func TestMinerPalladiusToken(t *testing.T) {
|
||
syl := buildPalladiusCyrSyllables(testLangPack(t))
|
||
// Positives: transliterated Chinese-name shapes segment cleanly.
|
||
for _, tok := range []string{"юань", "фан", "гу", "цзюй"} {
|
||
if !isPalladiusToken(tok, 1, syl) {
|
||
t.Errorf("isPalladiusToken(%q) = false, want true (Palladius name shape)", tok)
|
||
}
|
||
}
|
||
// Negative control (palladius.py): common Russian words are NOT Palladius tokens.
|
||
for _, tok := range []string{"человек", "который", "деревня", "камень"} {
|
||
if isPalladiusToken(tok, 1, syl) {
|
||
t.Errorf("isPalladiusToken(%q) = true, want false (common ru word)", tok)
|
||
}
|
||
}
|
||
// The «найти»=най+ти FALSE-POSITIVE the pymorphy3 is_name_lemma gate exists to block: it IS
|
||
// Palladius-shaped, which is exactly why default B drops the whole ru-side name confirmation channel
|
||
// (no morphology backend to run the gate) rather than trust the token shape alone.
|
||
if !isPalladiusToken("найти", 2, syl) {
|
||
t.Fatalf("найти must be Palladius-shaped (documents why default-B drops the ru name channel)")
|
||
}
|
||
}
|
||
|
||
// aliasSurf builds an aliasSurface for the rule tests.
|
||
func aliasSurf(src, typ, gender, approvedDst string) aliasSurface {
|
||
return aliasSurface{src: text.NormalizeSourceKey(src), typ: typ, gender: gender, approvedDst: approvedDst}
|
||
}
|
||
|
||
func TestMinerAliasRules(t *testing.T) {
|
||
seedSurf := func(ss ...string) map[string]bool {
|
||
m := map[string]bool{}
|
||
for _, s := range ss {
|
||
m[text.NormalizeSourceKey(s)] = true
|
||
}
|
||
return m
|
||
}
|
||
hasIdent := func(edges []aliasEdge, a, b string) bool {
|
||
an, bn := text.NormalizeSourceKey(a), text.NormalizeSourceKey(b)
|
||
for _, e := range edges {
|
||
if (e.a == an && e.b == bn) || (e.a == bn && e.b == an) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
pack := testLangPack(t)
|
||
|
||
// R1 extension: 方源 (seed entity) ⊂ 古月方源 → identity (same person, fuller surface).
|
||
t.Run("R1_extension", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
text.NormalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""),
|
||
text.NormalizeSourceKey("古月方源"): aliasSurf("古月方源", "name", "", ""),
|
||
}
|
||
ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("方源"), pack)
|
||
if !hasIdent(ident, "方源", "古月方源") {
|
||
t.Fatalf("方源 ⊂ 古月方源 must be an identity extension, got %+v", ident)
|
||
}
|
||
})
|
||
|
||
// Composite guard: 古月 (seed clan) + 族长 (seed title) = 古月族长 is a PHRASE, not an alias (blocks
|
||
// the clan↔title chaining). But 古月 + 方源 (name) stays a fullname alias.
|
||
t.Run("composite_guard", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
text.NormalizeSourceKey("古月"): aliasSurf("古月", "name", "", ""),
|
||
text.NormalizeSourceKey("族长"): aliasSurf("族长", "title", "", ""),
|
||
text.NormalizeSourceKey("古月族长"): aliasSurf("古月族长", "title", "", ""),
|
||
text.NormalizeSourceKey("古月方源"): aliasSurf("古月方源", "name", "", ""),
|
||
text.NormalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""),
|
||
}
|
||
seeds := seedSurf("古月", "族长", "方源")
|
||
ident, _, weak := proposeAliasEdges(surf, nil, seeds, pack)
|
||
if hasIdent(ident, "古月", "古月族长") {
|
||
t.Fatalf("古月+族长 (clan+title) must be a phrase, NOT an identity alias")
|
||
}
|
||
phraseFlagged := false
|
||
for _, e := range weak {
|
||
if e.kind == "phrase_not_alias" && e.a == text.NormalizeSourceKey("古月") && e.b == text.NormalizeSourceKey("古月族长") {
|
||
phraseFlagged = true
|
||
}
|
||
}
|
||
if !phraseFlagged {
|
||
t.Fatalf("古月+族长 must be flagged phrase_not_alias, got weak=%+v", weak)
|
||
}
|
||
if !hasIdent(ident, "古月", "古月方源") {
|
||
t.Fatalf("古月+方源 (clan+name) must stay a fullname identity alias, got %+v", ident)
|
||
}
|
||
})
|
||
|
||
// R4-iii/v: 族长 (approved «глава клана») ⊂ 四代族长 (approved «четвёртый глава клана») — DIFFERENT
|
||
// approved dst → different entities EVEN under containment (D38 §3). No identity edge.
|
||
t.Run("R4_different_approved_dst", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
text.NormalizeSourceKey("族长"): aliasSurf("族长", "title", "", "глава клана"),
|
||
text.NormalizeSourceKey("四代族长"): aliasSurf("四代族长", "title", "", "четвёртый глава клана"),
|
||
}
|
||
ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("族长", "四代族长"), pack)
|
||
if hasIdent(ident, "族长", "四代族长") {
|
||
t.Fatalf("族长 vs 四代族长 with different approved dst must NOT be identity, got %+v", ident)
|
||
}
|
||
})
|
||
|
||
// R4-ii: different confirmed gender blocks identity; hidden never blocks.
|
||
t.Run("R4_gender", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
text.NormalizeSourceKey("方源"): aliasSurf("方源", "name", "male", ""),
|
||
text.NormalizeSourceKey("方源儿"): aliasSurf("方源儿", "name", "female", ""),
|
||
}
|
||
ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("方源"), pack)
|
||
if hasIdent(ident, "方源", "方源儿") {
|
||
t.Fatalf("different confirmed gender must block identity, got %+v", ident)
|
||
}
|
||
})
|
||
|
||
// R4-i / R2: same surname, DIFFERENT given names → family, not identity (方源 vs 方正, surname 方).
|
||
t.Run("R2_family_R4i", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
text.NormalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""),
|
||
text.NormalizeSourceKey("方正"): aliasSurf("方正", "name", "", ""),
|
||
}
|
||
ident, family, _ := proposeAliasEdges(surf, nil, seedSurf("方源", "方正"), pack)
|
||
if hasIdent(ident, "方源", "方正") {
|
||
t.Fatalf("same surname + different given names must be family, NOT identity")
|
||
}
|
||
if len(family) == 0 {
|
||
t.Fatalf("方源 ~ 方正 must be a family edge, got none")
|
||
}
|
||
})
|
||
}
|
||
|
||
func TestMinerEmissionFilters(t *testing.T) {
|
||
subsumed := map[string]bool{"花酒行": true}
|
||
seeds := map[string]bool{text.NormalizeSourceKey("方源"): true}
|
||
pack := testLangPack(t)
|
||
cases := []struct {
|
||
name string
|
||
c scoredCand
|
||
want bool
|
||
}{
|
||
{"typed name freq5", scoredCand{Src: "龙公", Types: []string{"name"}, Freq: 5}, true},
|
||
{"below freq floor", scoredCand{Src: "龙公", Types: []string{"name"}, Freq: 4}, false},
|
||
{"untyped", scoredCand{Src: "什么", Types: []string{"term"}, Freq: 20}, false},
|
||
{"subsumed", scoredCand{Src: "花酒行", Types: []string{"place"}, Freq: 9}, false},
|
||
{"single char", scoredCand{Src: "蛊", Types: []string{"term", "name"}, Freq: 99}, false},
|
||
{"fragment", scoredCand{Src: text.NormalizeSourceKey("方源心"), Types: []string{"name"}, Freq: 9}, false},
|
||
}
|
||
for _, tc := range cases {
|
||
if got := emissionEligible(tc.c, subsumed, seeds, pack); got != tc.want {
|
||
t.Errorf("%s: emissionEligible = %v, want %v", tc.name, got, tc.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestMineBankDeterministicAndNonSeed(t *testing.T) {
|
||
chunks := mchunks(strings.Repeat("龙公来到青茅山。龙公很强。青茅山很高。", 4))
|
||
seed := []store.GlossaryEntry{{Src: "青茅山", Dst: "гора Цинмао", Type: "place", Status: "approved", Source: "seed"}}
|
||
cfg := FrozenConfig()
|
||
a := MineBank(chunks, smallContrast(t), seed, nil, cfg, testLangPack(t))
|
||
b := MineBank(chunks, smallContrast(t), seed, nil, cfg, testLangPack(t))
|
||
if !reflect.DeepEqual(a, b) {
|
||
t.Fatalf("MineBank must be deterministic byte-for-byte:\n a=%+v\n b=%+v", a, b)
|
||
}
|
||
// The seed surface 青茅山 must NOT be re-emitted as a new mined term (the curated entry owns it).
|
||
for _, m := range a {
|
||
if m.Src == text.NormalizeSourceKey("青茅山") {
|
||
t.Fatalf("MineBank re-emitted a seed surface as a new term: %+v", m)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestMineBankRejectSuppressesTerm pins R1-FL-B: a term the owner declined (mined_rejects) is dropped
|
||
// from the delta — it never re-enters a signature map or the auto-bank — while the OTHER proposals
|
||
// (which the owner has not decided) still emit. The reject is about the DELTA's content only: whether a
|
||
// run STOPS is the presented memory's question (D39.144, pipeline/mining.go), not this filter's.
|
||
func TestMineBankRejectSuppressesTerm(t *testing.T) {
|
||
chunks := mchunks(strings.Repeat("方源来到青茅山。方源很强。花家很大。花家的人。", 6))
|
||
seed := []store.GlossaryEntry{{Src: "青茅山", Dst: "гора Цинмао", Type: "place", Status: "approved", Source: "seed"}}
|
||
cfg := FrozenConfig()
|
||
fangYuan, huaJia := text.NormalizeSourceKey("方源"), text.NormalizeSourceKey("花家")
|
||
|
||
base := MineBank(chunks, smallContrast(t), seed, nil, cfg, testLangPack(t))
|
||
if !mineHas(base, fangYuan) || !mineHas(base, huaJia) {
|
||
t.Fatalf("fixture must propose both 方源 and 花家 for the reject test to be meaningful, got %+v", base)
|
||
}
|
||
|
||
// Reject 方源 (normalized as loadMinedRejects would): it must vanish, but 花家 (still undecided) fires.
|
||
got := MineBank(chunks, smallContrast(t), seed, map[string]bool{fangYuan: true}, cfg, testLangPack(t))
|
||
if mineHas(got, fangYuan) {
|
||
t.Fatalf("a rejected term was re-emitted → the stop would livelock: %+v", got)
|
||
}
|
||
if !mineHas(got, huaJia) {
|
||
t.Fatalf("rejecting one term must not drop an unrelated undecided term (花家): %+v", got)
|
||
}
|
||
|
||
// Reject BOTH → the delta empties → the stop auto-continues (every proposal is now decided).
|
||
both := MineBank(chunks, smallContrast(t), seed, map[string]bool{fangYuan: true, huaJia: true}, cfg, testLangPack(t))
|
||
if len(both) != 0 {
|
||
t.Fatalf("rejecting every proposal must empty the delta (auto-continue), got %d: %+v", len(both), both)
|
||
}
|
||
|
||
// A nil / empty rejects map must be a no-op (byte-identical to base), so an unconfigured book — and the
|
||
// frozen parity — are unaffected.
|
||
if noop := MineBank(chunks, smallContrast(t), seed, map[string]bool{}, cfg, testLangPack(t)); !reflect.DeepEqual(noop, base) {
|
||
t.Fatalf("an empty rejects map must be a no-op:\n base=%+v\n got=%+v", base, noop)
|
||
}
|
||
}
|
||
|
||
// mineHas reports whether the delta contains a term with the given normalized src.
|
||
func mineHas(delta []Term, src string) bool {
|
||
for _, m := range delta {
|
||
if m.Src == src {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func formantKeys(m map[rune]formantInfo) []string {
|
||
out := make([]string, 0, len(m))
|
||
for r := range m {
|
||
out = append(out, string(r))
|
||
}
|
||
return out
|
||
}
|