353 lines
15 KiB
Go
353 lines
15 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"reflect"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/lang"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// testLangPack loads the real in-repo zh→ru language pack (configs/langpacks/) for the miner fixtures.
|
||
// The pack files are byte-derived from the former constants (langpack move, D39.15), so the miner is
|
||
// exercised over the exact same data.
|
||
func testLangPack(t *testing.T) *lang.Pack {
|
||
t.Helper()
|
||
p, err := lang.Load("../../configs/langpacks", "zh", "ru")
|
||
if err != nil {
|
||
t.Fatalf("load langpack: %v", err)
|
||
}
|
||
return p
|
||
}
|
||
|
||
// miner_test.go: synthetic, in-repo unit fixtures for the WS3 default-B miner (no external book data —
|
||
// the full-book Go↔Python parity is the data-gated TestMinerFullBookParity). Each fixture pins one
|
||
// load-bearing algorithm piece: the c-value length-multiplier (蛊/转 non-degenerate), subsumption α, the
|
||
// pattern channels, is_palladius_token, the alias tier-1 rules (the plan's required §3(д) fixtures), the
|
||
// emission filters, determinism, and the Source:"mined" write path.
|
||
|
||
// mchunks builds miner chunks from raw zh strings (one chunk per chapter, normalized like the runner).
|
||
func mchunks(sources ...string) []MinerChunk {
|
||
out := make([]MinerChunk, len(sources))
|
||
for i, s := range sources {
|
||
out[i] = MinerChunk{Chapter: i + 1, ChunkIdx: 0, NSource: normalizeSourceKey(s)}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// smallContrast is a tiny general-zh reference: common function/name chars carry high freq; the domain
|
||
// formant 蛊 is deliberately ABSENT so its book over-representation is high (the formant signal).
|
||
func smallContrast(t *testing.T) *Contrast {
|
||
t.Helper()
|
||
const data = "的 100000 uj\n是 80000 v\n人 50000 n\n出 9000 v\n现 9000 v\n" +
|
||
"月 4000 n\n光 3800 n\n希 900 n\n望 3000 v\n影 800 n\n在 20000 p\n此 1200 r\n" +
|
||
"古 400 nr\n方 900 nr\n源 300 n\n族 500 n\n长 6000 a\n青 700 n\n茅 20 n\n山 5000 n\n" +
|
||
"四 8000 m\n代 3000 n\n甲 300 n\n等 12000 u\n正 4000 a\n"
|
||
c, err := LoadContrast(strings.NewReader(data))
|
||
if err != nil {
|
||
t.Fatalf("LoadContrast: %v", err)
|
||
}
|
||
return c
|
||
}
|
||
|
||
func TestMinerLengthMultNonDegenerate(t *testing.T) {
|
||
// §B1: g(L)=log2(L+1), NOT log2(L) — a single-char candidate (蛊/转) must keep a NON-ZERO length
|
||
// multiplier, else the catastrophe screen zeros out and 蛊 sinks to rank ~4753.
|
||
if got := lengthMult(1); got != 1.0 {
|
||
t.Fatalf("lengthMult(1) = %v, want 1.0 (log2(2)); log2(1)=0 would degenerate |a|=1 candidates", got)
|
||
}
|
||
if lengthMult(2) <= lengthMult(1) {
|
||
t.Fatalf("length multiplier must be monotone in L")
|
||
}
|
||
}
|
||
|
||
func TestMinerCValueSingleCharNested(t *testing.T) {
|
||
// 蛊 (len 1) nested in 蛊师 (len 2): its c-value is g(1)·(f(蛊) − f(蛊师)) — computed, finite, and its
|
||
// magnitude is non-degenerate because g(1)=1 (not 0). Pins the length-mult fix at the c-value layer.
|
||
cf := map[string]int{"蛊": 10, "蛊师": 4}
|
||
cv := computeCValue(cf)
|
||
want := 1.0 * (10.0 - 4.0/1.0) // g(1)·(f(蛊) − avg over its 1 container)
|
||
if got := cv["蛊"]; got != want {
|
||
t.Fatalf("cval[蛊] = %v, want %v (g(1)=1 non-degenerate)", got, want)
|
||
}
|
||
if cv["蛊师"] != lengthMult(2)*4 { // non-nested → g(2)·f
|
||
t.Fatalf("cval[蛊师] = %v, want %v", cv["蛊师"], lengthMult(2)*4)
|
||
}
|
||
}
|
||
|
||
func TestMinerSubsumption(t *testing.T) {
|
||
// 方源(559) survives 方源的(85) [ratio 0.15 < 0.80]; 花酒行(65) is dropped by 花酒行者(65) [ratio 1.0].
|
||
cf := map[string]int{"方源": 559, "方源的": 85, "花酒行": 65, "花酒行者": 65}
|
||
drop := subsumedCandidates(cf, minerSubsumeAlpha)
|
||
if drop["方源"] {
|
||
t.Fatalf("方源 must NOT be subsumed by 方源的 (85 < 0.8·559)")
|
||
}
|
||
if !drop["花酒行"] {
|
||
t.Fatalf("花酒行 must be subsumed by 花酒行者 (65 >= 0.8·65)")
|
||
}
|
||
}
|
||
|
||
func TestMinerPatternChannels(t *testing.T) {
|
||
chunks := mchunks(
|
||
"古月方源走进青茅山。四代族长甲等弟子。", // surname+name, topo, ordinal_title, rank_grade
|
||
)
|
||
// candFreq is only needed for the formant channel; the structural channels scan the source directly.
|
||
cf := map[string]int{}
|
||
pats, _ := patternCandidates(chunks, cf, smallContrast(t), 3, 15.0, testLangPack(t))
|
||
cases := []struct {
|
||
src, typ string
|
||
}{
|
||
{"古月方源", "name"}, // surname anchor (compound 古月 + 方源)
|
||
{"青茅山", "place"}, // topo suffix 山
|
||
{"四代族长", "title"}, // ordinal_title 四代+族长
|
||
{"甲等", "title"}, // rank_grade 甲+等
|
||
{"族长", "title"}, // title_suffix / title_bare
|
||
}
|
||
for _, c := range cases {
|
||
info, ok := pats[normalizeSourceKey(c.src)]
|
||
if !ok {
|
||
t.Errorf("pattern channel missed %q", c.src)
|
||
continue
|
||
}
|
||
if !containsStr(info.types, c.typ) {
|
||
t.Errorf("%q types = %v, want to include %q", c.src, info.types, c.typ)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestMinerFormantDetection(t *testing.T) {
|
||
// 蛊 binds ≥3 distinct morphemes among freq≥3 candidates AND is over-represented in-book vs the
|
||
// contrast (absent there) → auto-detected as a suffix formant; a common char (的) is NOT.
|
||
chunks := mchunks(strings.Repeat("月光蛊出现。希望蛊出现。月影蛊出现。", 3))
|
||
va := runVA(chunks, smallContrast(t), minerFreqFloor, minerSubsumeAlpha)
|
||
formants := detectFormants(chunks, va.candFreq, smallContrast(t), 3, 15.0)
|
||
if _, ok := formants['蛊']; !ok {
|
||
t.Fatalf("蛊 must be auto-detected as a productive formant, got %v", formantKeys(formants))
|
||
}
|
||
if _, ok := formants['的']; ok {
|
||
t.Fatalf("的 (common char) must NOT be a formant")
|
||
}
|
||
}
|
||
|
||
func TestMinerPalladiusToken(t *testing.T) {
|
||
syl := buildPalladiusCyrSyllables(testLangPack(t))
|
||
// Positives: transliterated Chinese-name shapes segment cleanly.
|
||
for _, tok := range []string{"юань", "фан", "гу", "цзюй"} {
|
||
if !isPalladiusToken(tok, 1, syl) {
|
||
t.Errorf("isPalladiusToken(%q) = false, want true (Palladius name shape)", tok)
|
||
}
|
||
}
|
||
// Negative control (palladius.py): common Russian words are NOT Palladius tokens.
|
||
for _, tok := range []string{"человек", "который", "деревня", "камень"} {
|
||
if isPalladiusToken(tok, 1, syl) {
|
||
t.Errorf("isPalladiusToken(%q) = true, want false (common ru word)", tok)
|
||
}
|
||
}
|
||
// The «найти»=най+ти FALSE-POSITIVE the pymorphy3 is_name_lemma gate exists to block: it IS
|
||
// Palladius-shaped, which is exactly why default B drops the whole ru-side name confirmation channel
|
||
// (no morphology backend to run the gate) rather than trust the token shape alone.
|
||
if !isPalladiusToken("найти", 2, syl) {
|
||
t.Fatalf("найти must be Palladius-shaped (documents why default-B drops the ru name channel)")
|
||
}
|
||
}
|
||
|
||
// aliasSurf builds an aliasSurface for the rule tests.
|
||
func aliasSurf(src, typ, gender, approvedDst string) aliasSurface {
|
||
return aliasSurface{src: normalizeSourceKey(src), typ: typ, gender: gender, approvedDst: approvedDst}
|
||
}
|
||
|
||
func TestMinerAliasRules(t *testing.T) {
|
||
seedSurf := func(ss ...string) map[string]bool {
|
||
m := map[string]bool{}
|
||
for _, s := range ss {
|
||
m[normalizeSourceKey(s)] = true
|
||
}
|
||
return m
|
||
}
|
||
hasIdent := func(edges []aliasEdge, a, b string) bool {
|
||
an, bn := normalizeSourceKey(a), normalizeSourceKey(b)
|
||
for _, e := range edges {
|
||
if (e.a == an && e.b == bn) || (e.a == bn && e.b == an) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
pack := testLangPack(t)
|
||
|
||
// R1 extension: 方源 (seed entity) ⊂ 古月方源 → identity (same person, fuller surface).
|
||
t.Run("R1_extension", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
normalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""),
|
||
normalizeSourceKey("古月方源"): aliasSurf("古月方源", "name", "", ""),
|
||
}
|
||
ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("方源"), pack)
|
||
if !hasIdent(ident, "方源", "古月方源") {
|
||
t.Fatalf("方源 ⊂ 古月方源 must be an identity extension, got %+v", ident)
|
||
}
|
||
})
|
||
|
||
// Composite guard: 古月 (seed clan) + 族长 (seed title) = 古月族长 is a PHRASE, not an alias (blocks
|
||
// the clan↔title chaining). But 古月 + 方源 (name) stays a fullname alias.
|
||
t.Run("composite_guard", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
normalizeSourceKey("古月"): aliasSurf("古月", "name", "", ""),
|
||
normalizeSourceKey("族长"): aliasSurf("族长", "title", "", ""),
|
||
normalizeSourceKey("古月族长"): aliasSurf("古月族长", "title", "", ""),
|
||
normalizeSourceKey("古月方源"): aliasSurf("古月方源", "name", "", ""),
|
||
normalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""),
|
||
}
|
||
seeds := seedSurf("古月", "族长", "方源")
|
||
ident, _, weak := proposeAliasEdges(surf, nil, seeds, pack)
|
||
if hasIdent(ident, "古月", "古月族长") {
|
||
t.Fatalf("古月+族长 (clan+title) must be a phrase, NOT an identity alias")
|
||
}
|
||
phraseFlagged := false
|
||
for _, e := range weak {
|
||
if e.kind == "phrase_not_alias" && e.a == normalizeSourceKey("古月") && e.b == normalizeSourceKey("古月族长") {
|
||
phraseFlagged = true
|
||
}
|
||
}
|
||
if !phraseFlagged {
|
||
t.Fatalf("古月+族长 must be flagged phrase_not_alias, got weak=%+v", weak)
|
||
}
|
||
if !hasIdent(ident, "古月", "古月方源") {
|
||
t.Fatalf("古月+方源 (clan+name) must stay a fullname identity alias, got %+v", ident)
|
||
}
|
||
})
|
||
|
||
// R4-iii/v: 族长 (approved «глава клана») ⊂ 四代族长 (approved «четвёртый глава клана») — DIFFERENT
|
||
// approved dst → different entities EVEN under containment (D38 §3). No identity edge.
|
||
t.Run("R4_different_approved_dst", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
normalizeSourceKey("族长"): aliasSurf("族长", "title", "", "глава клана"),
|
||
normalizeSourceKey("四代族长"): aliasSurf("四代族长", "title", "", "четвёртый глава клана"),
|
||
}
|
||
ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("族长", "四代族长"), pack)
|
||
if hasIdent(ident, "族长", "四代族长") {
|
||
t.Fatalf("族长 vs 四代族长 with different approved dst must NOT be identity, got %+v", ident)
|
||
}
|
||
})
|
||
|
||
// R4-ii: different confirmed gender blocks identity; hidden never blocks.
|
||
t.Run("R4_gender", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
normalizeSourceKey("方源"): aliasSurf("方源", "name", "male", ""),
|
||
normalizeSourceKey("方源儿"): aliasSurf("方源儿", "name", "female", ""),
|
||
}
|
||
ident, _, _ := proposeAliasEdges(surf, nil, seedSurf("方源"), pack)
|
||
if hasIdent(ident, "方源", "方源儿") {
|
||
t.Fatalf("different confirmed gender must block identity, got %+v", ident)
|
||
}
|
||
})
|
||
|
||
// R4-i / R2: same surname, DIFFERENT given names → family, not identity (方源 vs 方正, surname 方).
|
||
t.Run("R2_family_R4i", func(t *testing.T) {
|
||
surf := map[string]aliasSurface{
|
||
normalizeSourceKey("方源"): aliasSurf("方源", "name", "", ""),
|
||
normalizeSourceKey("方正"): aliasSurf("方正", "name", "", ""),
|
||
}
|
||
ident, family, _ := proposeAliasEdges(surf, nil, seedSurf("方源", "方正"), pack)
|
||
if hasIdent(ident, "方源", "方正") {
|
||
t.Fatalf("same surname + different given names must be family, NOT identity")
|
||
}
|
||
if len(family) == 0 {
|
||
t.Fatalf("方源 ~ 方正 must be a family edge, got none")
|
||
}
|
||
})
|
||
}
|
||
|
||
func TestMinerEmissionFilters(t *testing.T) {
|
||
subsumed := map[string]bool{"花酒行": true}
|
||
seeds := map[string]bool{normalizeSourceKey("方源"): true}
|
||
pack := testLangPack(t)
|
||
cases := []struct {
|
||
name string
|
||
c ScoredCand
|
||
want bool
|
||
}{
|
||
{"typed name freq5", ScoredCand{Src: "龙公", Types: []string{"name"}, Freq: 5}, true},
|
||
{"below freq floor", ScoredCand{Src: "龙公", Types: []string{"name"}, Freq: 4}, false},
|
||
{"untyped", ScoredCand{Src: "什么", Types: []string{"term"}, Freq: 20}, false},
|
||
{"subsumed", ScoredCand{Src: "花酒行", Types: []string{"place"}, Freq: 9}, false},
|
||
{"single char", ScoredCand{Src: "蛊", Types: []string{"term", "name"}, Freq: 99}, false},
|
||
{"fragment", ScoredCand{Src: normalizeSourceKey("方源心"), Types: []string{"name"}, Freq: 9}, false},
|
||
}
|
||
for _, tc := range cases {
|
||
if got := emissionEligible(tc.c, subsumed, seeds, pack); got != tc.want {
|
||
t.Errorf("%s: emissionEligible = %v, want %v", tc.name, got, tc.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestMineBankDeterministicAndNonSeed(t *testing.T) {
|
||
chunks := mchunks(strings.Repeat("龙公来到青茅山。龙公很强。青茅山很高。", 4))
|
||
seed := []store.GlossaryEntry{{Src: "青茅山", Dst: "гора Цинмао", Type: "place", Status: "approved", Source: "seed"}}
|
||
cfg := frozenMinerConfig()
|
||
a := MineBank(chunks, smallContrast(t), seed, cfg, testLangPack(t))
|
||
b := MineBank(chunks, smallContrast(t), seed, cfg, testLangPack(t))
|
||
if !reflect.DeepEqual(a, b) {
|
||
t.Fatalf("MineBank must be deterministic byte-for-byte:\n a=%+v\n b=%+v", a, b)
|
||
}
|
||
// The seed surface 青茅山 must NOT be re-emitted as a new mined term (the curated entry owns it).
|
||
for _, m := range a {
|
||
if m.Src == normalizeSourceKey("青茅山") {
|
||
t.Fatalf("MineBank re-emitted a seed surface as a new term: %+v", m)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestMinedToCandidatesStampsMinedSource(t *testing.T) {
|
||
mined := []MinedTerm{
|
||
{Src: "龙公", Type: "name", SinceCh: 3, Freq: 7, Aliases: []string{"龙公子"}},
|
||
{Src: "青茅山", Type: "place", SinceCh: 1, Freq: 9}, // covered by manual seed → skipped
|
||
}
|
||
manual := map[string]bool{"青茅山": true}
|
||
got := minedToCandidates(mined, manual)
|
||
if len(got) != 1 {
|
||
t.Fatalf("want 1 mined candidate (青茅山 skipped as manual), got %d", len(got))
|
||
}
|
||
e := got[0]
|
||
if e.Source != "mined" {
|
||
t.Fatalf("mined candidate Source = %q, want \"mined\" (NOT seed — else it moves base-bank-version)", e.Source)
|
||
}
|
||
if e.Status != "auto" || e.Dst != "" {
|
||
t.Fatalf("mined candidate must be status=auto with no dst, got status=%q dst=%q", e.Status, e.Dst)
|
||
}
|
||
if e.SinceCh != 3 || e.Confidence != 7 {
|
||
t.Fatalf("mined candidate metadata wrong: %+v", e)
|
||
}
|
||
if len(e.Aliases) != 1 || e.Aliases[0].Alias != "龙公子" || e.Aliases[0].AliasType != "mined" {
|
||
t.Fatalf("mined alias wrong: %+v", e.Aliases)
|
||
}
|
||
}
|
||
|
||
func TestMinedDeltaStaysBaseBankStable(t *testing.T) {
|
||
// The mined delta (Source:"mined") must NOT move BaseVersion — the load-bearing «переоплата ОДНА»
|
||
// invariant (§1в): a W1.5 mined-row addition moves only the enriched version. (Sibling of the
|
||
// Block-A TestBaseEnrichedMemoryVersionSplit, here through the real mined-write path.)
|
||
seed := []store.GlossaryEntry{{Src: "方源", Dst: "Фан Юань", Status: "approved", Source: "seed"}}
|
||
base0 := materializeMemory(seed, false)
|
||
mined := minedToCandidates([]MinedTerm{{Src: "龙公", Type: "name", SinceCh: 1, Freq: 5}}, map[string]bool{"方源": true})
|
||
// A mined candidate is status=auto (not folded even into enriched when gate off) — promote it to
|
||
// approved to exercise the split, as the owner sign would at W1.5.
|
||
mined[0].Status = "approved"
|
||
mined[0].Dst = "Лун Гун"
|
||
enriched := append(append([]store.GlossaryEntry{}, seed...), mined...)
|
||
bank1 := materializeMemory(enriched, false)
|
||
if base0.BaseVersion() != bank1.BaseVersion() {
|
||
t.Fatalf("base-bank-version moved on a Source:mined addition — snapshot_W1 would re-bill W1")
|
||
}
|
||
if base0.Version() == bank1.Version() {
|
||
t.Fatalf("enriched version must move on a mined approved addition")
|
||
}
|
||
}
|
||
|
||
func formantKeys(m map[rune]formantInfo) []string {
|
||
out := make([]string, 0, len(m))
|
||
for r := range m {
|
||
out = append(out, string(r))
|
||
}
|
||
return out
|
||
}
|