114 lines
4.5 KiB
Go
114 lines
4.5 KiB
Go
package miner
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"textmachine/backend/internal/lang"
|
|
"textmachine/backend/internal/text"
|
|
)
|
|
|
|
// emissioncap_test.go: the ANCHOR of a ping, not a fix. The emission cap (emitRankCap) slices the
|
|
// ranked pool BEFORE the eligibility/seed/reject filters (miner_emit.go), so an eligible candidate
|
|
// below the cap is structurally unreachable: no owner decision frees its slot — declining everything
|
|
// inside the window surfaces nothing from below it. Moving the cap or the filter order touches the
|
|
// frozen reference parity and is a separate decision (fix2 ping, PROGRESS «Бэкенд» 27.08); this test
|
|
// asserts the CURRENT behaviour so it reddens the day someone moves either, and the mover finds the
|
|
// ping instead of shipping the change silently.
|
|
|
|
// capProbeCorpus builds >200 rankable candidates from the pack's own surname list: 600 distinct
|
|
// three-char names, six occurrences each (the freq≥5 floor), in six different sentence frames so the
|
|
// boundary n-grams stay under the floor.
|
|
func capProbeCorpus(t *testing.T) string {
|
|
t.Helper()
|
|
f, err := os.Open("../../configs/langpacks/zh/surnames-single.txt")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
var surnames []rune
|
|
sc := bufio.NewScanner(f)
|
|
for sc.Scan() {
|
|
line := strings.TrimSpace(sc.Text())
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
surnames = append(surnames, []rune(line)...)
|
|
}
|
|
given := []rune("源月山河星辰风云雷电火冰霜雪玉石金木水土")
|
|
var names []string
|
|
for i, sn := range surnames {
|
|
g1, g2 := given[i%len(given)], given[(i*7+3)%len(given)]
|
|
if g1 == g2 {
|
|
g2 = given[(i*7+4)%len(given)]
|
|
}
|
|
names = append(names, string(sn)+string(g1)+string(g2))
|
|
if len(names) >= 600 {
|
|
break
|
|
}
|
|
}
|
|
templates := []string{"%s说。", "%s走了。", "这是%s。", "%s不在。", "%s回去。", "%s很强。"}
|
|
var b strings.Builder
|
|
for _, n := range names {
|
|
for i := 0; i < 6; i++ {
|
|
fmt.Fprintf(&b, templates[i%len(templates)], n)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func TestTheEmissionCapPrecedesTheFiltersAndTheTailIsUnreachable(t *testing.T) {
|
|
pack, err := lang.Load("../../configs/langpacks", "zh", "ru")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
contrast, err := LoadContrast(strings.NewReader("的 100000 uj\n是 80000 v\n了 90000 u\n来 20000 v\n人 50000 n\n"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
chunks := []Chunk{{Chapter: 1, ChunkIdx: 0, NSource: text.NormalizeSourceKey(capProbeCorpus(t))}}
|
|
|
|
// The anchor's teeth: the three carriers of this ping promise the test reddens the day the cap
|
|
// moves, and the funnel premises alone cannot see a move that stays between the fixture's numbers
|
|
// (150 and 400 both slid through green before this line existed — review-workflow finding). The
|
|
// VALUE is pinned on purpose: moving it is precisely the decision the ping exists to route.
|
|
if emitRankCap != 200 {
|
|
t.Fatalf("emitRankCap moved (200 → %d): this is the fix2 ping's subject — find «майнерский хвост за потолком» in docs/PROGRESS.md «Бэкенд» and take the frozen-parity decision before shipping", emitRankCap)
|
|
}
|
|
mr := mineDetect(chunks, contrast, FrozenConfig(), pack)
|
|
eligibleEverywhere := 0
|
|
for _, c := range mr.ranked {
|
|
if emissionEligible(c, mr.subsumed, map[string]bool{}, pack) {
|
|
eligibleEverywhere++
|
|
}
|
|
}
|
|
terms, stats := MineBankStats(chunks, contrast, nil, nil, FrozenConfig(), pack)
|
|
if stats.Ranked <= emitRankCap || stats.AfterCap != emitRankCap {
|
|
t.Fatalf("premise: the alphabet must overflow the cap: ranked=%d after_cap=%d", stats.Ranked, stats.AfterCap)
|
|
}
|
|
if eligibleEverywhere <= stats.Eligible || len(terms) == 0 {
|
|
t.Fatalf("premise: an eligible tail must exist below the cap: everywhere=%d in_window=%d emitted=%d",
|
|
eligibleEverywhere, stats.Eligible, len(terms))
|
|
}
|
|
|
|
// The owner declines EVERYTHING the window offered — every emitted cluster, all members.
|
|
rejects := map[string]bool{}
|
|
for _, tm := range terms {
|
|
rejects[text.NormalizeSourceKey(tm.Src)] = true
|
|
for _, a := range tm.Aliases {
|
|
rejects[text.NormalizeSourceKey(a)] = true
|
|
}
|
|
}
|
|
terms2, stats2 := MineBankStats(chunks, contrast, nil, rejects, FrozenConfig(), pack)
|
|
for _, tm := range terms2 {
|
|
if !rejects[text.NormalizeSourceKey(tm.Src)] {
|
|
t.Fatalf("a below-cap candidate surfaced after the in-window set was declined (%q) — the cap no longer precedes the filters; find the fix2 ping before shipping this", tm.Src)
|
|
}
|
|
}
|
|
if len(terms2) != 0 {
|
|
t.Fatalf("declining every emitted cluster must empty the emission, got %d (rejected=%d)", len(terms2), stats2.Rejected)
|
|
}
|
|
}
|