package terminology import ( "os" "path/filepath" "sort" "strings" "testing" ) // loadBankFullSurfaces reads the distilled BANK-FULL source surfaces (fix-pack §ж fixture) as candidates. func loadBankFullSurfaces(t *testing.T) []Candidate { t.Helper() b, err := os.ReadFile(filepath.Join("testdata", "bankfull-surfaces.txt")) if err != nil { t.Fatal(err) } var cands []Candidate for _, ln := range strings.Split(string(b), "\n") { s := strings.TrimSpace(ln) if s == "" || strings.HasPrefix(s, "#") { continue } cands = append(cands, cand(s, 1)) } return cands } // TestDetectSeriesOnBankFullFixtureIsStable pins the head-aware rule's LIVE property on the real corpus it was // calibrated on (fix-pack §ж): exactly the measured grade/rank series form, none of them a family blob, and // 方源/元石/元海 (the protagonist and two same-prefix-different-head entities) stay OUT of every series — the // naive "differ in one position" rule produced an 11-surface blob here. func TestDetectSeriesOnBankFullFixtureIsStable(t *testing.T) { got := DetectSeries(loadBankFullSurfaces(t), zhSeries) // The FOUR grade/rank series the corpus forms, each sharing one real head — 转 (rank), 等 (grade), 等资质 // (graded aptitude), 阶 (tier). Legitimately large is not a blob: 一转…九转 is nine ranks sharing 转, the // canonical §1 case. A blob is a series that MIXES families under a shared rune, which the head-aware rule // forbids by construction — pinned here by requiring each series be EXACTLY its expected group. expected := [][]string{ {"一转", "二转", "三转", "四转", "五转", "六转", "九转"}, {"甲等", "乙等", "丙等", "丁等"}, {"甲等资质", "乙等资质", "丙等资质"}, {"初阶", "中阶", "高阶"}, } ids := map[int]bool{} size := map[int]int{} for _, id := range got { ids[id] = true size[id]++ } if len(ids) != 4 { grouped := map[int][]string{} for k, id := range got { grouped[id] = append(grouped[id], k) } for id, ms := range grouped { sort.Strings(ms) t.Logf("series %d: %v", id, ms) } t.Fatalf("expected exactly 4 series on the distilled bank, got %d", len(ids)) } for _, grp := range expected { id := got[grp[0]] if id == 0 { t.Fatalf("expected group %v did not form a series", grp) } for _, k := range grp { if got[k] != id { t.Fatalf("%s must join series %d with %s, got %d", k, id, grp[0], got[k]) } } if size[id] != len(grp) { t.Fatalf("series %d has %d members, expected exactly %d (%v) — a member leaked in (a blob)", id, size[id], len(grp), grp) } } // The three entities that must NEVER be series members: the protagonist 方源 and the two same-prefix, // different-HEAD entities 元石/元海 (石/海 differ IN the head → different entities, the §2 type step's job). for _, k := range []string{"方源", "元石", "元海"} { if got[k] != 0 { t.Fatalf("%s must stay OUT of every series, got id %d", k, got[k]) } } } var zhSeries = SeriesParams{Enabled: true, HeadFinal: true} func cand(key string, freq int) Candidate { return Candidate{Key: key, Src: key, Type: "term", Freq: freq} } // TestDetectSeriesHeadAware pins the head-aware rule against the exact patterns the naive "differ in one // position" rule broke on (measured on BANK-FULL): a grade set clusters, but a set differing IN the head // does not (that is the type step's job), a shared head-rune across two families never forms a blob, and a // one-rune surface is never a member. func TestDetectSeriesHeadAware(t *testing.T) { cands := []Candidate{ cand("甲等", 3), cand("乙等", 3), cand("丙等", 3), cand("丁等", 3), // grades of 等 → ONE series cand("元石", 40), cand("元海", 40), cand("元火", 5), // differ IN the head 石/海/火 → NOT a series (§2's job) cand("元气", 6), cand("酒气", 6), // share head 气 but only two → below min, no blob cand("转", 9), // one rune → never a member } got := DetectSeries(cands, zhSeries) grade := []string{"甲等", "乙等", "丙等", "丁等"} id := got[grade[0]] if id == 0 { t.Fatalf("the grade set must form a series: %v", got) } for _, k := range grade { if got[k] != id { t.Fatalf("%s must join the grade series (id %d), got %d", k, id, got[k]) } } // The §1↔§2 reconciliation: 元石/元海 differ in the HEAD, so they are different entities, not a series. for _, k := range []string{"元石", "元海", "元火", "元气", "酒气", "转"} { if got[k] != 0 { t.Fatalf("%s must NOT be a series member (id %d) — the naive rule's failure mode", k, got[k]) } } } // TestDetectSeriesLongerHead covers a multi-rune shared head (甲等资质/乙等资质/丙等资质): the differing rune // is the leading modifier, the whole 等资质 tail is the head. func TestDetectSeriesLongerHead(t *testing.T) { cands := []Candidate{cand("甲等资质", 2), cand("乙等资质", 2), cand("丙等资质", 2)} got := DetectSeries(cands, zhSeries) if got["甲等资质"] == 0 || got["甲等资质"] != got["乙等资质"] || got["乙等资质"] != got["丙等资质"] { t.Fatalf("a set sharing the trailing head 等资质 must form one series: %v", got) } } // TestDetectSeriesRespectsPairData: the channel is off for a pair whose source is not dense-script — the same // single-position difference that clusters for zh must NOT cluster when Enabled is false (care/core is a // spelling coincidence), and the head side flips with HeadFinal. func TestDetectSeriesRespectsPairData(t *testing.T) { alpha := []Candidate{cand("care", 3), cand("core", 3), cand("cure", 3)} if got := DetectSeries(alpha, SeriesParams{Enabled: false, HeadFinal: true}); len(got) != 0 { t.Fatalf("series channel must be inert for a non-series pair, got %v", got) } // Head-INITIAL direction: the head is the leading rune, the modifier is non-initial. 등X where the tail // varies and 등 is shared as the head. headInit := []Candidate{cand("등가", 1), cand("등나", 1), cand("등다", 1)} got := DetectSeries(headInit, SeriesParams{Enabled: true, HeadFinal: false}) if got["등가"] == 0 || got["등가"] != got["등나"] || got["등나"] != got["등다"] { t.Fatalf("with HeadFinal=false the leading rune is the head and the trailing modifier varies: %v", got) } // The same set under HeadFinal=true shares no head (they differ at the last position) → no series. if got := DetectSeries(headInit, zhSeries); len(got) != 0 { t.Fatalf("under head-final the trailing-varying set is not a series, got %v", got) } } // TestBatchKeepsSeriesWhole: a series scattered across key order lands in ONE batch even under a budget so // tight every singleton is its own batch — co-batching is the whole mechanism. func TestBatchKeepsSeriesWhole(t *testing.T) { // Key-sorted order interleaves the series with a non-member; a tiny budget would otherwise split it. cands := []Candidate{cand("丁等", 3), cand("丙等", 3), cand("乙等", 3), cand("甲等", 3), cand("中间", 50)} seriesID := DetectSeries(cands, zhSeries) batches := Batch(cands, 10, seriesID) // 10 runes: every unit overflows, so packing cannot help by luck var seriesBatch []Candidate for _, b := range batches { for _, c := range b { if seriesID[c.Key] != 0 { seriesBatch = b } } } if len(seriesBatch) != 4 { t.Fatalf("all four grade members must share one batch, got %d: %v", len(seriesBatch), batches) } // Every candidate still appears exactly once across all batches. seen := map[string]int{} for _, b := range batches { for _, c := range b { seen[c.Key]++ } } if len(seen) != len(cands) { t.Fatalf("batching dropped or duplicated a candidate: %v", seen) } } // TestBatchRunesFlagsOversize pins the fix-pack §в observability seam: BatchRunes measures a batch the way // Batch packs it, so the terminologist can WARN when §1 keeps a unit whole past the budget instead of // silently over-running the output cap. A lone candidate too big for the budget stays ONE over-cap batch. func TestBatchRunesFlagsOversize(t *testing.T) { big := Candidate{Key: "元", Src: "元", Type: "term", KWIC: []string{strings.Repeat("к", 500)}} if BatchRunes([]Candidate{big}) <= 100 { t.Fatal("a candidate carrying a 500-rune context must render well over a 100-rune budget") } batches := Batch([]Candidate{big}, 100, nil) if len(batches) != 1 || BatchRunes(batches[0]) <= 100 { t.Fatalf("an oversize lone candidate must stay one over-cap batch the caller can WARN on: %v", batches) } } // TestBatchValueOrder: batches are processed most-frequent first, so a budget ceiling drops the rarest terms // (feed_cap) rather than the lexicographic tail. Content of each batch is untouched by the ordering. func TestBatchValueOrder(t *testing.T) { // Three singletons, each its own batch under a tiny budget; frequency is the only thing that differs. cands := []Candidate{cand("阿", 1), cand("布", 99), cand("此", 50)} batches := Batch(cands, 5, nil) if len(batches) != 3 { t.Fatalf("expected three singleton batches, got %d", len(batches)) } if batches[0][0].Key != "布" || batches[len(batches)-1][0].Key != "阿" { t.Fatalf("batches must run most-frequent first, least-frequent last: %v", batches) } } // TestBatchNilSeriesMatchesLegacyOrder guards the alphabetic/off path: with no series and uniform frequency, // Batch preserves the incoming key order exactly (the pre-§1 behaviour), so a book that forms no series takes // a byte-identical path. func TestBatchNilSeriesMatchesLegacyOrder(t *testing.T) { var cands []Candidate for _, k := range []string{"a", "b", "c", "d", "e"} { cands = append(cands, Candidate{Key: k, Src: k, KWIC: []string{strings.Repeat("к", 200)}}) } batches := Batch(cands, 400, nil) var order []string for _, b := range batches { for _, c := range b { order = append(order, c.Key) } } if strings.Join(order, "") != "abcde" { t.Fatalf("uniform-frequency, series-free batching must preserve key order, got %v", order) } }