228 lines
9.8 KiB
Go
228 lines
9.8 KiB
Go
package terminology
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// loadBankFullSurfaces reads the distilled BANK-FULL source surfaces (fix-pack §ж fixture) as candidates.
|
|
func loadBankFullSurfaces(t *testing.T) []Candidate {
|
|
t.Helper()
|
|
b, err := os.ReadFile(filepath.Join("testdata", "bankfull-surfaces.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cands []Candidate
|
|
for _, ln := range strings.Split(string(b), "\n") {
|
|
s := strings.TrimSpace(ln)
|
|
if s == "" || strings.HasPrefix(s, "#") {
|
|
continue
|
|
}
|
|
cands = append(cands, cand(s, 1))
|
|
}
|
|
return cands
|
|
}
|
|
|
|
// TestDetectSeriesOnBankFullFixtureIsStable pins the head-aware rule's LIVE property on the real corpus it was
|
|
// calibrated on (fix-pack §ж): exactly the measured grade/rank series form, none of them a family blob, and
|
|
// 方源/元石/元海 (the protagonist and two same-prefix-different-head entities) stay OUT of every series — the
|
|
// naive "differ in one position" rule produced an 11-surface blob here.
|
|
func TestDetectSeriesOnBankFullFixtureIsStable(t *testing.T) {
|
|
got := DetectSeries(loadBankFullSurfaces(t), zhSeries)
|
|
|
|
// The FOUR grade/rank series the corpus forms, each sharing one real head — 转 (rank), 等 (grade), 等资质
|
|
// (graded aptitude), 阶 (tier). Legitimately large is not a blob: 一转…九转 is nine ranks sharing 转, the
|
|
// canonical §1 case. A blob is a series that MIXES families under a shared rune, which the head-aware rule
|
|
// forbids by construction — pinned here by requiring each series be EXACTLY its expected group.
|
|
expected := [][]string{
|
|
{"一转", "二转", "三转", "四转", "五转", "六转", "九转"},
|
|
{"甲等", "乙等", "丙等", "丁等"},
|
|
{"甲等资质", "乙等资质", "丙等资质"},
|
|
{"初阶", "中阶", "高阶"},
|
|
}
|
|
ids := map[int]bool{}
|
|
size := map[int]int{}
|
|
for _, id := range got {
|
|
ids[id] = true
|
|
size[id]++
|
|
}
|
|
if len(ids) != 4 {
|
|
grouped := map[int][]string{}
|
|
for k, id := range got {
|
|
grouped[id] = append(grouped[id], k)
|
|
}
|
|
for id, ms := range grouped {
|
|
sort.Strings(ms)
|
|
t.Logf("series %d: %v", id, ms)
|
|
}
|
|
t.Fatalf("expected exactly 4 series on the distilled bank, got %d", len(ids))
|
|
}
|
|
for _, grp := range expected {
|
|
id := got[grp[0]]
|
|
if id == 0 {
|
|
t.Fatalf("expected group %v did not form a series", grp)
|
|
}
|
|
for _, k := range grp {
|
|
if got[k] != id {
|
|
t.Fatalf("%s must join series %d with %s, got %d", k, id, grp[0], got[k])
|
|
}
|
|
}
|
|
if size[id] != len(grp) {
|
|
t.Fatalf("series %d has %d members, expected exactly %d (%v) — a member leaked in (a blob)", id, size[id], len(grp), grp)
|
|
}
|
|
}
|
|
// The three entities that must NEVER be series members: the protagonist 方源 and the two same-prefix,
|
|
// different-HEAD entities 元石/元海 (石/海 differ IN the head → different entities, the §2 type step's job).
|
|
for _, k := range []string{"方源", "元石", "元海"} {
|
|
if got[k] != 0 {
|
|
t.Fatalf("%s must stay OUT of every series, got id %d", k, got[k])
|
|
}
|
|
}
|
|
}
|
|
|
|
var zhSeries = SeriesParams{Enabled: true, HeadFinal: true}
|
|
|
|
func cand(key string, freq int) Candidate { return Candidate{Key: key, Src: key, Type: "term", Freq: freq} }
|
|
|
|
// TestDetectSeriesHeadAware pins the head-aware rule against the exact patterns the naive "differ in one
|
|
// position" rule broke on (measured on BANK-FULL): a grade set clusters, but a set differing IN the head
|
|
// does not (that is the type step's job), a shared head-rune across two families never forms a blob, and a
|
|
// one-rune surface is never a member.
|
|
func TestDetectSeriesHeadAware(t *testing.T) {
|
|
cands := []Candidate{
|
|
cand("甲等", 3), cand("乙等", 3), cand("丙等", 3), cand("丁等", 3), // grades of 等 → ONE series
|
|
cand("元石", 40), cand("元海", 40), cand("元火", 5), // differ IN the head 石/海/火 → NOT a series (§2's job)
|
|
cand("元气", 6), cand("酒气", 6), // share head 气 but only two → below min, no blob
|
|
cand("转", 9), // one rune → never a member
|
|
}
|
|
got := DetectSeries(cands, zhSeries)
|
|
|
|
grade := []string{"甲等", "乙等", "丙等", "丁等"}
|
|
id := got[grade[0]]
|
|
if id == 0 {
|
|
t.Fatalf("the grade set must form a series: %v", got)
|
|
}
|
|
for _, k := range grade {
|
|
if got[k] != id {
|
|
t.Fatalf("%s must join the grade series (id %d), got %d", k, id, got[k])
|
|
}
|
|
}
|
|
// The §1↔§2 reconciliation: 元石/元海 differ in the HEAD, so they are different entities, not a series.
|
|
for _, k := range []string{"元石", "元海", "元火", "元气", "酒气", "转"} {
|
|
if got[k] != 0 {
|
|
t.Fatalf("%s must NOT be a series member (id %d) — the naive rule's failure mode", k, got[k])
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestDetectSeriesLongerHead covers a multi-rune shared head (甲等资质/乙等资质/丙等资质): the differing rune
|
|
// is the leading modifier, the whole 等资质 tail is the head.
|
|
func TestDetectSeriesLongerHead(t *testing.T) {
|
|
cands := []Candidate{cand("甲等资质", 2), cand("乙等资质", 2), cand("丙等资质", 2)}
|
|
got := DetectSeries(cands, zhSeries)
|
|
if got["甲等资质"] == 0 || got["甲等资质"] != got["乙等资质"] || got["乙等资质"] != got["丙等资质"] {
|
|
t.Fatalf("a set sharing the trailing head 等资质 must form one series: %v", got)
|
|
}
|
|
}
|
|
|
|
// TestDetectSeriesRespectsPairData: the channel is off for a pair whose source is not dense-script — the same
|
|
// single-position difference that clusters for zh must NOT cluster when Enabled is false (care/core is a
|
|
// spelling coincidence), and the head side flips with HeadFinal.
|
|
func TestDetectSeriesRespectsPairData(t *testing.T) {
|
|
alpha := []Candidate{cand("care", 3), cand("core", 3), cand("cure", 3)}
|
|
if got := DetectSeries(alpha, SeriesParams{Enabled: false, HeadFinal: true}); len(got) != 0 {
|
|
t.Fatalf("series channel must be inert for a non-series pair, got %v", got)
|
|
}
|
|
// Head-INITIAL direction: the head is the leading rune, the modifier is non-initial. 등X where the tail
|
|
// varies and 등 is shared as the head.
|
|
headInit := []Candidate{cand("등가", 1), cand("등나", 1), cand("등다", 1)}
|
|
got := DetectSeries(headInit, SeriesParams{Enabled: true, HeadFinal: false})
|
|
if got["등가"] == 0 || got["등가"] != got["등나"] || got["등나"] != got["등다"] {
|
|
t.Fatalf("with HeadFinal=false the leading rune is the head and the trailing modifier varies: %v", got)
|
|
}
|
|
// The same set under HeadFinal=true shares no head (they differ at the last position) → no series.
|
|
if got := DetectSeries(headInit, zhSeries); len(got) != 0 {
|
|
t.Fatalf("under head-final the trailing-varying set is not a series, got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestBatchKeepsSeriesWhole: a series scattered across key order lands in ONE batch even under a budget so
|
|
// tight every singleton is its own batch — co-batching is the whole mechanism.
|
|
func TestBatchKeepsSeriesWhole(t *testing.T) {
|
|
// Key-sorted order interleaves the series with a non-member; a tiny budget would otherwise split it.
|
|
cands := []Candidate{cand("丁等", 3), cand("丙等", 3), cand("乙等", 3), cand("甲等", 3), cand("中间", 50)}
|
|
seriesID := DetectSeries(cands, zhSeries)
|
|
batches := Batch(cands, 10, seriesID) // 10 runes: every unit overflows, so packing cannot help by luck
|
|
var seriesBatch []Candidate
|
|
for _, b := range batches {
|
|
for _, c := range b {
|
|
if seriesID[c.Key] != 0 {
|
|
seriesBatch = b
|
|
}
|
|
}
|
|
}
|
|
if len(seriesBatch) != 4 {
|
|
t.Fatalf("all four grade members must share one batch, got %d: %v", len(seriesBatch), batches)
|
|
}
|
|
// Every candidate still appears exactly once across all batches.
|
|
seen := map[string]int{}
|
|
for _, b := range batches {
|
|
for _, c := range b {
|
|
seen[c.Key]++
|
|
}
|
|
}
|
|
if len(seen) != len(cands) {
|
|
t.Fatalf("batching dropped or duplicated a candidate: %v", seen)
|
|
}
|
|
}
|
|
|
|
// TestBatchRunesFlagsOversize pins the fix-pack §в observability seam: BatchRunes measures a batch the way
|
|
// Batch packs it, so the terminologist can WARN when §1 keeps a unit whole past the budget instead of
|
|
// silently over-running the output cap. A lone candidate too big for the budget stays ONE over-cap batch.
|
|
func TestBatchRunesFlagsOversize(t *testing.T) {
|
|
big := Candidate{Key: "元", Src: "元", Type: "term", KWIC: []string{strings.Repeat("к", 500)}}
|
|
if BatchRunes([]Candidate{big}) <= 100 {
|
|
t.Fatal("a candidate carrying a 500-rune context must render well over a 100-rune budget")
|
|
}
|
|
batches := Batch([]Candidate{big}, 100, nil)
|
|
if len(batches) != 1 || BatchRunes(batches[0]) <= 100 {
|
|
t.Fatalf("an oversize lone candidate must stay one over-cap batch the caller can WARN on: %v", batches)
|
|
}
|
|
}
|
|
|
|
// TestBatchValueOrder: batches are processed most-frequent first, so a budget ceiling drops the rarest terms
|
|
// (feed_cap) rather than the lexicographic tail. Content of each batch is untouched by the ordering.
|
|
func TestBatchValueOrder(t *testing.T) {
|
|
// Three singletons, each its own batch under a tiny budget; frequency is the only thing that differs.
|
|
cands := []Candidate{cand("阿", 1), cand("布", 99), cand("此", 50)}
|
|
batches := Batch(cands, 5, nil)
|
|
if len(batches) != 3 {
|
|
t.Fatalf("expected three singleton batches, got %d", len(batches))
|
|
}
|
|
if batches[0][0].Key != "布" || batches[len(batches)-1][0].Key != "阿" {
|
|
t.Fatalf("batches must run most-frequent first, least-frequent last: %v", batches)
|
|
}
|
|
}
|
|
|
|
// TestBatchNilSeriesMatchesLegacyOrder guards the alphabetic/off path: with no series and uniform frequency,
|
|
// Batch preserves the incoming key order exactly (the pre-§1 behaviour), so a book that forms no series takes
|
|
// a byte-identical path.
|
|
func TestBatchNilSeriesMatchesLegacyOrder(t *testing.T) {
|
|
var cands []Candidate
|
|
for _, k := range []string{"a", "b", "c", "d", "e"} {
|
|
cands = append(cands, Candidate{Key: k, Src: k, KWIC: []string{strings.Repeat("к", 200)}})
|
|
}
|
|
batches := Batch(cands, 400, nil)
|
|
var order []string
|
|
for _, b := range batches {
|
|
for _, c := range b {
|
|
order = append(order, c.Key)
|
|
}
|
|
}
|
|
if strings.Join(order, "") != "abcde" {
|
|
t.Fatalf("uniform-frequency, series-free batching must preserve key order, got %v", order)
|
|
}
|
|
}
|