textmachine/backend/internal/terminology/family_test.go

package terminology

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"testing"
)

// family_test.go: the §G1 family channel. What it has to prove is not "groups form" but the four properties
// the measurement asked for — a family stays whole across a batch boundary, a variable-length rank line
// stays whole, an equal-length series does not regress, and neither of them turns into a blob.

// zhFamily mirrors the han rows the bank-data plane ships (internal/lang/bankdata/family-morphology.txt):
// names lead with the clan morpheme, realia end with the generic head, both need two runes of root, a family
// is two surfaces, a unit is at most 24.
var zhFamily = FamilyParams{
	Enabled: true, HeadFinal: true,
	Affix: map[string]FamilyAffix{
		"name":     {MinRunes: 2},
		"nickname": {MinRunes: 2},
		"place":    {Suffix: true, MinRunes: 2},
		"title":    {Suffix: true, MinRunes: 2},
		"term":     {Suffix: true, MinRunes: 2},
	},
	MinMembers: 2, MaxMembers: 24, ContainmentRunes: 2,
}

func typed(key, typ string, freq int) Candidate {
	return Candidate{Key: key, Src: key, Type: typ, Freq: freq}
}

// unitsOf inverts a unit map into sorted member lists, for readable assertions.
func unitsOf(unitID map[string]int) map[int][]string {
	out := map[int][]string{}
	for k, id := range unitID {
		out[id] = append(out[id], k)
	}
	for _, v := range out {
		sort.Strings(v)
	}
	return out
}

func sameUnit(t *testing.T, unitID map[string]int, keys ...string) int {
	t.Helper()
	id := unitID[keys[0]]
	if id == 0 {
		t.Fatalf("%s is in no unit: %v", keys[0], unitsOf(unitID))
	}
	for _, k := range keys[1:] {
		if unitID[k] != id {
			t.Fatalf("%s must share a unit with %s (%d vs %d): %v", k, keys[0], unitID[k], id, unitsOf(unitID))
		}
	}
	return id
}

// TestDetectFamiliesGroupsBySharedRootAsymmetrically is the measured case (research/24 §B4): the 古月 family
// is eight surfaces of DIFFERENT lengths, which DetectSeries cannot see, and the drafts rendered it two ways
// on either side of a batch boundary. The clan morpheme LEADS a name and the generic head ENDS a realia
// term, and the asymmetry is data — so the same string in the other position forms nothing.
func TestDetectFamiliesGroupsBySharedRootAsymmetrically(t *testing.T) {
	cands := []Candidate{
		typed("古月方源", "name", 40), typed("古月正", "name", 9), typed("古月赤练", "name", 5),
		typed("元海空窍", "term", 7), typed("天海空窍", "term", 4), // realia sharing the trailing 海空窍
		typed("方源古月", "name", 2), // the clan morpheme TRAILING a name: not a family under the name rule
	}
	fams := DetectFamilies(cands, zhFamily)
	byAnchor := map[string][]string{}
	for _, f := range fams {
		byAnchor[f.Anchor] = f.Keys
	}
	if got := byAnchor["古月"]; len(got) != 3 {
		t.Fatalf("the clan family must hold its three names, got %v (all: %v)", got, byAnchor)
	}
	if got := byAnchor["海空窍"]; len(got) != 2 {
		t.Fatalf("two realia sharing the trailing 海空窍 are one family, got %v", got)
	}
	// The asymmetry is real: 方源古月 shares 古月 as a SUFFIX, and the name rule reads prefixes.
	for _, f := range fams {
		if f.Anchor == "古月" {
			for _, k := range f.Keys {
				if k == "方源古月" {
					t.Fatal("a name sharing the clan morpheme as a SUFFIX must not join the prefix family")
				}
			}
		}
	}
}

// TestFamilyKeepsAVariableLengthRankLineWhole is the acceptance criterion of the fix-pack's own §2(а): the
// 1..12 rank line must be ONE batch unit. 一转…九转 is an equal-length series; 十一转 and 十二转 are three
// runes long and so fall outside it, and on the live corpus they landed in another batch — a rank scale
// split in half is exactly the chimera the co-batching exists to prevent.
func TestFamilyKeepsAVariableLengthRankLineWhole(t *testing.T) {
	var cands []Candidate
	for _, k := range []string{"一转", "二转", "三转", "四转", "五转", "六转", "七转", "八转", "九转", "十一转", "十二转"} {
		cands = append(cands, typed(k, "term", 5))
	}
	cands = append(cands, typed("中间", "term", 90)) // an unrelated term, so a whole-list unit would prove nothing
	sort.Slice(cands, func(i, j int) bool { return cands[i].Key < cands[j].Key })

	seriesID := DetectSeries(cands, zhSeries)
	if seriesID["十一转"] != 0 {
		t.Fatal("test premise broken: a three-rune surface cannot be in an equal-length series")
	}
	unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
	if st.Refused != 0 || st.Held != 0 {
		t.Fatalf("nothing here should hit a guard: %+v", st)
	}
	id := sameUnit(t, unitID, "一转", "二转", "九转", "十一转", "十二转")
	if unitID["中间"] == id {
		t.Fatal("an unrelated term must not be swept into the rank unit")
	}
	// And the unit survives the batcher under a budget so tight every singleton is its own batch.
	batches := Batch(cands, 10, unitID)
	var rank []Candidate
	for _, b := range batches {
		for _, c := range b {
			if unitID[c.Key] == id {
				rank = b
			}
		}
	}
	if len(rank) != 11 {
		t.Fatalf("the whole 1..12 line must ride ONE call, got %d: %v", len(rank), rank)
	}
}

// TestFamilyDoesNotRegressAnEqualLengthSeries: the other half of the same criterion. 甲等/乙等/丙等 are two
// runes, so no two-rune root is a PROPER affix of them and the family channel has nothing to say; the series
// must come through untouched.
func TestFamilyDoesNotRegressAnEqualLengthSeries(t *testing.T) {
	cands := []Candidate{typed("甲等", "term", 3), typed("乙等", "term", 3), typed("丙等", "term", 3), typed("丁等", "term", 3), typed("中间", "term", 50)}
	seriesID := DetectSeries(cands, zhSeries)
	unitID, _ := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
	id := sameUnit(t, unitID, "甲等", "乙等", "丙等", "丁等")
	for k, got := range unitID {
		if got == id && !strings.HasSuffix(k, "等") {
			t.Fatalf("%s joined the grade unit and shares no morpheme with it: %v", k, unitsOf(unitID))
		}
	}
}

// TestContainmentPairSharesAUnit is §2(б): two MINED surfaces where one contains the other (元海 ⊂ 元海空窍)
// had neither a Related record (Merge computes those for banknote-only rows) nor any atomicity in the
// batcher — so the compound could be consolidated in one call and its own part in another, which is how a
// rendering stops carrying the element it is built from.
func TestContainmentPairSharesAUnit(t *testing.T) {
	cands := []Candidate{typed("元海", "term", 30), typed("元海空窍", "term", 8), typed("青茅山", "place", 12)}
	unitID, _ := MergeUnits(nil, DetectFamilies(cands, zhFamily), zhFamily)
	sameUnit(t, unitID, "元海", "元海空窍")
	if unitID["青茅山"] != 0 {
		t.Fatalf("an unrelated surface must stay a singleton: %v", unitsOf(unitID))
	}
	// The anchor bound is real: a ONE-rune surface may not anchor the channel, or a generic morpheme sweeps
	// half the bank into one call.
	short := []Candidate{typed("蛊", "term", 900), typed("蛊虫", "term", 40), typed("蛊师", "term", 30), typed("月光蛊", "term", 10)}
	if unit, _ := MergeUnits(nil, DetectFamilies(short, zhFamily), zhFamily); unit["蛊"] != 0 {
		t.Fatalf("a one-rune anchor is below ContainmentRunes and must form no unit: %v", unitsOf(unit))
	}
}

// TestFamilyMergeRefusesAnOversizeUnit: the merge is capped and the refusal is REPORTED, not silent — a
// family split across two calls is the defect this channel exists to close, so the caller has to be able to
// say it happened.
func TestFamilyMergeRefusesAnOversizeUnit(t *testing.T) {
	var cands []Candidate
	for _, k := range []string{"古月方源", "古月正", "古月赤练", "古月山寨"} {
		cands = append(cands, typed(k, "name", 5))
	}
	tight := zhFamily
	tight.MaxMembers = 3
	unitID, st := MergeUnits(nil, DetectFamilies(cands, tight), tight)
	if st.Refused == 0 {
		t.Fatalf("a four-member family under a three-member cap must be refused: %v", unitsOf(unitID))
	}
	for _, v := range unitsOf(unitID) {
		if len(v) > tight.MaxMembers {
			t.Fatalf("a unit past the cap was built anyway: %v", v)
		}
	}
}

// TestFamilyChannelOffIsTheSeriesMap guards the generality answer: a source that declares no family data
// takes the byte-identical path it took before the channel existed — the SAME map object, so nothing
// downstream can even observe a difference.
func TestFamilyChannelOffIsTheSeriesMap(t *testing.T) {
	cands := []Candidate{typed("古月方源", "name", 5), typed("古月正", "name", 5), typed("甲等", "term", 3), typed("乙等", "term", 3), typed("丙等", "term", 3)}
	seriesID := DetectSeries(cands, zhSeries)
	off := FamilyParams{} // no data → inert
	if fams := DetectFamilies(cands, off); fams != nil {
		t.Fatalf("an undeclared family channel must detect nothing, got %v", fams)
	}
	unitID, st := MergeUnits(seriesID, DetectFamilies(cands, off), off)
	if st != (MergeStats{}) {
		t.Fatalf("an inert channel gives nothing up, got %+v", st)
	}
	// The SAME map object, not a copy: with the channel off nothing downstream can even observe a difference.
	if reflect.ValueOf(unitID).Pointer() != reflect.ValueOf(seriesID).Pointer() {
		t.Fatalf("the inert path must hand back the series map itself: %v vs %v", unitID, seriesID)
	}
	for k, v := range seriesID {
		if unitID[k] != v {
			t.Fatalf("series membership changed with the channel off: %v vs %v", unitID, seriesID)
		}
	}
}

// TestFamiliesOnBankFullFixtureAreBoundedAndWhole runs the channel over the real corpus DetectSeries was
// calibrated on. Two live properties, both of which a synthetic fixture cannot show: the 古月 family — the
// one the drafts actually split across a batch boundary — comes out as ONE unit, and no unit grows past the
// declared bound, so the co-batching cannot quietly turn into "the whole bank in one call".
func TestFamiliesOnBankFullFixtureAreBoundedAndWhole(t *testing.T) {
	cands := loadBankFullSurfaces(t)
	seriesID := DetectSeries(cands, zhSeries)
	unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
	units := unitsOf(unitID)

	clan := unitID["古月"]
	if clan == 0 {
		t.Fatalf("the clan surface must be in a unit with the names built on it: %v", units)
	}
	clanMembers := 0
	for _, k := range units[clan] {
		if strings.HasPrefix(k, "古月") {
			clanMembers++
		}
	}
	if clanMembers < 10 {
		t.Fatalf("the 古月 family is the measured chimera and must ride one call, got %d of %v", clanMembers, units[clan])
	}
	for id, keys := range units {
		if len(keys) > zhFamily.MaxMembers {
			t.Fatalf("unit %d grew past the declared bound (%d): %v", id, zhFamily.MaxMembers, keys)
		}
	}
	// EXACT numbers, not bounds. Two mutations survived a bounds-only version of this test: swapping the
	// relatedness test to substring containment (which splits 丙等 from 丙等资质 — the §B4 chimera) and
	// inverting the family ranking (which changes which candidates share a CALL, i.e. the money). Both leave
	// every inequality above satisfied, so only the composition can catch them.
	if len(units) != 14 || largestUnit(units) != 20 || st.Refused != 0 || st.Held != 2 {
		t.Fatalf("the corpus shape moved: %d units, largest %d, %+v — if this is intended, re-measure and update the numbers, do not relax the test",
			len(units), largestUnit(units), st)
	}
	// The rank line and the grade family, whole, by composition.
	sameUnit(t, unitID, "一转", "九转", "一转蛊师", "九转境界")
	sameUnit(t, unitID, "丙等", "丙等资质", "甲等", "甲等资质")
	// And the exact SHAPE of the partition. The aggregate counts above survive an inverted family ranking —
	// which silently re-cuts which candidates share a CALL, i.e. what the run buys — so the sizes are pinned.
	sizes := make([]int, 0, len(units))
	for _, keys := range units {
		sizes = append(sizes, len(keys))
	}
	sort.Sort(sort.Reverse(sort.IntSlice(sizes)))
	want := []int{20, 13, 10, 10, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2}
	if fmt.Sprint(sizes) != fmt.Sprint(want) {
		t.Fatalf("the partition SHAPE moved: %v want %v — re-measure before relaxing this", sizes, want)
	}
	t.Logf("bankfull: %d candidates → %d units, largest %d, gave up %+v",
		len(cands), len(units), largestUnit(units), st)
}

// TestGradeFamilyMergesWithItsSeriesThroughASharedRune is the case that made the relatedness test wrong the
// first time round, kept as its own pin: the 等资质 series and the {丙等, 丙等资, 丙等资质} family share no
// substring in either direction, but they plainly share the morpheme 等 — and 丙等 landing in a different
// call from the 丙等资质 built on it is the research/24 §B4 chimera, term for term.
func TestGradeFamilyMergesWithItsSeriesThroughASharedRune(t *testing.T) {
	var cands []Candidate
	for _, k := range []string{"甲等", "乙等", "丙等", "甲等资质", "乙等资质", "丙等资质", "丙等资"} {
		cands = append(cands, typed(k, "term", 5))
	}
	unitID, st := MergeUnits(DetectSeries(cands, zhSeries), DetectFamilies(cands, zhFamily), zhFamily)
	if st.Held != 0 {
		t.Fatalf("a family sharing 等 with the series must not be held back: %+v (%v)", st, unitsOf(unitID))
	}
	sameUnit(t, unitID, "丙等", "丙等资质", "甲等", "甲等资质")
}

func largestUnit(units map[int][]string) int {
	n := 0
	for _, keys := range units {
		if len(keys) > n {
			n = len(keys)
		}
	}
	return n
}

// TestSeriesHoldsOnlyItsOwnMembers is the guard's honest boundary, and it exists because the first cut of it
// was wrong in exactly the way that matters. «Series stronger than family» must hold back the SERIES — not
// dissolve the family around it. Here the clan 古月 has five names, three of which happen to form a 雄-series;
// the series' root (雄) has nothing to do with the clan morpheme, so the three keep their own unit, and the
// other two must still be co-batched as the clan. Refusing the whole merge instead leaves the clan as five
// singletons — the batch-boundary chimera of research/24 §B4, arriving through the guard meant to prevent it.
func TestSeriesHoldsOnlyItsOwnMembers(t *testing.T) {
	cands := []Candidate{
		typed("古月方源", "name", 40), typed("古月正", "name", 9),
		typed("古月大雄", "name", 5), typed("古月二雄", "name", 5), typed("古月三雄", "name", 5),
	}
	seriesID := DetectSeries(cands, zhSeries)
	if seriesID["古月大雄"] == 0 {
		t.Fatal("test premise broken: the three *雄 names must form an equal-length series")
	}
	unitID, st := MergeUnits(seriesID, DetectFamilies(cands, zhFamily), zhFamily)
	if st.Held == 0 {
		t.Fatalf("holding part of a family back is a split the owner must be told about: %+v", st)
	}
	// The series keeps its three members …
	sameUnit(t, unitID, "古月大雄", "古月二雄", "古月三雄")
	// … and the REST of the clan is still one unit, not two singletons.
	clan := sameUnit(t, unitID, "古月方源", "古月正")
	if clan == unitID["古月大雄"] {
		t.Fatalf("an unrelated series must not be swallowed by the family: %v", unitsOf(unitID))
	}
}

// TestFamilyMergesAcrossASeriesWhenTheRootIsShared is the other side of the same guard: when the series' own
// root IS the family's anchor (the 转 case), nothing is held back and the whole line rides one call.
func TestFamilyMergesAcrossASeriesWhenTheRootIsShared(t *testing.T) {
	var cands []Candidate
	for _, k := range []string{"一转", "二转", "三转", "四转", "五转", "十一转"} {
		cands = append(cands, typed(k, "term", 5))
	}
	_, st := MergeUnits(DetectSeries(cands, zhSeries), DetectFamilies(cands, zhFamily), zhFamily)
	if st.Held != 0 {
		t.Fatalf("a family sharing the series' own root must not be held back: %+v", st)
	}
}

// TestFamilyRulesCoverEveryTypeACandidateCanCarry: the family rules are keyed on a candidate's TYPE, and the
// set a candidate can carry is wider than the classifier's answer vocabulary — the banknote channel accepts
// `nickname` from a draft and that type rides onto a draft-side-only candidate. A type with no rule forms no
// families, silently, which is a whole class of the bank going un-co-batched for no stated reason.
func TestFamilyRulesCoverEveryTypeACandidateCanCarry(t *testing.T) {
	for _, typ := range TypeNames(CandidateTypes) {
		if _, has := zhFamily.Affix[typ]; !has {
			t.Fatalf("type %q can reach a candidate and has no family rule — declare one or say why in the data file", typ)
		}
	}
	// And it is live, not just declared: two nicknames sharing the clan morpheme are one family.
	cands := []Candidate{typed("方小子", "nickname", 5), typed("方小鬼", "nickname", 4)}
	if fams := DetectFamilies(cands, zhFamily); len(fams) == 0 {
		t.Fatalf("a nickname family must form: %v", fams)
	}
}