package terminology import ( "sort" "strings" ) // series.go: the SERIES channel (bank-quality §1, D39.68). A grade/rank series — 甲等/乙等/丙等, 一转…九转 — // shares one generic HEAD and varies only on a modifier. Measured (D39.65 row 21): the drafts disagree on the // head across batch boundaries (甲等→«класс», 乙丙丁等→«ранг»), because Merge orders candidates by key and the // shared head sorts by its DIFFERING prefix. The fix (live probe) is co-batching — shown the set at once the // model picks ONE head; the first-occurrence test (§7.7) showed order/frequency tricks cannot do it, only // co-batching. This file detects the series; the batcher keeps each one whole in a single call. // SeriesParams is the pair-data that governs the channel. It arrives as a value (the package stays pure and // pair-agnostic): the pipeline resolves it from the source language's declared morphology. type SeriesParams struct { // Enabled gates the whole channel. It is true only for DENSE scripts where one rune ≈ one morpheme, so a // single-rune difference is a real minimal pair; in an alphabetic source care/core differ in one letter by // coincidence, not by morphology, so the channel is off and Batch behaves exactly as before. Enabled bool // HeadFinal says the shared generic word is the TRAILING rune(s) (the CJK modifier-head norm). The differing // modifier rune is then non-final; with HeadFinal false the head is leading and the modifier is non-initial. HeadFinal bool // MinMembers is the smallest set that counts as a series (≤0 → 3). Below three a "series" is just a pair that // happens to share a character — too weak to override the key order for. MinMembers int } // DetectSeries returns key → series id (>0) for every candidate that belongs to a head-aware series; a key in // no series is absent from the map. Empty when the channel is off. // // A series is ≥MinMembers surfaces of EQUAL rune length that are identical except in ONE rune position, and // that position is NOT the head (the last rune for HeadFinal, the first otherwise). So the members share a // generic head and vary only on the modifier — 甲等/乙等/丙等 (grades of 等). 元石/元海 differ IN the head // (石/海): different entities, the type step's job (§2), never one series. There is no transitive closure: // membership is by an exact shared skeleton, so 元气~酒气 (a shared head-rune 气 on two families) never chains // into a blob the way a naive "differ in one position" rule did (measured: an 11-surface blob mixing three // families and the protagonist's name). func DetectSeries(cands []Candidate, p SeriesParams) map[string]int { min := p.MinMembers if min <= 0 { min = 3 } if !p.Enabled || len(cands) < min { return nil } // group[skeleton] = the distinct keys that reduce to it by blanking their one modifier position. type gkey struct { n, pos int skel string } groups := map[gkey][]string{} seenInGroup := map[gkey]map[string]bool{} var order []gkey // first-seen group order, for deterministic assignment for _, c := range cands { rs := []rune(c.Key) n := len(rs) if n < 2 { // a one-rune surface is all head, no modifier to vary continue } for pos := 0; pos < n; pos++ { if p.HeadFinal && pos == n-1 { // the last rune is (part of) the head continue } if !p.HeadFinal && pos == 0 { continue } g := gkey{n, pos, blankAt(rs, pos)} if seenInGroup[g] == nil { seenInGroup[g] = map[string]bool{} order = append(order, g) } if !seenInGroup[g][c.Key] { seenInGroup[g][c.Key] = true groups[g] = append(groups[g], c.Key) } } } // A key can satisfy several groups (varying at more than one modifier position). Assign biggest-first so // the strongest series claims its members; a group left below min after its members were claimed elsewhere // simply does not form. Deterministic: sort by descending size, then by first-seen order. sort.SliceStable(order, func(i, j int) bool { return len(groups[order[i]]) > len(groups[order[j]]) }) out := map[string]int{} next := 1 for _, g := range order { var fresh []string for _, k := range groups[g] { if out[k] == 0 { fresh = append(fresh, k) } } if len(fresh) >= min { for _, k := range fresh { out[k] = next } next++ } } return out } // blankAt returns the key with the rune at pos replaced by a byte no source key contains, so surfaces that // agree everywhere except pos share a skeleton and nothing else collides onto it. func blankAt(rs []rune, pos int) string { out := make([]rune, len(rs)) copy(out, rs) out[pos] = 0 return string(out) } // orderByUnit returns the candidates with every batch unit's members made contiguous, anchored at the // position of the unit's first member in the input order; a candidate in no unit keeps its place. The input // is already key-sorted, so the result is deterministic and disturbs the key order minimally. func orderByUnit(cands []Candidate, unitID map[string]int) []Candidate { if len(unitID) == 0 { return cands } byID := map[int][]Candidate{} for _, c := range cands { if id := unitID[c.Key]; id != 0 { byID[id] = append(byID[id], c) } } out := make([]Candidate, 0, len(cands)) emitted := map[int]bool{} for _, c := range cands { id := unitID[c.Key] if id == 0 { out = append(out, c) continue } if emitted[id] { continue // a later member of an already-emitted unit } out = append(out, byID[id]...) emitted[id] = true } return out } // --- the FAMILY channel (fix-pack §G1) --------------------------------------------------------------- // // A SERIES is the narrow case: equal-length surfaces differing in ONE non-head rune. It leaves the wider // one open, and the wider one is where the measured damage is (research/24 §B4): the 古月-family — eight // surfaces of DIFFERENT lengths sharing the clan morpheme — came back «Гу Юэ» in batch 0 and «Гуюэ» in // batch 2 from ONE model, a split that correlates with the batch boundary perfectly. Nothing in the // terminologist can fix a disagreement it is never shown; only co-batching can. // // A family is anchored on a shared ROOT MORPHEME, and the side that root sits on is ASYMMETRIC by class — // a name leads with its clan surname, a realia term ends with its generic head. Both the side and the // required root length are pair/script DATA (lang.FamilyMorphology), so a source with no rows leaves the // channel inert and takes the byte-identical, family-free path. // FamilyAffix is one type's family rule: which side carries the shared root and how long it must be. type FamilyAffix struct { Suffix bool MinRunes int } // FamilyParams is the pair-data that governs the family channel. Like SeriesParams it arrives as a VALUE: // the package stays pure and pair-agnostic, and the pipeline resolves it from the source's declared // morphology. type FamilyParams struct { // Enabled gates the whole channel — true only for DENSE scripts, for the same reason the series channel // is: one rune ≈ one morpheme, so a shared affix is a shared MORPHEME and not a coincidence of spelling. Enabled bool // HeadFinal says the generic head is the trailing morpheme. It is used to read a SERIES' own root when // deciding whether a family may join it (see MergeUnits). HeadFinal bool // Affix is the per-type rule (name|place|title|term); a type absent here forms no families. Affix map[string]FamilyAffix // MinMembers is the smallest set that counts as a family (<2 → 2). MinMembers int // MaxMembers bounds the batch unit a family merge may produce; 0 = unbounded. A unit past it is a review // set nobody reads and an output cap nobody planned, so the merge is refused and counted. MaxMembers int // ContainmentRunes is the shortest candidate that may anchor the COMPOSITIONAL channel: a candidate that // is itself a substring of another candidate (元海 ⊂ 元海空窍 — both mined, so neither Related nor the // batcher held them together before this pack). Bounding the anchor is what keeps one generic morpheme // from sweeping half the bank into a single call. 0 → the channel is off. ContainmentRunes int } // Family is one detected family: the shared ANCHOR morpheme and the candidate keys carrying it (key-sorted). type Family struct { Anchor string Keys []string } // DetectFamilies returns the families of a candidate list, strongest first (larger set, then longer — i.e. // more specific — anchor, then first-seen). It does NOT assign membership: a candidate legitimately shows // up in several families, and which of them becomes a batch unit is decided in MergeUnits, where the // existing series are already on the table. Deterministic; nothing here iterates a map for output. // // There is no transitive closure over runes: an anchor is a WHOLE root morpheme of at least MinRunes (or a // whole candidate, for the compositional channel), never "these two differ in one position", which is the // rule that produced an 11-surface blob mixing three families and the protagonist's name. func DetectFamilies(cands []Candidate, p FamilyParams) []Family { if !p.Enabled || len(cands) < 2 { return nil } min := p.MinMembers if min < 2 { min = 2 } type gkey struct{ side, anchor string } members := map[gkey][]string{} seen := map[gkey]map[string]bool{} var order []gkey add := func(g gkey, key string) { m := seen[g] if m == nil { m = map[string]bool{} seen[g] = m order = append(order, g) } if !m[key] { m[key] = true members[g] = append(members[g], key) } } isCand := make(map[string]bool, len(cands)) for _, c := range cands { if c.Key != "" { isCand[c.Key] = true } } for _, c := range cands { if c.Key == "" { continue } rule, ok := p.Affix[typeOr(c.Type)] if !ok || rule.MinRunes <= 0 { continue } side := "prefix" if rule.Suffix { side = "suffix" } rs := []rune(c.Key) // PROPER affixes only (l < len): a surface is not a member of a family whose root IS the surface — // it is that root, and it joins below, once some other surface actually carries it. for l := rule.MinRunes; l < len(rs); l++ { anchor := string(rs[:l]) if rule.Suffix { anchor = string(rs[len(rs)-l:]) } g := gkey{side, anchor} add(g, c.Key) if isCand[anchor] { // The surface that IS the root belongs to its own family. Leaving it out is how the head of a // family ends up in a different call from the family it heads. // // ⚠ Bounded: an anchor is at least MinRunes long, so a ONE-rune head that is itself a bank term // (蛊, 转, 窍) is NOT reachable this way and does stay in another call from its family. That is // a data decision, not a Go one — family_containment_runes 1 turns it on — and it is left off // because a single generic morpheme anchors half the bank; the cost is measured at the cold run. add(g, anchor) } } } if p.ContainmentRunes > 0 { for _, c := range cands { if c.Key == "" || len([]rune(c.Key)) < p.ContainmentRunes { continue } g := gkey{"contains", c.Key} for _, d := range cands { if d.Key == "" || d.Key == c.Key { continue } if strings.Contains(d.Key, c.Key) { add(g, c.Key) add(g, d.Key) } } } } all := make([]Family, 0, len(order)) for _, g := range order { if len(members[g]) < min { continue } keys := append([]string(nil), members[g]...) sort.Strings(keys) all = append(all, Family{Anchor: g.anchor, Keys: keys}) } sort.SliceStable(all, func(i, j int) bool { if len(all[i].Keys) != len(all[j].Keys) { return len(all[i].Keys) > len(all[j].Keys) } ai, aj := len([]rune(all[i].Anchor)), len([]rune(all[j].Anchor)) if ai != aj { return ai > aj // the more SPECIFIC root wins the earlier merge } return all[i].Anchor < all[j].Anchor }) // Dedupe AFTER the ranking, so the survivor of a duplicated set is the strongest description of it. The // affix and containment channels legitimately describe the same set from two directions (元海空窍/天海空窍 // share the suffix 空窍 AND the longer 海空窍), and one of them is enough: a duplicate merges nothing the // first did not, but it double-counts in the merge's give-up statistics — and a number an operator reads // has to mean what it says. out := make([]Family, 0, len(all)) seenSet := map[string]bool{} for _, f := range all { set := strings.Join(f.Keys, "\x00") if seenSet[set] { continue } seenSet[set] = true out = append(out, f) } return out } // MergeUnits folds the series and the families into ONE batch-unit map — the map Batch packs against. // // Membership is a PARTITION (a candidate is in at most one unit), built by merging whole units rather than // by claiming candidates: an exclusive assignment cannot express the case the fix-pack exists for. The // rank line 一转…九转 is a series; 十一转 is three runes long and so is not, and under exclusive assignment // it stays outside forever — the batch boundary splits a rank scale in half. Merging the family that spans // both puts the whole 1..12 line in one call, which is the stated criterion. // // Two guards keep the merge from becoming a blob, and BOTH are counted rather than silent — a guard that // splits a family is doing the same damage the channel exists to prevent, so it has to be visible: // - SERIES PRECEDENCE: a series whose own root is unrelated to a family's anchor keeps its members; the // family then forms WITHOUT them (转-case: root 转, anchors 转 / 一转 — related, so they merge instead). // - MaxMembers: a merge that would produce a unit larger than the pair declares is refused. // // With no families (the channel off, or nothing detected) the series map is returned UNCHANGED, so a source // with no family data takes the byte-identical path. func MergeUnits(seriesID map[string]int, fams []Family, p FamilyParams) (unitID map[string]int, stats MergeStats) { if len(fams) == 0 { return seriesID, MergeStats{} } u := newUnitSet() cappedSet := map[string]bool{} // key sets whose merge the member cap turned down bySeries := map[int][]string{} for k, id := range seriesID { if id != 0 { bySeries[id] = append(bySeries[id], k) } } ids := make([]int, 0, len(bySeries)) for id := range bySeries { ids = append(ids, id) } sort.Ints(ids) for _, id := range ids { mem := bySeries[id] sort.Strings(mem) for _, k := range mem[1:] { u.union(mem[0], k) } if root := commonAffix(mem, p.HeadFinal); root != "" { r := u.find(mem[0]) u.roots[r] = append(u.roots[r], root) } } for _, f := range fams { var targets []string seen := map[string]bool{} for _, k := range f.Keys { r := u.find(k) if seen[r] { continue } seen[r] = true targets = append(targets, r) } if len(targets) < 2 { continue // already one unit } // SERIES PRECEDENCE, applied per TARGET rather than to the whole family. A series whose root is // unrelated to this anchor is HELD BACK — and only it. Refusing the whole merge instead would // dissolve the family around it: the clan 古月, minus three of its names that happen to form a // 雄-series, would go back to being five singletons, which is the batch-boundary chimera this // channel exists to close, arriving through the guard that was supposed to protect a series. var join []string size := 0 for _, r := range targets { blocked := false for _, sr := range u.roots[r] { if !sharesMorpheme(f.Anchor, sr) { blocked = true break } } if blocked { continue } join = append(join, r) size += u.size[r] } if len(join) < 2 { continue } if p.MaxMembers > 0 && size > p.MaxMembers { cappedSet[strings.Join(f.Keys, "\x00")] = true continue } for _, r := range join[1:] { u.union(join[0], r) } } // The give-up statistics are read off the FINAL partition, never off the attempts. A guard that fired // mid-way is not a family the owner meets split: a later merge routinely puts those keys back together, // and counting the attempt reports a split that does not exist. Measured on the coldrun-a distillation: // counting attempts claimed three, of which one (蛊师 against 一转蛊师) had ended up in ONE unit anyway. // A number an operator reads has to mean what it says — the standard this file sets for itself. final := u.ids() for _, f := range fams { if !splitAcrossUnits(f, final) { continue } if cappedSet[strings.Join(f.Keys, "\x00")] { stats.Refused++ continue } stats.Held++ } return final, stats } // splitAcrossUnits reports whether a family's keys ended up in more than one batch unit (a key in no unit // counts as its own). func splitAcrossUnits(f Family, unitID map[string]int) bool { first, seen := 0, false for _, k := range f.Keys { id := unitID[k] if !seen { first, seen = id, true continue } if id != first || id == 0 { return true } } return false } // MergeStats is what the unit merge had to give up. Both numbers mean "a family the owner will meet split // across two calls", which is the defect the channel exists to close — so neither may be silent. type MergeStats struct { // Refused counts merges the member cap turned down. Refused int // Held counts families a SERIES held part of back: the series' own root is unrelated to the family's // anchor, so the series keeps its members and the family forms without them. Held int } // sharesMorpheme reports whether two anchors share a root MORPHEME — the "делят одну корневую морфему" test // behind the series-vs-family precedence. In a dense script a morpheme is a rune, so that is the test. // // Substring containment is NOT it, and the difference was measured, not reasoned: on the coldrun-a bank the // containment version held back five families, every one of them wrongly. The 等资质 series (甲等资质/乙等资质/ // 丙等资质, root 等资质) sits beside the family {丙等, 丙等资, 丙等资质}, anchored 丙等. Neither string contains // the other, so containment called them unrelated and split the grade 丙等 away from the graded aptitude // built on it — while they plainly share 等, which is the very morpheme whose rendering has to agree. The // blob these guards protect against is bounded by MaxMembers, not by making relatedness narrow. func sharesMorpheme(a, b string) bool { if a == "" || b == "" { return true } return sharedRunes(a, b) > 0 } // commonAffix returns the longest common trailing (headFinal) or leading morpheme of the keys — a series' // own root, read off its members so DetectSeries keeps its signature. func commonAffix(keys []string, headFinal bool) string { if len(keys) == 0 { return "" } best := []rune(keys[0]) for _, k := range keys[1:] { rs := []rune(k) n := 0 for n < len(best) && n < len(rs) { if headFinal { if best[len(best)-1-n] != rs[len(rs)-1-n] { break } } else if best[n] != rs[n] { break } n++ } if headFinal { best = best[len(best)-n:] } else { best = best[:n] } if len(best) == 0 { return "" } } return string(best) } // unitSet is the disjoint-set the merge runs on: parent/size by key, plus the series roots a unit carries // (the precedence guard reads them). Roots are the smallest member key, so the structure is deterministic // without any map iteration reaching the output. type unitSet struct { parent map[string]string size map[string]int roots map[string][]string } func newUnitSet() *unitSet { return &unitSet{parent: map[string]string{}, size: map[string]int{}, roots: map[string][]string{}} } func (u *unitSet) find(k string) string { p, ok := u.parent[k] if !ok { u.parent[k], u.size[k] = k, 1 return k } if p == k { return k } r := u.find(p) u.parent[k] = r return r } func (u *unitSet) union(a, b string) { ra, rb := u.find(a), u.find(b) if ra == rb { return } if rb < ra { // the smaller key is always the root → the same input yields the same structure ra, rb = rb, ra } u.parent[rb] = ra u.size[ra] += u.size[rb] u.roots[ra] = append(u.roots[ra], u.roots[rb]...) delete(u.size, rb) delete(u.roots, rb) } // ids numbers the units of ≥2 members, in the order of their smallest key, so the map is a pure function of // the input. func (u *unitSet) ids() map[string]int { roots := make([]string, 0, len(u.size)) for r, n := range u.size { if n > 1 { roots = append(roots, r) } } sort.Strings(roots) num := make(map[string]int, len(roots)) for i, r := range roots { num[r] = i + 1 } out := make(map[string]int, len(u.parent)) for k := range u.parent { if id := num[u.find(k)]; id != 0 { out[k] = id } } return out }