textmachine/backend/internal/terminology/series.go

564 lines
20 KiB
Go

package terminology
import (
"sort"
"strings"
)
// series.go: the SERIES channel (bank-quality §1, D39.68). A grade/rank series — 甲等/乙等/丙等, 一转…九转 —
// shares one generic HEAD and varies only on a modifier. Measured (D39.65 row 21): the drafts disagree on the
// head across batch boundaries (甲等→«класс», 乙丙丁等→«ранг»), because Merge orders candidates by key and the
// shared head sorts by its DIFFERING prefix. The fix (live probe) is co-batching — shown the set at once the
// model picks ONE head; the first-occurrence test (§7.7) showed order/frequency tricks cannot do it, only
// co-batching. This file detects the series; the batcher keeps each one whole in a single call.
// SeriesParams is the pair-data that governs the channel. It arrives as a value (the package stays pure and
// pair-agnostic): the pipeline resolves it from the source language's declared morphology.
type SeriesParams struct {
// Enabled gates the whole channel. It is true only for DENSE scripts where one rune ≈ one morpheme, so a
// single-rune difference is a real minimal pair; in an alphabetic source care/core differ in one letter by
// coincidence, not by morphology, so the channel is off and Batch behaves exactly as before.
Enabled bool
// HeadFinal says the shared generic word is the TRAILING rune(s) (the CJK modifier-head norm). The differing
// modifier rune is then non-final; with HeadFinal false the head is leading and the modifier is non-initial.
HeadFinal bool
// MinMembers is the smallest set that counts as a series (≤0 → 3). Below three a "series" is just a pair that
// happens to share a character — too weak to override the key order for.
MinMembers int
}
// DetectSeries returns key → series id (>0) for every candidate that belongs to a head-aware series; a key in
// no series is absent from the map. Empty when the channel is off.
//
// A series is ≥MinMembers surfaces of EQUAL rune length that are identical except in ONE rune position, and
// that position is NOT the head (the last rune for HeadFinal, the first otherwise). So the members share a
// generic head and vary only on the modifier — 甲等/乙等/丙等 (grades of 等). 元石/元海 differ IN the head
// (石/海): different entities, the type step's job (§2), never one series. There is no transitive closure:
// membership is by an exact shared skeleton, so 元气~酒气 (a shared head-rune 气 on two families) never chains
// into a blob the way a naive "differ in one position" rule did (measured: an 11-surface blob mixing three
// families and the protagonist's name).
func DetectSeries(cands []Candidate, p SeriesParams) map[string]int {
min := p.MinMembers
if min <= 0 {
min = 3
}
if !p.Enabled || len(cands) < min {
return nil
}
// group[skeleton] = the distinct keys that reduce to it by blanking their one modifier position.
type gkey struct {
n, pos int
skel string
}
groups := map[gkey][]string{}
seenInGroup := map[gkey]map[string]bool{}
var order []gkey // first-seen group order, for deterministic assignment
for _, c := range cands {
rs := []rune(c.Key)
n := len(rs)
if n < 2 { // a one-rune surface is all head, no modifier to vary
continue
}
for pos := 0; pos < n; pos++ {
if p.HeadFinal && pos == n-1 { // the last rune is (part of) the head
continue
}
if !p.HeadFinal && pos == 0 {
continue
}
g := gkey{n, pos, blankAt(rs, pos)}
if seenInGroup[g] == nil {
seenInGroup[g] = map[string]bool{}
order = append(order, g)
}
if !seenInGroup[g][c.Key] {
seenInGroup[g][c.Key] = true
groups[g] = append(groups[g], c.Key)
}
}
}
// A key can satisfy several groups (varying at more than one modifier position). Assign biggest-first so
// the strongest series claims its members; a group left below min after its members were claimed elsewhere
// simply does not form. Deterministic: sort by descending size, then by first-seen order.
sort.SliceStable(order, func(i, j int) bool { return len(groups[order[i]]) > len(groups[order[j]]) })
out := map[string]int{}
next := 1
for _, g := range order {
var fresh []string
for _, k := range groups[g] {
if out[k] == 0 {
fresh = append(fresh, k)
}
}
if len(fresh) >= min {
for _, k := range fresh {
out[k] = next
}
next++
}
}
return out
}
// blankAt returns the key with the rune at pos replaced by a byte no source key contains, so surfaces that
// agree everywhere except pos share a skeleton and nothing else collides onto it.
func blankAt(rs []rune, pos int) string {
out := make([]rune, len(rs))
copy(out, rs)
out[pos] = 0
return string(out)
}
// orderByUnit returns the candidates with every batch unit's members made contiguous, anchored at the
// position of the unit's first member in the input order; a candidate in no unit keeps its place. The input
// is already key-sorted, so the result is deterministic and disturbs the key order minimally.
func orderByUnit(cands []Candidate, unitID map[string]int) []Candidate {
if len(unitID) == 0 {
return cands
}
byID := map[int][]Candidate{}
for _, c := range cands {
if id := unitID[c.Key]; id != 0 {
byID[id] = append(byID[id], c)
}
}
out := make([]Candidate, 0, len(cands))
emitted := map[int]bool{}
for _, c := range cands {
id := unitID[c.Key]
if id == 0 {
out = append(out, c)
continue
}
if emitted[id] {
continue // a later member of an already-emitted unit
}
out = append(out, byID[id]...)
emitted[id] = true
}
return out
}
// --- the FAMILY channel (fix-pack §G1) ---------------------------------------------------------------
//
// A SERIES is the narrow case: equal-length surfaces differing in ONE non-head rune. It leaves the wider
// one open, and the wider one is where the measured damage is (research/24 §B4): the 古月-family — eight
// surfaces of DIFFERENT lengths sharing the clan morpheme — came back «Гу Юэ» in batch 0 and «Гуюэ» in
// batch 2 from ONE model, a split that correlates with the batch boundary perfectly. Nothing in the
// terminologist can fix a disagreement it is never shown; only co-batching can.
//
// A family is anchored on a shared ROOT MORPHEME, and the side that root sits on is ASYMMETRIC by class —
// a name leads with its clan surname, a realia term ends with its generic head. Both the side and the
// required root length are pair/script DATA (lang.FamilyMorphology), so a source with no rows leaves the
// channel inert and takes the byte-identical, family-free path.
// FamilyAffix is one type's family rule: which side carries the shared root and how long it must be.
type FamilyAffix struct {
Suffix bool
MinRunes int
}
// FamilyParams is the pair-data that governs the family channel. Like SeriesParams it arrives as a VALUE:
// the package stays pure and pair-agnostic, and the pipeline resolves it from the source's declared
// morphology.
type FamilyParams struct {
// Enabled gates the whole channel — true only for DENSE scripts, for the same reason the series channel
// is: one rune ≈ one morpheme, so a shared affix is a shared MORPHEME and not a coincidence of spelling.
Enabled bool
// HeadFinal says the generic head is the trailing morpheme. It is used to read a SERIES' own root when
// deciding whether a family may join it (see MergeUnits).
HeadFinal bool
// Affix is the per-type rule (name|place|title|term); a type absent here forms no families.
Affix map[string]FamilyAffix
// MinMembers is the smallest set that counts as a family (<2 → 2).
MinMembers int
// MaxMembers bounds the batch unit a family merge may produce; 0 = unbounded. A unit past it is a review
// set nobody reads and an output cap nobody planned, so the merge is refused and counted.
MaxMembers int
// ContainmentRunes is the shortest candidate that may anchor the COMPOSITIONAL channel: a candidate that
// is itself a substring of another candidate (元海 ⊂ 元海空窍 — both mined, so neither Related nor the
// batcher held them together before this pack). Bounding the anchor is what keeps one generic morpheme
// from sweeping half the bank into a single call. 0 → the channel is off.
ContainmentRunes int
}
// Family is one detected family: the shared ANCHOR morpheme and the candidate keys carrying it (key-sorted).
type Family struct {
Anchor string
Keys []string
}
// DetectFamilies returns the families of a candidate list, strongest first (larger set, then longer — i.e.
// more specific — anchor, then first-seen). It does NOT assign membership: a candidate legitimately shows
// up in several families, and which of them becomes a batch unit is decided in MergeUnits, where the
// existing series are already on the table. Deterministic; nothing here iterates a map for output.
//
// There is no transitive closure over runes: an anchor is a WHOLE root morpheme of at least MinRunes (or a
// whole candidate, for the compositional channel), never "these two differ in one position", which is the
// rule that produced an 11-surface blob mixing three families and the protagonist's name.
func DetectFamilies(cands []Candidate, p FamilyParams) []Family {
if !p.Enabled || len(cands) < 2 {
return nil
}
min := p.MinMembers
if min < 2 {
min = 2
}
type gkey struct{ side, anchor string }
members := map[gkey][]string{}
seen := map[gkey]map[string]bool{}
var order []gkey
add := func(g gkey, key string) {
m := seen[g]
if m == nil {
m = map[string]bool{}
seen[g] = m
order = append(order, g)
}
if !m[key] {
m[key] = true
members[g] = append(members[g], key)
}
}
isCand := make(map[string]bool, len(cands))
for _, c := range cands {
if c.Key != "" {
isCand[c.Key] = true
}
}
for _, c := range cands {
if c.Key == "" {
continue
}
rule, ok := p.Affix[typeOr(c.Type)]
if !ok || rule.MinRunes <= 0 {
continue
}
side := "prefix"
if rule.Suffix {
side = "suffix"
}
rs := []rune(c.Key)
// PROPER affixes only (l < len): a surface is not a member of a family whose root IS the surface —
// it is that root, and it joins below, once some other surface actually carries it.
for l := rule.MinRunes; l < len(rs); l++ {
anchor := string(rs[:l])
if rule.Suffix {
anchor = string(rs[len(rs)-l:])
}
g := gkey{side, anchor}
add(g, c.Key)
if isCand[anchor] {
// The surface that IS the root belongs to its own family. Leaving it out is how the head of a
// family ends up in a different call from the family it heads.
//
// ⚠ Bounded: an anchor is at least MinRunes long, so a ONE-rune head that is itself a bank term
// (蛊, 转, 窍) is NOT reachable this way and does stay in another call from its family. That is
// a data decision, not a Go one — family_containment_runes 1 turns it on — and it is left off
// because a single generic morpheme anchors half the bank; the cost is measured at the cold run.
add(g, anchor)
}
}
}
if p.ContainmentRunes > 0 {
for _, c := range cands {
if c.Key == "" || len([]rune(c.Key)) < p.ContainmentRunes {
continue
}
g := gkey{"contains", c.Key}
for _, d := range cands {
if d.Key == "" || d.Key == c.Key {
continue
}
if strings.Contains(d.Key, c.Key) {
add(g, c.Key)
add(g, d.Key)
}
}
}
}
all := make([]Family, 0, len(order))
for _, g := range order {
if len(members[g]) < min {
continue
}
keys := append([]string(nil), members[g]...)
sort.Strings(keys)
all = append(all, Family{Anchor: g.anchor, Keys: keys})
}
sort.SliceStable(all, func(i, j int) bool {
if len(all[i].Keys) != len(all[j].Keys) {
return len(all[i].Keys) > len(all[j].Keys)
}
ai, aj := len([]rune(all[i].Anchor)), len([]rune(all[j].Anchor))
if ai != aj {
return ai > aj // the more SPECIFIC root wins the earlier merge
}
return all[i].Anchor < all[j].Anchor
})
// Dedupe AFTER the ranking, so the survivor of a duplicated set is the strongest description of it. The
// affix and containment channels legitimately describe the same set from two directions (元海空窍/天海空窍
// share the suffix 空窍 AND the longer 海空窍), and one of them is enough: a duplicate merges nothing the
// first did not, but it double-counts in the merge's give-up statistics — and a number an operator reads
// has to mean what it says.
out := make([]Family, 0, len(all))
seenSet := map[string]bool{}
for _, f := range all {
set := strings.Join(f.Keys, "\x00")
if seenSet[set] {
continue
}
seenSet[set] = true
out = append(out, f)
}
return out
}
// MergeUnits folds the series and the families into ONE batch-unit map — the map Batch packs against.
//
// Membership is a PARTITION (a candidate is in at most one unit), built by merging whole units rather than
// by claiming candidates: an exclusive assignment cannot express the case the fix-pack exists for. The
// rank line 一转…九转 is a series; 十一转 is three runes long and so is not, and under exclusive assignment
// it stays outside forever — the batch boundary splits a rank scale in half. Merging the family that spans
// both puts the whole 1..12 line in one call, which is the stated criterion.
//
// Two guards keep the merge from becoming a blob, and BOTH are counted rather than silent — a guard that
// splits a family is doing the same damage the channel exists to prevent, so it has to be visible:
// - SERIES PRECEDENCE: a series whose own root is unrelated to a family's anchor keeps its members; the
// family then forms WITHOUT them (转-case: root 转, anchors 转 / 一转 — related, so they merge instead).
// - MaxMembers: a merge that would produce a unit larger than the pair declares is refused.
//
// With no families (the channel off, or nothing detected) the series map is returned UNCHANGED, so a source
// with no family data takes the byte-identical path.
func MergeUnits(seriesID map[string]int, fams []Family, p FamilyParams) (unitID map[string]int, stats MergeStats) {
if len(fams) == 0 {
return seriesID, MergeStats{}
}
u := newUnitSet()
cappedSet := map[string]bool{} // key sets whose merge the member cap turned down
bySeries := map[int][]string{}
for k, id := range seriesID {
if id != 0 {
bySeries[id] = append(bySeries[id], k)
}
}
ids := make([]int, 0, len(bySeries))
for id := range bySeries {
ids = append(ids, id)
}
sort.Ints(ids)
for _, id := range ids {
mem := bySeries[id]
sort.Strings(mem)
for _, k := range mem[1:] {
u.union(mem[0], k)
}
if root := commonAffix(mem, p.HeadFinal); root != "" {
r := u.find(mem[0])
u.roots[r] = append(u.roots[r], root)
}
}
for _, f := range fams {
var targets []string
seen := map[string]bool{}
for _, k := range f.Keys {
r := u.find(k)
if seen[r] {
continue
}
seen[r] = true
targets = append(targets, r)
}
if len(targets) < 2 {
continue // already one unit
}
// SERIES PRECEDENCE, applied per TARGET rather than to the whole family. A series whose root is
// unrelated to this anchor is HELD BACK — and only it. Refusing the whole merge instead would
// dissolve the family around it: the clan 古月, minus three of its names that happen to form a
// 雄-series, would go back to being five singletons, which is the batch-boundary chimera this
// channel exists to close, arriving through the guard that was supposed to protect a series.
var join []string
size := 0
for _, r := range targets {
blocked := false
for _, sr := range u.roots[r] {
if !sharesMorpheme(f.Anchor, sr) {
blocked = true
break
}
}
if blocked {
continue
}
join = append(join, r)
size += u.size[r]
}
if len(join) < 2 {
continue
}
if p.MaxMembers > 0 && size > p.MaxMembers {
cappedSet[strings.Join(f.Keys, "\x00")] = true
continue
}
for _, r := range join[1:] {
u.union(join[0], r)
}
}
// The give-up statistics are read off the FINAL partition, never off the attempts. A guard that fired
// mid-way is not a family the owner meets split: a later merge routinely puts those keys back together,
// and counting the attempt reports a split that does not exist. Measured on the coldrun-a distillation:
// counting attempts claimed three, of which one (蛊师 against 一转蛊师) had ended up in ONE unit anyway.
// A number an operator reads has to mean what it says — the standard this file sets for itself.
final := u.ids()
for _, f := range fams {
if !splitAcrossUnits(f, final) {
continue
}
if cappedSet[strings.Join(f.Keys, "\x00")] {
stats.Refused++
continue
}
stats.Held++
}
return final, stats
}
// splitAcrossUnits reports whether a family's keys ended up in more than one batch unit (a key in no unit
// counts as its own).
func splitAcrossUnits(f Family, unitID map[string]int) bool {
first, seen := 0, false
for _, k := range f.Keys {
id := unitID[k]
if !seen {
first, seen = id, true
continue
}
if id != first || id == 0 {
return true
}
}
return false
}
// MergeStats is what the unit merge had to give up. Both numbers mean "a family the owner will meet split
// across two calls", which is the defect the channel exists to close — so neither may be silent.
type MergeStats struct {
// Refused counts merges the member cap turned down.
Refused int
// Held counts families a SERIES held part of back: the series' own root is unrelated to the family's
// anchor, so the series keeps its members and the family forms without them.
Held int
}
// sharesMorpheme reports whether two anchors share a root MORPHEME — the "делят одну корневую морфему" test
// behind the series-vs-family precedence. In a dense script a morpheme is a rune, so that is the test.
//
// Substring containment is NOT it, and the difference was measured, not reasoned: on the coldrun-a bank the
// containment version held back five families, every one of them wrongly. The 等资质 series (甲等资质/乙等资质/
// 丙等资质, root 等资质) sits beside the family {丙等, 丙等资, 丙等资质}, anchored 丙等. Neither string contains
// the other, so containment called them unrelated and split the grade 丙等 away from the graded aptitude
// built on it — while they plainly share 等, which is the very morpheme whose rendering has to agree. The
// blob these guards protect against is bounded by MaxMembers, not by making relatedness narrow.
func sharesMorpheme(a, b string) bool {
if a == "" || b == "" {
return true
}
return sharedRunes(a, b) > 0
}
// commonAffix returns the longest common trailing (headFinal) or leading morpheme of the keys — a series'
// own root, read off its members so DetectSeries keeps its signature.
func commonAffix(keys []string, headFinal bool) string {
if len(keys) == 0 {
return ""
}
best := []rune(keys[0])
for _, k := range keys[1:] {
rs := []rune(k)
n := 0
for n < len(best) && n < len(rs) {
if headFinal {
if best[len(best)-1-n] != rs[len(rs)-1-n] {
break
}
} else if best[n] != rs[n] {
break
}
n++
}
if headFinal {
best = best[len(best)-n:]
} else {
best = best[:n]
}
if len(best) == 0 {
return ""
}
}
return string(best)
}
// unitSet is the disjoint-set the merge runs on: parent/size by key, plus the series roots a unit carries
// (the precedence guard reads them). Roots are the smallest member key, so the structure is deterministic
// without any map iteration reaching the output.
type unitSet struct {
parent map[string]string
size map[string]int
roots map[string][]string
}
func newUnitSet() *unitSet {
return &unitSet{parent: map[string]string{}, size: map[string]int{}, roots: map[string][]string{}}
}
func (u *unitSet) find(k string) string {
p, ok := u.parent[k]
if !ok {
u.parent[k], u.size[k] = k, 1
return k
}
if p == k {
return k
}
r := u.find(p)
u.parent[k] = r
return r
}
func (u *unitSet) union(a, b string) {
ra, rb := u.find(a), u.find(b)
if ra == rb {
return
}
if rb < ra { // the smaller key is always the root → the same input yields the same structure
ra, rb = rb, ra
}
u.parent[rb] = ra
u.size[ra] += u.size[rb]
u.roots[ra] = append(u.roots[ra], u.roots[rb]...)
delete(u.size, rb)
delete(u.roots, rb)
}
// ids numbers the units of ≥2 members, in the order of their smallest key, so the map is a pure function of
// the input.
func (u *unitSet) ids() map[string]int {
roots := make([]string, 0, len(u.size))
for r, n := range u.size {
if n > 1 {
roots = append(roots, r)
}
}
sort.Strings(roots)
num := make(map[string]int, len(roots))
for i, r := range roots {
num[r] = i + 1
}
out := make(map[string]int, len(u.parent))
for k := range u.parent {
if id := num[u.find(k)]; id != 0 {
out[k] = id
}
}
return out
}