package terminology import ( "reflect" "strings" "testing" ) // terminology_test.go: the pure fixtures for the terminologist's deterministic core. Each pins one // load-bearing decision of the merge/rank/parse layer — the ones where being wrong is silent: a merged // entity that should have stayed separate, a rendering that wins on frequency alone, a reply line that // injects a term nobody asked about. func TestMergeExactAndAliasCollapseButContainmentDoesNot(t *testing.T) { mined := []Mined{ {Key: "方源", Src: "方源", Type: "name", Freq: 12, SinceCh: 1, Aliases: []string{"方小子"}}, {Key: "花家", Src: "花家", Type: "name", Freq: 8, SinceCh: 2}, } observed := []Observed{ {Key: "方源", Src: "方源", Type: "name", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 3}}}, {Key: "方小子", Src: "方小子", Type: "nickname", Proposals: []Proposal{{Dst: "малец Фан", Chunks: 1}}}, {Key: "古月方源", Src: "古月方源", Type: "name", Proposals: []Proposal{{Dst: "Гуюэ Фан Юань", Chunks: 2}}}, } got := Merge(mined, observed) byKey := map[string]Candidate{} for _, c := range got { byKey[c.Key] = c } fy := byKey["方源"] if fy.Origin != OriginBoth { t.Fatalf("an exact-key proposal must mark the candidate as seen by both channels, got %q", fy.Origin) } // The ALIAS proposal is a rendering OF the same entity → it merges. if fy.Spread() != 2 { t.Fatalf("the alias proposal must merge into its entity, variants=%+v", fy.Variants) } // The CONTAINING surface is a DIFFERENT entity: «Гуюэ Фан Юань» is not a rendering of 方源, and folding // it in would put a wrong variant on a term the owner is about to sign. gy, ok := byKey["古月方源"] if !ok { t.Fatal("a containing banknote-only surface must become its own candidate (the reverse section)") } if gy.Origin != OriginBanknote { t.Fatalf("reverse-section origin = %q, want %q", gy.Origin, OriginBanknote) } if !reflect.DeepEqual(gy.Related, []string{"方源"}) { t.Fatalf("the containment relation must be recorded, got %v", gy.Related) } for _, v := range fy.Variants { if v.Dst == "Гуюэ Фан Юань" { t.Fatal("a containing surface's rendering leaked into the contained term's variants") } } // Output order is by key, never map order. for i := 1; i < len(got); i++ { if got[i-1].Key >= got[i].Key { t.Fatalf("merge output must be key-ordered: %v", keysOf(got)) } } } // TestMergeFoldsIdenticalRenderingsOfOneEntity pins the alias fold. Two SURFACES of one entity (方源 and // its alias 方小子) can propose the SAME rendering; appended raw that reads as two competing variants, so // Spread() — the disagreement column the owner reads at the stop — reports a conflict that does not exist, // and the §C2-3 frequency factor scores the consensus on half its evidence. func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) { mined := []Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子"}}} observed := []Observed{ {Key: "方源", Src: "方源", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 3}}}, {Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 2}, {Dst: "малец Фан", Chunks: 1}}}, } var c Candidate for _, got := range Merge(mined, observed) { if got.Key == "方源" { // select by key: the alias surface is now a candidate of its own too c = got } } if c.Spread() != 2 { t.Fatalf("one rendering from two surfaces is ONE variant (plus the genuinely different one): %+v", c.Variants) } if c.Variants[0].Dst != "Фан Юань" || c.Variants[0].Chunks != 5 { t.Fatalf("identical renderings must pool their evidence: %+v", c.Variants) } } // TestMergeKeepsAnAliasSwallowedSurface is the polygon's G-series address reproduced and closed at the // input-assembly boundary. Their measurement: a surname cluster (葛家) held a clan title (族长) as an alias, // and the title vanished from the delta over 50 chapters. Reproduced HERE by execution before the fix: the // merge folded the observed 族长 into 葛家 and the surface disappeared from the terminologist's input // entirely, while its rendering «глава клана» was re-labelled as a rendering OF the family name — so the // role would have been asked to consolidate a surname into a title. // // The MINIMUM the addendum asks for is "do not lose such candidates when assembling the terminologist's // input", and that is exactly the scope here: the surface stands on its own, the cluster still gets the // evidence (ranking unchanged), the rendering carries its provenance, and Origin `alias` keeps the row OUT // of the emitted delta — proposing it for signature would be the clustering fix, which is a separate call. func TestMergeKeepsAnAliasSwallowedSurface(t *testing.T) { mined := []Mined{{Key: "葛家", Src: "葛家", Type: "name", Freq: 40, Aliases: []string{"族长"}}} observed := []Observed{{Key: "族长", Src: "族长", Type: "title", Proposals: []Proposal{{Dst: "глава клана", Chunks: 3}}}} got := Merge(mined, observed) var title, cluster *Candidate for i := range got { switch got[i].Key { case "族长": title = &got[i] case "葛家": cluster = &got[i] } } if title == nil { t.Fatalf("the swallowed surface must still reach the terminologist's input: %+v", got) } if title.Origin != OriginAlias { t.Fatalf("it must be marked as claimed by a cluster (and so kept out of the delta), got %q", title.Origin) } if len(title.Related) != 1 || title.Related[0] != "葛家" { t.Fatalf("the row must name the cluster that claims it: %+v", title.Related) } if title.Best() != "глава клана" { t.Fatalf("it must carry its own renderings: %+v", title.Variants) } // The cluster keeps the evidence — ranking is unchanged — but the rendering is no longer presented as // its own: a mis-clustered alias must not silently re-label a surname to a title. if cluster == nil || len(cluster.Variants) != 1 || cluster.Variants[0].Via != "族长" { t.Fatalf("the cluster must keep the evidence WITH its provenance: %+v", cluster) } if !strings.Contains(RenderBatch([]Candidate{*cluster}), "proposed for 族长") { t.Fatalf("the provenance must be visible on the wire:\n%s", RenderBatch([]Candidate{*cluster})) } } func TestMergeIsDeterministic(t *testing.T) { mined := []Mined{{Key: "b", Src: "b"}, {Key: "a", Src: "a"}, {Key: "c", Src: "c"}} observed := []Observed{ {Key: "z", Src: "z", Proposals: []Proposal{{Dst: "Z", Chunks: 1}}}, {Key: "a", Src: "a", Proposals: []Proposal{{Dst: "A", Chunks: 1}}}, } first := keysOf(Merge(mined, observed)) for i := 0; i < 20; i++ { // map iteration would show up here, not on one run if got := keysOf(Merge(mined, observed)); !reflect.DeepEqual(got, first) { t.Fatalf("merge is order-unstable: %v vs %v", got, first) } } } func keysOf(cs []Candidate) []string { out := make([]string, len(cs)) for i, c := range cs { out[i] = c.Key } return out } func TestAttachKWICGivesSourceContextsInReadingOrder(t *testing.T) { chunks := []Chunk{ {Chapter: 2, ChunkIdx: 0, NSource: "потом方源снова"}, {Chapter: 1, ChunkIdx: 0, NSource: "сначала方源тут и方源там"}, } cands := AttachKWIC([]Candidate{{Key: "方源"}}, chunks, 3, 4) got := cands[0].KWIC if len(got) != 3 { t.Fatalf("want 3 contexts, got %d (%v)", len(got), got) } // Chapter 1 comes first even though it is SECOND in the input slice, and its two hits precede // chapter 2's. if !strings.HasSuffix(got[0], "тут") || !strings.HasSuffix(got[1], "там") || !strings.HasPrefix(got[2], "отом") { t.Fatalf("contexts must be in (chapter, chunk, offset) order, got %v", got) } // A term the source does not contain gets an HONESTLY empty list — that emptiness is the signal that // the draft invented it. if k := AttachKWIC([]Candidate{{Key: "нетутакого"}}, chunks, 3, 4)[0].KWIC; len(k) != 0 { t.Fatalf("an absent term must have no contexts, got %v", k) } } func TestScoreVariantsIsNotMajority(t *testing.T) { // The frequent rendering is a semantic guess; the rare one is a proper transliteration. §C2-3 says the // canon is chosen over ALL the evidence, not by counting — so conformance must be able to beat count. c := Candidate{Key: "方源", Src: "方源", Type: "name", Variants: []Variant{ {Dst: "Источник", Chunks: 5}, {Dst: "Фан Юань", Chunks: 1}, }} ScoreVariants(&c, ScoreOpts{Conformance: func(dst, typ string) float64 { if dst == "Фан Юань" { return 1 } return 0 }}) if c.Best() != "Фан Юань" { t.Fatalf("plain majority won: %+v", c.Variants) } // …and with NO other signal, frequency does decide (the factor is real, just not sovereign). c2 := Candidate{Key: "x", Src: "x", Variants: []Variant{{Dst: "Редкий", Chunks: 1}, {Dst: "Частый", Chunks: 5}}} ScoreVariants(&c2, ScoreOpts{}) if c2.Best() != "Частый" { t.Fatalf("with no other evidence the frequent rendering should lead: %+v", c2.Variants) } } func TestScoreVariantsNeighbourAnchorAndMalformed(t *testing.T) { // A rendering that agrees with an APPROVED sibling of the same source series outranks a MORE FREQUENT // one that does not. The competing rendering deliberately shares no lexeme with the signed sibling and // leads on count — otherwise both variants take the bonus, the winner falls to the alphabetical // tie-break, and the assertion pins nothing (audit of 26.07: that is exactly what this test used to do). c := Candidate{Key: "元火", Src: "元火", Type: "term", Variants: []Variant{ {Dst: "пламя духа", Chunks: 2}, {Dst: "изначальный огонь", Chunks: 1}, }} ns := []Neighbour{{Src: "元水", Dst: "изначальная вода"}} ScoreVariants(&c, ScoreOpts{Neighbours: ns}) if c.Best() != "изначальный огонь" { t.Fatalf("agreement with a signed sibling must beat a more frequent stranger: %+v", c.Variants) } // …and the factor is what did it: with no signed siblings the frequent rendering leads. bare := Candidate{Key: "元火", Src: "元火", Type: "term", Variants: []Variant{ {Dst: "пламя духа", Chunks: 2}, {Dst: "изначальный огонь", Chunks: 1}, }} ScoreVariants(&bare, ScoreOpts{}) if bare.Best() != "пламя духа" { t.Fatalf("without the neighbour anchor the count must decide, else the test proves nothing: %+v", bare.Variants) } // D39.47: there is no pair/genre factor to test beside the neighbour one. The formula's ONLY consistency // anchor is the signed bank, and a rendering nobody signed is decided by count and the pair's // transliteration table — not by an industry default the market does not actually have. The competing // «совершенствующийся»/«практик культивации» case that used to live here is now, correctly, a count // decision until the owner signs one of them. g := Candidate{Key: "修炼者", Src: "修炼者", Type: "term", Variants: []Variant{ {Dst: "совершенствующийся", Chunks: 3}, {Dst: "практик культивации", Chunks: 1}, }} ScoreVariants(&g, ScoreOpts{}) if g.Best() != "совершенствующийся" { t.Fatalf("with nothing signed the count decides — no register is imposed on the book: %+v", g.Variants) } for _, v := range g.Variants { if contains(v.Signals, "genre") { t.Fatalf("the genre factor is removed (D39.47); a signal naming it means it came back: %+v", v) } } // A SHORT word that merely opens a longer one is not the same lexeme: «гу» (蛊, the book's own term) // against «Гуюэ» (a clan), «ад» against «адрес». Without the floor the anchor factors fire on // renderings that have nothing to do with the signed row, and their signal string says they legitimately did. short := Candidate{Key: "蛊虫", Src: "蛊虫", Type: "term", Variants: []Variant{ {Dst: "червь гуюэ", Chunks: 1}, {Dst: "гу-червь", Chunks: 1}, }} ScoreVariants(&short, ScoreOpts{Neighbours: []Neighbour{{Src: "蛊师", Dst: "гу"}}}) for _, v := range short.Variants { if v.Dst == "червь гуюэ" && contains(v.Signals, "neighbour") { t.Fatalf("a 2-letter prefix must not count as a shared lexeme: %+v", v) } } // A visibly mangled form is penalised and says so. m := Candidate{Key: "y", Src: "y", Variants: []Variant{{Dst: "Хорошо", Chunks: 1}, {Dst: "Плохо-", Chunks: 5}}} ScoreVariants(&m, ScoreOpts{}) if m.Best() != "Хорошо" { t.Fatalf("a mangled form must not win on count alone: %+v", m.Variants) } if !contains(m.Variants[1].Signals, "malformed") { t.Fatalf("the penalty must be visible in the signals: %+v", m.Variants[1]) } } // TestCanonForRanksContainmentFirstAndCaps pins the anchor selection: the signed rows a batch actually // needs, strongest relation first, bounded. Containment is ranked above a shared character because it is // the case that failed live — 元海 inside 元海空窍, where ignoring the signature produced a rendering that // contradicted two approved rows at once. func TestCanonForRanksContainmentFirstAndCaps(t *testing.T) { batch := []Candidate{{Key: "元海空窍", Src: "元海空窍"}} ns := []Neighbour{ {Src: "元火", Dst: "изначальный огонь"}, // shares 元 only {Src: "元海", Dst: "море истинной ци"}, // CONTAINED in the candidate {Src: "花家", Dst: "клан Хуа"}, // unrelated } got := CanonFor(batch, ns, 10) if len(got) != 2 { t.Fatalf("only the related rows belong in the anchor, got %v", got) } if got[0][0] != "元海" { t.Fatalf("containment must outrank a bare shared character: %v", got) } // The cap is real: an anchor that grows with the bank would eventually cost more than the batch. if capped := CanonFor(batch, ns, 1); len(capped) != 1 || capped[0][0] != "元海" { t.Fatalf("cap must keep the STRONGEST relation, got %v", capped) } // A candidate that IS the signed row learns nothing from itself. if self := CanonFor([]Candidate{{Key: "元海", Src: "元海"}}, []Neighbour{{Src: "元海", Dst: "море истинной ци"}}, 10); len(self) != 0 { t.Fatalf("a row must not anchor itself: %v", self) } // Nothing signed → no block at all, so a book with no canon takes the pre-existing path byte for byte. if a := RenderCanonAnchor(nil); a != "" { t.Fatalf("an empty anchor must render to nothing, got %q", a) } // One marker, one block, and the signed rows verbatim — D39.47 left exactly one anchor, so the rendering // is fully pinned here rather than probed for a prefix. if a := RenderCanonAnchor([][2]string{{"元海", "море истинной ци"}, {"空窍", "апертура"}}); a != CanonMarker+"\n元海\tморе истинной ци\n空窍\tапертура" { t.Fatalf("canon anchor rendering = %q", a) } } // TestCanonConflictsFlagsOnlyRealContradictions pins the deterministic half: the anchor is a nudge, this // is the check. Live on 26.07 the role rendered 一代族长 as «Первый глава рода» with 族长 signed «глава // клана» — a contradiction with no signal anywhere. It must flag that, and must NOT flag a rendering that // carries the signed word in another case (Russian inflection is not a contradiction). func TestCanonConflictsFlagsOnlyRealContradictions(t *testing.T) { cands := []Candidate{ {Key: "一代族长", Src: "一代族长"}, {Key: "四代族长", Src: "四代族长"}, {Key: "花家", Src: "花家"}, } ns := []Neighbour{{Src: "族长", Dst: "глава клана"}} got := CanonConflicts(cands, map[string]string{ "一代族长": "Первый глава рода", // contradicts: no lexeme of the signed rendering "四代族长": "Четвёртый глава клана", // agrees, in another form "花家": "клан Хуа", // contains no signed source at all }, ns) if len(got) != 1 || got[0].Src != "一代族长" || got[0].CanonSrc != "族长" { t.Fatalf("exactly the contradicting rendering must be flagged, got %+v", got) } // Nothing signed, or nothing consolidated → nothing to say. if c := CanonConflicts(cands, map[string]string{"一代族长": "Первый глава рода"}, nil); len(c) != 0 { t.Fatalf("with no signed bank there is no canon to contradict: %+v", c) } // The live probe's other half: a compound that DOES carry the signature, in an oblique case, must not // be flagged. «море истинной ци» signed, «апертура моря истинной ци» consolidated — at a flat // five-character stem test «море»/«моря» failed to match and this false-flagged. ok := CanonConflicts( []Candidate{{Key: "元海空窍", Src: "元海空窍"}}, map[string]string{"元海空窍": "апертура моря истинной ци"}, []Neighbour{{Src: "元海", Dst: "море истинной ци"}}) if len(ok) != 0 { t.Fatalf("a compound carrying the signed rendering in another case is not a contradiction: %+v", ok) } } // TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds pins two silent-loss holes the audit surfaced: a // stray double space inside a rendering must not truncate it (the splitter is tolerant BY DESIGN, and that // tolerance cuts both ways), and a batch must actually render within the budget it was split for. func TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds(t *testing.T) { got, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil) if st.Bad != 0 || got["方源"] != "Фан Юань" { t.Fatalf("a double space inside a rendering must not bank half a name: %v (bad=%d)", got, st.Bad) } // A rendering that IS the source is not a rendering: a model echoing the term back would otherwise be // banked as this book's canon and shown to the editor as «方源 ⟨проверить⟩». echo, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil) if _, banked := echo["方源"]; banked || st.Bad != 1 { t.Fatalf("an echoed source must be refused and counted: %v (bad=%d)", echo, st.Bad) } cands := make([]Candidate, 200) for i := range cands { cands[i] = Candidate{Key: "k", Src: "s", Type: "term"} } for i, b := range Batch(cands, 600, nil) { if n := len([]rune(RenderBatch(b))); n > 600 { t.Fatalf("batch %d renders %d runes over a 600-rune budget", i, n) } } } func contains(ss []string, want string) bool { for _, s := range ss { if s == want { return true } } return false } func TestParseReplyAcceptsOnlyAskedTerms(t *testing.T) { id := func(s string) string { return s } reply := strings.Join([]string{ "方源\tФан Юань", "花家 Дом Хуа", // two spaces instead of a tab "青茅山 | гора Цинмао", "чужой\tЧужой", // never asked about → must not enter the bank "忘却\t" + NoDst, "мусор", }, "\n") got, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil) want := map[string]string{"方源": "Фан Юань", "花家": "Дом Хуа", "青茅山": "гора Цинмао", "忘却": ""} if !reflect.DeepEqual(got, want) { t.Fatalf("parse = %#v, want %#v", got, want) } if st.Bad != 2 { t.Fatalf("the unusable lines must be COUNTED (silence about them is how a paid call turns into an empty bank), got %d", st.Bad) } // The declined term is present with an EMPTY rendering — a decision, distinct from "no line at all". if v, ok := got["忘却"]; !ok || v != "" { t.Fatalf("the decline sentinel must map to an explicit empty rendering, got %q/%v", v, ok) } } func TestParseReplyNormalizesTheKey(t *testing.T) { // The caller passes the engine's source-key normalizer; a reply written in another orthography must // still land on its term. lower := strings.ToLower got, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil) if got["fang"] != "Фан" { t.Fatalf("the reply key must be normalized by the caller's function, got %#v", got) } } func TestBatchCoversEveryCandidateExactlyOnce(t *testing.T) { var cands []Candidate for _, k := range []string{"a", "b", "c", "d", "e"} { cands = append(cands, Candidate{Key: k, Src: k, KWIC: []string{strings.Repeat("к", 200)}}) } batches := Batch(cands, 400, nil) if len(batches) < 2 { t.Fatalf("the fixture must actually split, got %d batch(es)", len(batches)) } seen := map[string]int{} for _, b := range batches { for _, c := range b { seen[c.Key]++ } } if len(seen) != len(cands) { t.Fatalf("batching dropped candidates: %v", seen) } for k, n := range seen { if n != 1 { t.Fatalf("candidate %q appears in %d batches", k, n) } } // A single oversized candidate is never silently dropped. big := []Candidate{{Key: "big", Src: "big", KWIC: []string{strings.Repeat("ю", 5000)}}} if got := Batch(big, 10, nil); len(got) != 1 || len(got[0]) != 1 { t.Fatalf("an oversized candidate must still get its own batch, got %v", got) } } func TestRenderBatchIsDeterministicAndCarriesTheEvidence(t *testing.T) { c := Candidate{ Key: "方源", Src: "方源", Type: "name", Freq: 12, SinceCh: 1, Origin: OriginBoth, Aliases: []string{"方小子"}, Related: []string{"古月方源"}, Evidence: []string{"surname:方"}, Variants: []Variant{{Dst: "Фан Юань", Chunks: 3}, {Dst: "Фань Юань", Chunks: 1}}, KWIC: []string{"方源来到"}, } first := RenderBatch([]Candidate{c}) for i := 0; i < 10; i++ { if RenderBatch([]Candidate{c}) != first { t.Fatal("the rendered block must be byte-stable across renders") } } for _, want := range []string{"方源", "type: name", "origin: both", "freq: 12", "aliases: 方小子", "related: 古月方源", "drafts: Фан Юань ×3 | Фань Юань ×1", "ctx: 方源来到"} { if !strings.Contains(first, want) { t.Fatalf("the block must carry %q:\n%s", want, first) } } } // TestAttachKWICCountsOccurrencesUncapped: the KWIC list is CAPPED, the frequency is not. Using the list // length as the count would print "freq: 3" for a term that occurs forty times — in exactly the column // the owner reads to judge whether a term is worth signing. func TestAttachKWICCountsOccurrencesUncapped(t *testing.T) { chunks := []Chunk{{Chapter: 1, NSource: strings.Repeat("方源и", 40)}} got := AttachKWIC([]Candidate{{Key: "方源"}}, chunks, 3, 4) if len(got[0].KWIC) != 3 { t.Fatalf("the KWIC list must stay capped, got %d", len(got[0].KWIC)) } if got[0].Freq != 40 { t.Fatalf("the frequency must be the real occurrence count, not the capped list length: %d", got[0].Freq) } // A detector-supplied frequency is authoritative and must not be overwritten. kept := AttachKWIC([]Candidate{{Key: "方源", Freq: 7}}, chunks, 3, 4) if kept[0].Freq != 7 { t.Fatalf("a mined candidate's own frequency must survive, got %d", kept[0].Freq) } }