textmachine/backend/internal/terminology/terminology_test.go

475 lines
23 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package terminology
import (
"reflect"
"strings"
"testing"
)
// terminology_test.go: the pure fixtures for the terminologist's deterministic core. Each pins one
// load-bearing decision of the merge/rank/parse layer — the ones where being wrong is silent: a merged
// entity that should have stayed separate, a rendering that wins on frequency alone, a reply line that
// injects a term nobody asked about.
func TestMergeExactAndAliasCollapseButContainmentDoesNot(t *testing.T) {
mined := []Mined{
{Key: "方源", Src: "方源", Type: "name", Freq: 12, SinceCh: 1, Aliases: []string{"方小子"}},
{Key: "花家", Src: "花家", Type: "name", Freq: 8, SinceCh: 2},
}
observed := []Observed{
{Key: "方源", Src: "方源", Type: "name", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 3}}},
{Key: "方小子", Src: "方小子", Type: "nickname", Proposals: []Proposal{{Dst: "малец Фан", Chunks: 1}}},
{Key: "古月方源", Src: "古月方源", Type: "name", Proposals: []Proposal{{Dst: "Гуюэ Фан Юань", Chunks: 2}}},
}
got := Merge(mined, observed)
byKey := map[string]Candidate{}
for _, c := range got {
byKey[c.Key] = c
}
fy := byKey["方源"]
if fy.Origin != OriginBoth {
t.Fatalf("an exact-key proposal must mark the candidate as seen by both channels, got %q", fy.Origin)
}
// The ALIAS proposal is a rendering OF the same entity → it merges.
if fy.Spread() != 2 {
t.Fatalf("the alias proposal must merge into its entity, variants=%+v", fy.Variants)
}
// The CONTAINING surface is a DIFFERENT entity: «Гуюэ Фан Юань» is not a rendering of 方源, and folding
// it in would put a wrong variant on a term the owner is about to sign.
gy, ok := byKey["古月方源"]
if !ok {
t.Fatal("a containing banknote-only surface must become its own candidate (the reverse section)")
}
if gy.Origin != OriginBanknote {
t.Fatalf("reverse-section origin = %q, want %q", gy.Origin, OriginBanknote)
}
if !reflect.DeepEqual(gy.Related, []string{"方源"}) {
t.Fatalf("the containment relation must be recorded, got %v", gy.Related)
}
for _, v := range fy.Variants {
if v.Dst == "Гуюэ Фан Юань" {
t.Fatal("a containing surface's rendering leaked into the contained term's variants")
}
}
// Output order is by key, never map order.
for i := 1; i < len(got); i++ {
if got[i-1].Key >= got[i].Key {
t.Fatalf("merge output must be key-ordered: %v", keysOf(got))
}
}
}
// TestMergeFoldsIdenticalRenderingsOfOneEntity pins the alias fold. Two SURFACES of one entity (方源 and
// its alias 方小子) can propose the SAME rendering; appended raw that reads as two competing variants, so
// Spread() — the disagreement column the owner reads at the stop — reports a conflict that does not exist,
// and the §C2-3 frequency factor scores the consensus on half its evidence.
func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) {
mined := []Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子"}}}
observed := []Observed{
{Key: "方源", Src: "方源", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 3}}},
{Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 2}, {Dst: "малец Фан", Chunks: 1}}},
}
var c Candidate
for _, got := range Merge(mined, observed) {
if got.Key == "方源" { // select by key: the alias surface is now a candidate of its own too
c = got
}
}
if c.Spread() != 2 {
t.Fatalf("one rendering from two surfaces is ONE variant (plus the genuinely different one): %+v", c.Variants)
}
if c.Variants[0].Dst != "Фан Юань" || c.Variants[0].Chunks != 5 {
t.Fatalf("identical renderings must pool their evidence: %+v", c.Variants)
}
}
// TestMergeKeepsAnAliasSwallowedSurface is the polygon's G-series address reproduced and closed at the
// input-assembly boundary. Their measurement: a surname cluster (葛家) held a clan title (族长) as an alias,
// and the title vanished from the delta over 50 chapters. Reproduced HERE by execution before the fix: the
// merge folded the observed 族长 into 葛家 and the surface disappeared from the terminologist's input
// entirely, while its rendering «глава клана» was re-labelled as a rendering OF the family name — so the
// role would have been asked to consolidate a surname into a title.
//
// The MINIMUM the addendum asks for is "do not lose such candidates when assembling the terminologist's
// input", and that is exactly the scope here: the surface stands on its own, the cluster still gets the
// evidence (ranking unchanged), the rendering carries its provenance, and Origin `alias` keeps the row OUT
// of the emitted delta — proposing it for signature would be the clustering fix, which is a separate call.
func TestMergeKeepsAnAliasSwallowedSurface(t *testing.T) {
mined := []Mined{{Key: "葛家", Src: "葛家", Type: "name", Freq: 40, Aliases: []string{"族长"}}}
observed := []Observed{{Key: "族长", Src: "族长", Type: "title", Proposals: []Proposal{{Dst: "глава клана", Chunks: 3}}}}
got := Merge(mined, observed)
var title, cluster *Candidate
for i := range got {
switch got[i].Key {
case "族长":
title = &got[i]
case "葛家":
cluster = &got[i]
}
}
if title == nil {
t.Fatalf("the swallowed surface must still reach the terminologist's input: %+v", got)
}
if title.Origin != OriginAlias {
t.Fatalf("it must be marked as claimed by a cluster (and so kept out of the delta), got %q", title.Origin)
}
if len(title.Related) != 1 || title.Related[0] != "葛家" {
t.Fatalf("the row must name the cluster that claims it: %+v", title.Related)
}
if title.Best() != "глава клана" {
t.Fatalf("it must carry its own renderings: %+v", title.Variants)
}
// The cluster keeps the evidence — ranking is unchanged — but the rendering is no longer presented as
// its own: a mis-clustered alias must not silently re-label a surname to a title.
if cluster == nil || len(cluster.Variants) != 1 || cluster.Variants[0].Via != "族长" {
t.Fatalf("the cluster must keep the evidence WITH its provenance: %+v", cluster)
}
if !strings.Contains(RenderBatch([]Candidate{*cluster}), "proposed for 族长") {
t.Fatalf("the provenance must be visible on the wire:\n%s", RenderBatch([]Candidate{*cluster}))
}
}
func TestMergeIsDeterministic(t *testing.T) {
mined := []Mined{{Key: "b", Src: "b"}, {Key: "a", Src: "a"}, {Key: "c", Src: "c"}}
observed := []Observed{
{Key: "z", Src: "z", Proposals: []Proposal{{Dst: "Z", Chunks: 1}}},
{Key: "a", Src: "a", Proposals: []Proposal{{Dst: "A", Chunks: 1}}},
}
first := keysOf(Merge(mined, observed))
for i := 0; i < 20; i++ { // map iteration would show up here, not on one run
if got := keysOf(Merge(mined, observed)); !reflect.DeepEqual(got, first) {
t.Fatalf("merge is order-unstable: %v vs %v", got, first)
}
}
}
func keysOf(cs []Candidate) []string {
out := make([]string, len(cs))
for i, c := range cs {
out[i] = c.Key
}
return out
}
func TestAttachKWICGivesSourceContextsInReadingOrder(t *testing.T) {
chunks := []Chunk{
{Chapter: 2, ChunkIdx: 0, NSource: "потом方源снова"},
{Chapter: 1, ChunkIdx: 0, NSource: "сначала方源тут и方源там"},
}
cands := AttachKWIC([]Candidate{{Key: "方源"}}, chunks, 3, 4)
got := cands[0].KWIC
if len(got) != 3 {
t.Fatalf("want 3 contexts, got %d (%v)", len(got), got)
}
// Chapter 1 comes first even though it is SECOND in the input slice, and its two hits precede
// chapter 2's.
if !strings.HasSuffix(got[0], "тут") || !strings.HasSuffix(got[1], "там") || !strings.HasPrefix(got[2], "отом") {
t.Fatalf("contexts must be in (chapter, chunk, offset) order, got %v", got)
}
// A term the source does not contain gets an HONESTLY empty list — that emptiness is the signal that
// the draft invented it.
if k := AttachKWIC([]Candidate{{Key: "нетутакого"}}, chunks, 3, 4)[0].KWIC; len(k) != 0 {
t.Fatalf("an absent term must have no contexts, got %v", k)
}
}
func TestScoreVariantsIsNotMajority(t *testing.T) {
// The frequent rendering is a semantic guess; the rare one is a proper transliteration. §C2-3 says the
// canon is chosen over ALL the evidence, not by counting — so conformance must be able to beat count.
c := Candidate{Key: "方源", Src: "方源", Type: "name", Variants: []Variant{
{Dst: "Источник", Chunks: 5},
{Dst: "Фан Юань", Chunks: 1},
}}
ScoreVariants(&c, ScoreOpts{Conformance: func(dst, typ string) float64 {
if dst == "Фан Юань" {
return 1
}
return 0
}})
if c.Best() != "Фан Юань" {
t.Fatalf("plain majority won: %+v", c.Variants)
}
// …and with NO other signal, frequency does decide (the factor is real, just not sovereign).
c2 := Candidate{Key: "x", Src: "x", Variants: []Variant{{Dst: "Редкий", Chunks: 1}, {Dst: "Частый", Chunks: 5}}}
ScoreVariants(&c2, ScoreOpts{})
if c2.Best() != "Частый" {
t.Fatalf("with no other evidence the frequent rendering should lead: %+v", c2.Variants)
}
}
func TestScoreVariantsNeighbourAnchorAndMalformed(t *testing.T) {
// A rendering that agrees with an APPROVED sibling of the same source series outranks a MORE FREQUENT
// one that does not. The competing rendering deliberately shares no lexeme with the signed sibling and
// leads on count — otherwise both variants take the bonus, the winner falls to the alphabetical
// tie-break, and the assertion pins nothing (audit of 26.07: that is exactly what this test used to do).
c := Candidate{Key: "元火", Src: "元火", Type: "term", Variants: []Variant{
{Dst: "пламя духа", Chunks: 2},
{Dst: "изначальный огонь", Chunks: 1},
}}
ns := []Neighbour{{Src: "元水", Dst: "изначальная вода"}}
ScoreVariants(&c, ScoreOpts{Neighbours: ns})
if c.Best() != "изначальный огонь" {
t.Fatalf("agreement with a signed sibling must beat a more frequent stranger: %+v", c.Variants)
}
// …and the factor is what did it: with no signed siblings the frequent rendering leads.
bare := Candidate{Key: "元火", Src: "元火", Type: "term", Variants: []Variant{
{Dst: "пламя духа", Chunks: 2},
{Dst: "изначальный огонь", Chunks: 1},
}}
ScoreVariants(&bare, ScoreOpts{})
if bare.Best() != "пламя духа" {
t.Fatalf("without the neighbour anchor the count must decide, else the test proves nothing: %+v", bare.Variants)
}
// D39.47: there is no pair/genre factor to test beside the neighbour one. The formula's ONLY consistency
// anchor is the signed bank, and a rendering nobody signed is decided by count and the pair's
// transliteration table — not by an industry default the market does not actually have. The competing
// «совершенствующийся»/«практик культивации» case that used to live here is now, correctly, a count
// decision until the owner signs one of them.
g := Candidate{Key: "修炼者", Src: "修炼者", Type: "term", Variants: []Variant{
{Dst: "совершенствующийся", Chunks: 3},
{Dst: "практик культивации", Chunks: 1},
}}
ScoreVariants(&g, ScoreOpts{})
if g.Best() != "совершенствующийся" {
t.Fatalf("with nothing signed the count decides — no register is imposed on the book: %+v", g.Variants)
}
for _, v := range g.Variants {
if contains(v.Signals, "genre") {
t.Fatalf("the genre factor is removed (D39.47); a signal naming it means it came back: %+v", v)
}
}
// A SHORT word that merely opens a longer one is not the same lexeme: «гу» (蛊, the book's own term)
// against «Гуюэ» (a clan), «ад» against «адрес». Without the floor the anchor factors fire on
// renderings that have nothing to do with the signed row, and their signal string says they legitimately did.
short := Candidate{Key: "蛊虫", Src: "蛊虫", Type: "term", Variants: []Variant{
{Dst: "червь гуюэ", Chunks: 1},
{Dst: "гу-червь", Chunks: 1},
}}
ScoreVariants(&short, ScoreOpts{Neighbours: []Neighbour{{Src: "蛊师", Dst: "гу"}}})
for _, v := range short.Variants {
if v.Dst == "червь гуюэ" && contains(v.Signals, "neighbour") {
t.Fatalf("a 2-letter prefix must not count as a shared lexeme: %+v", v)
}
}
// A visibly mangled form is penalised and says so.
m := Candidate{Key: "y", Src: "y", Variants: []Variant{{Dst: "Хорошо", Chunks: 1}, {Dst: "Плохо-", Chunks: 5}}}
ScoreVariants(&m, ScoreOpts{})
if m.Best() != "Хорошо" {
t.Fatalf("a mangled form must not win on count alone: %+v", m.Variants)
}
if !contains(m.Variants[1].Signals, "malformed") {
t.Fatalf("the penalty must be visible in the signals: %+v", m.Variants[1])
}
}
// TestCanonForRanksContainmentFirstAndCaps pins the anchor selection: the signed rows a batch actually
// needs, strongest relation first, bounded. Containment is ranked above a shared character because it is
// the case that failed live — 元海 inside 元海空窍, where ignoring the signature produced a rendering that
// contradicted two approved rows at once.
func TestCanonForRanksContainmentFirstAndCaps(t *testing.T) {
batch := []Candidate{{Key: "元海空窍", Src: "元海空窍"}}
ns := []Neighbour{
{Src: "元火", Dst: "изначальный огонь"}, // shares 元 only
{Src: "元海", Dst: "море истинной ци"}, // CONTAINED in the candidate
{Src: "花家", Dst: "клан Хуа"}, // unrelated
}
got := CanonFor(batch, ns, 10)
if len(got) != 2 {
t.Fatalf("only the related rows belong in the anchor, got %v", got)
}
if got[0][0] != "元海" {
t.Fatalf("containment must outrank a bare shared character: %v", got)
}
// The cap is real: an anchor that grows with the bank would eventually cost more than the batch.
if capped := CanonFor(batch, ns, 1); len(capped) != 1 || capped[0][0] != "元海" {
t.Fatalf("cap must keep the STRONGEST relation, got %v", capped)
}
// A candidate that IS the signed row learns nothing from itself.
if self := CanonFor([]Candidate{{Key: "元海", Src: "元海"}}, []Neighbour{{Src: "元海", Dst: "море истинной ци"}}, 10); len(self) != 0 {
t.Fatalf("a row must not anchor itself: %v", self)
}
// Nothing signed → no block at all, so a book with no canon takes the pre-existing path byte for byte.
if a := RenderCanonAnchor(nil); a != "" {
t.Fatalf("an empty anchor must render to nothing, got %q", a)
}
// One marker, one block, and the signed rows verbatim — D39.47 left exactly one anchor, so the rendering
// is fully pinned here rather than probed for a prefix.
if a := RenderCanonAnchor([][2]string{{"元海", "море истинной ци"}, {"空窍", "апертура"}}); a != CanonMarker+"\n元海\tморе истинной ци\n空窍\tапертура" {
t.Fatalf("canon anchor rendering = %q", a)
}
}
// TestCanonConflictsFlagsOnlyRealContradictions pins the deterministic half: the anchor is a nudge, this
// is the check. Live on 26.07 the role rendered 一代族长 as «Первый глава рода» with 族长 signed «глава
// клана» — a contradiction with no signal anywhere. It must flag that, and must NOT flag a rendering that
// carries the signed word in another case (Russian inflection is not a contradiction).
func TestCanonConflictsFlagsOnlyRealContradictions(t *testing.T) {
cands := []Candidate{
{Key: "一代族长", Src: "一代族长"},
{Key: "四代族长", Src: "四代族长"},
{Key: "花家", Src: "花家"},
}
ns := []Neighbour{{Src: "族长", Dst: "глава клана"}}
got := CanonConflicts(cands, map[string]string{
"一代族长": "Первый глава рода", // contradicts: no lexeme of the signed rendering
"四代族长": "Четвёртый глава клана", // agrees, in another form
"花家": "клан Хуа", // contains no signed source at all
}, ns)
if len(got) != 1 || got[0].Src != "一代族长" || got[0].CanonSrc != "族长" {
t.Fatalf("exactly the contradicting rendering must be flagged, got %+v", got)
}
// Nothing signed, or nothing consolidated → nothing to say.
if c := CanonConflicts(cands, map[string]string{"一代族长": "Первый глава рода"}, nil); len(c) != 0 {
t.Fatalf("with no signed bank there is no canon to contradict: %+v", c)
}
// The live probe's other half: a compound that DOES carry the signature, in an oblique case, must not
// be flagged. «море истинной ци» signed, «апертура моря истинной ци» consolidated — at a flat
// five-character stem test «море»/«моря» failed to match and this false-flagged.
ok := CanonConflicts(
[]Candidate{{Key: "元海空窍", Src: "元海空窍"}},
map[string]string{"元海空窍": "апертура моря истинной ци"},
[]Neighbour{{Src: "元海", Dst: "море истинной ци"}})
if len(ok) != 0 {
t.Fatalf("a compound carrying the signed rendering in another case is not a contradiction: %+v", ok)
}
}
// TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds pins two silent-loss holes the audit surfaced: a
// stray double space inside a rendering must not truncate it (the splitter is tolerant BY DESIGN, and that
// tolerance cuts both ways), and a batch must actually render within the budget it was split for.
func TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds(t *testing.T) {
got, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
if st.Bad != 0 || got["方源"] != "Фан Юань" {
t.Fatalf("a double space inside a rendering must not bank half a name: %v (bad=%d)", got, st.Bad)
}
// A rendering that IS the source is not a rendering: a model echoing the term back would otherwise be
// banked as this book's canon and shown to the editor as «方源 ⟨проверить⟩».
echo, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
if _, banked := echo["方源"]; banked || st.Bad != 1 {
t.Fatalf("an echoed source must be refused and counted: %v (bad=%d)", echo, st.Bad)
}
cands := make([]Candidate, 200)
for i := range cands {
cands[i] = Candidate{Key: "k", Src: "s", Type: "term"}
}
for i, b := range Batch(cands, 600, nil) {
if n := len([]rune(RenderBatch(b))); n > 600 {
t.Fatalf("batch %d renders %d runes over a 600-rune budget", i, n)
}
}
}
func contains(ss []string, want string) bool {
for _, s := range ss {
if s == want {
return true
}
}
return false
}
func TestParseReplyAcceptsOnlyAskedTerms(t *testing.T) {
id := func(s string) string { return s }
reply := strings.Join([]string{
"方源\tФан Юань",
"花家 Дом Хуа", // two spaces instead of a tab
"青茅山 | гора Цинмао",
"чужой\tЧужой", // never asked about → must not enter the bank
"忘却\t" + NoDst,
"мусор",
}, "\n")
got, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil)
want := map[string]string{"方源": "Фан Юань", "花家": "Дом Хуа", "青茅山": "гора Цинмао", "忘却": ""}
if !reflect.DeepEqual(got, want) {
t.Fatalf("parse = %#v, want %#v", got, want)
}
if st.Bad != 2 {
t.Fatalf("the unusable lines must be COUNTED (silence about them is how a paid call turns into an empty bank), got %d", st.Bad)
}
// The declined term is present with an EMPTY rendering — a decision, distinct from "no line at all".
if v, ok := got["忘却"]; !ok || v != "" {
t.Fatalf("the decline sentinel must map to an explicit empty rendering, got %q/%v", v, ok)
}
}
func TestParseReplyNormalizesTheKey(t *testing.T) {
// The caller passes the engine's source-key normalizer; a reply written in another orthography must
// still land on its term.
lower := strings.ToLower
got, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil)
if got["fang"] != "Фан" {
t.Fatalf("the reply key must be normalized by the caller's function, got %#v", got)
}
}
func TestBatchCoversEveryCandidateExactlyOnce(t *testing.T) {
var cands []Candidate
for _, k := range []string{"a", "b", "c", "d", "e"} {
cands = append(cands, Candidate{Key: k, Src: k, KWIC: []string{strings.Repeat("к", 200)}})
}
batches := Batch(cands, 400, nil)
if len(batches) < 2 {
t.Fatalf("the fixture must actually split, got %d batch(es)", len(batches))
}
seen := map[string]int{}
for _, b := range batches {
for _, c := range b {
seen[c.Key]++
}
}
if len(seen) != len(cands) {
t.Fatalf("batching dropped candidates: %v", seen)
}
for k, n := range seen {
if n != 1 {
t.Fatalf("candidate %q appears in %d batches", k, n)
}
}
// A single oversized candidate is never silently dropped.
big := []Candidate{{Key: "big", Src: "big", KWIC: []string{strings.Repeat("ю", 5000)}}}
if got := Batch(big, 10, nil); len(got) != 1 || len(got[0]) != 1 {
t.Fatalf("an oversized candidate must still get its own batch, got %v", got)
}
}
func TestRenderBatchIsDeterministicAndCarriesTheEvidence(t *testing.T) {
c := Candidate{
Key: "方源", Src: "方源", Type: "name", Freq: 12, SinceCh: 1, Origin: OriginBoth,
Aliases: []string{"方小子"}, Related: []string{"古月方源"}, Evidence: []string{"surname:方"},
Variants: []Variant{{Dst: "Фан Юань", Chunks: 3}, {Dst: "Фань Юань", Chunks: 1}},
KWIC: []string{"方源来到"},
}
first := RenderBatch([]Candidate{c})
for i := 0; i < 10; i++ {
if RenderBatch([]Candidate{c}) != first {
t.Fatal("the rendered block must be byte-stable across renders")
}
}
for _, want := range []string{"方源", "type: name", "origin: both", "freq: 12", "aliases: 方小子",
"related: 古月方源", "drafts: Фан Юань ×3 | Фань Юань ×1", "ctx: 方源来到"} {
if !strings.Contains(first, want) {
t.Fatalf("the block must carry %q:\n%s", want, first)
}
}
}
// TestAttachKWICCountsOccurrencesUncapped: the KWIC list is CAPPED, the frequency is not. Using the list
// length as the count would print "freq: 3" for a term that occurs forty times — in exactly the column
// the owner reads to judge whether a term is worth signing.
func TestAttachKWICCountsOccurrencesUncapped(t *testing.T) {
chunks := []Chunk{{Chapter: 1, NSource: strings.Repeat("方源и", 40)}}
got := AttachKWIC([]Candidate{{Key: "方源"}}, chunks, 3, 4)
if len(got[0].KWIC) != 3 {
t.Fatalf("the KWIC list must stay capped, got %d", len(got[0].KWIC))
}
if got[0].Freq != 40 {
t.Fatalf("the frequency must be the real occurrence count, not the capped list length: %d", got[0].Freq)
}
// A detector-supplied frequency is authoritative and must not be overwritten.
kept := AttachKWIC([]Candidate{{Key: "方源", Freq: 7}}, chunks, 3, 4)
if kept[0].Freq != 7 {
t.Fatalf("a mined candidate's own frequency must survive, got %d", kept[0].Freq)
}
}