475 lines
23 KiB
Go
475 lines
23 KiB
Go
package terminology
|
||
|
||
import (
|
||
"reflect"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// terminology_test.go: the pure fixtures for the terminologist's deterministic core. Each pins one
|
||
// load-bearing decision of the merge/rank/parse layer — the ones where being wrong is silent: a merged
|
||
// entity that should have stayed separate, a rendering that wins on frequency alone, a reply line that
|
||
// injects a term nobody asked about.
|
||
|
||
func TestMergeExactAndAliasCollapseButContainmentDoesNot(t *testing.T) {
|
||
mined := []Mined{
|
||
{Key: "方源", Src: "方源", Type: "name", Freq: 12, SinceCh: 1, Aliases: []string{"方小子"}},
|
||
{Key: "花家", Src: "花家", Type: "name", Freq: 8, SinceCh: 2},
|
||
}
|
||
observed := []Observed{
|
||
{Key: "方源", Src: "方源", Type: "name", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 3}}},
|
||
{Key: "方小子", Src: "方小子", Type: "nickname", Proposals: []Proposal{{Dst: "малец Фан", Chunks: 1}}},
|
||
{Key: "古月方源", Src: "古月方源", Type: "name", Proposals: []Proposal{{Dst: "Гуюэ Фан Юань", Chunks: 2}}},
|
||
}
|
||
got := Merge(mined, observed)
|
||
|
||
byKey := map[string]Candidate{}
|
||
for _, c := range got {
|
||
byKey[c.Key] = c
|
||
}
|
||
fy := byKey["方源"]
|
||
if fy.Origin != OriginBoth {
|
||
t.Fatalf("an exact-key proposal must mark the candidate as seen by both channels, got %q", fy.Origin)
|
||
}
|
||
// The ALIAS proposal is a rendering OF the same entity → it merges.
|
||
if fy.Spread() != 2 {
|
||
t.Fatalf("the alias proposal must merge into its entity, variants=%+v", fy.Variants)
|
||
}
|
||
// The CONTAINING surface is a DIFFERENT entity: «Гуюэ Фан Юань» is not a rendering of 方源, and folding
|
||
// it in would put a wrong variant on a term the owner is about to sign.
|
||
gy, ok := byKey["古月方源"]
|
||
if !ok {
|
||
t.Fatal("a containing banknote-only surface must become its own candidate (the reverse section)")
|
||
}
|
||
if gy.Origin != OriginBanknote {
|
||
t.Fatalf("reverse-section origin = %q, want %q", gy.Origin, OriginBanknote)
|
||
}
|
||
if !reflect.DeepEqual(gy.Related, []string{"方源"}) {
|
||
t.Fatalf("the containment relation must be recorded, got %v", gy.Related)
|
||
}
|
||
for _, v := range fy.Variants {
|
||
if v.Dst == "Гуюэ Фан Юань" {
|
||
t.Fatal("a containing surface's rendering leaked into the contained term's variants")
|
||
}
|
||
}
|
||
// Output order is by key, never map order.
|
||
for i := 1; i < len(got); i++ {
|
||
if got[i-1].Key >= got[i].Key {
|
||
t.Fatalf("merge output must be key-ordered: %v", keysOf(got))
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestMergeFoldsIdenticalRenderingsOfOneEntity pins the alias fold. Two SURFACES of one entity (方源 and
|
||
// its alias 方小子) can propose the SAME rendering; appended raw that reads as two competing variants, so
|
||
// Spread() — the disagreement column the owner reads at the stop — reports a conflict that does not exist,
|
||
// and the §C2-3 frequency factor scores the consensus on half its evidence.
|
||
func TestMergeFoldsIdenticalRenderingsOfOneEntity(t *testing.T) {
|
||
mined := []Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子"}}}
|
||
observed := []Observed{
|
||
{Key: "方源", Src: "方源", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 3}}},
|
||
{Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 2}, {Dst: "малец Фан", Chunks: 1}}},
|
||
}
|
||
var c Candidate
|
||
for _, got := range Merge(mined, observed) {
|
||
if got.Key == "方源" { // select by key: the alias surface is now a candidate of its own too
|
||
c = got
|
||
}
|
||
}
|
||
if c.Spread() != 2 {
|
||
t.Fatalf("one rendering from two surfaces is ONE variant (plus the genuinely different one): %+v", c.Variants)
|
||
}
|
||
if c.Variants[0].Dst != "Фан Юань" || c.Variants[0].Chunks != 5 {
|
||
t.Fatalf("identical renderings must pool their evidence: %+v", c.Variants)
|
||
}
|
||
}
|
||
|
||
// TestMergeKeepsAnAliasSwallowedSurface is the polygon's G-series address reproduced and closed at the
|
||
// input-assembly boundary. Their measurement: a surname cluster (葛家) held a clan title (族长) as an alias,
|
||
// and the title vanished from the delta over 50 chapters. Reproduced HERE by execution before the fix: the
|
||
// merge folded the observed 族长 into 葛家 and the surface disappeared from the terminologist's input
|
||
// entirely, while its rendering «глава клана» was re-labelled as a rendering OF the family name — so the
|
||
// role would have been asked to consolidate a surname into a title.
|
||
//
|
||
// The MINIMUM the addendum asks for is "do not lose such candidates when assembling the terminologist's
|
||
// input", and that is exactly the scope here: the surface stands on its own, the cluster still gets the
|
||
// evidence (ranking unchanged), the rendering carries its provenance, and Origin `alias` keeps the row OUT
|
||
// of the emitted delta — proposing it for signature would be the clustering fix, which is a separate call.
|
||
func TestMergeKeepsAnAliasSwallowedSurface(t *testing.T) {
|
||
mined := []Mined{{Key: "葛家", Src: "葛家", Type: "name", Freq: 40, Aliases: []string{"族长"}}}
|
||
observed := []Observed{{Key: "族长", Src: "族长", Type: "title", Proposals: []Proposal{{Dst: "глава клана", Chunks: 3}}}}
|
||
got := Merge(mined, observed)
|
||
|
||
var title, cluster *Candidate
|
||
for i := range got {
|
||
switch got[i].Key {
|
||
case "族长":
|
||
title = &got[i]
|
||
case "葛家":
|
||
cluster = &got[i]
|
||
}
|
||
}
|
||
if title == nil {
|
||
t.Fatalf("the swallowed surface must still reach the terminologist's input: %+v", got)
|
||
}
|
||
if title.Origin != OriginAlias {
|
||
t.Fatalf("it must be marked as claimed by a cluster (and so kept out of the delta), got %q", title.Origin)
|
||
}
|
||
if len(title.Related) != 1 || title.Related[0] != "葛家" {
|
||
t.Fatalf("the row must name the cluster that claims it: %+v", title.Related)
|
||
}
|
||
if title.Best() != "глава клана" {
|
||
t.Fatalf("it must carry its own renderings: %+v", title.Variants)
|
||
}
|
||
// The cluster keeps the evidence — ranking is unchanged — but the rendering is no longer presented as
|
||
// its own: a mis-clustered alias must not silently re-label a surname to a title.
|
||
if cluster == nil || len(cluster.Variants) != 1 || cluster.Variants[0].Via != "族长" {
|
||
t.Fatalf("the cluster must keep the evidence WITH its provenance: %+v", cluster)
|
||
}
|
||
if !strings.Contains(RenderBatch([]Candidate{*cluster}), "proposed for 族长") {
|
||
t.Fatalf("the provenance must be visible on the wire:\n%s", RenderBatch([]Candidate{*cluster}))
|
||
}
|
||
}
|
||
|
||
func TestMergeIsDeterministic(t *testing.T) {
|
||
mined := []Mined{{Key: "b", Src: "b"}, {Key: "a", Src: "a"}, {Key: "c", Src: "c"}}
|
||
observed := []Observed{
|
||
{Key: "z", Src: "z", Proposals: []Proposal{{Dst: "Z", Chunks: 1}}},
|
||
{Key: "a", Src: "a", Proposals: []Proposal{{Dst: "A", Chunks: 1}}},
|
||
}
|
||
first := keysOf(Merge(mined, observed))
|
||
for i := 0; i < 20; i++ { // map iteration would show up here, not on one run
|
||
if got := keysOf(Merge(mined, observed)); !reflect.DeepEqual(got, first) {
|
||
t.Fatalf("merge is order-unstable: %v vs %v", got, first)
|
||
}
|
||
}
|
||
}
|
||
|
||
func keysOf(cs []Candidate) []string {
|
||
out := make([]string, len(cs))
|
||
for i, c := range cs {
|
||
out[i] = c.Key
|
||
}
|
||
return out
|
||
}
|
||
|
||
func TestAttachKWICGivesSourceContextsInReadingOrder(t *testing.T) {
|
||
chunks := []Chunk{
|
||
{Chapter: 2, ChunkIdx: 0, NSource: "потом方源снова"},
|
||
{Chapter: 1, ChunkIdx: 0, NSource: "сначала方源тут и方源там"},
|
||
}
|
||
cands := AttachKWIC([]Candidate{{Key: "方源"}}, chunks, 3, 4)
|
||
got := cands[0].KWIC
|
||
if len(got) != 3 {
|
||
t.Fatalf("want 3 contexts, got %d (%v)", len(got), got)
|
||
}
|
||
// Chapter 1 comes first even though it is SECOND in the input slice, and its two hits precede
|
||
// chapter 2's.
|
||
if !strings.HasSuffix(got[0], "тут") || !strings.HasSuffix(got[1], "там") || !strings.HasPrefix(got[2], "отом") {
|
||
t.Fatalf("contexts must be in (chapter, chunk, offset) order, got %v", got)
|
||
}
|
||
// A term the source does not contain gets an HONESTLY empty list — that emptiness is the signal that
|
||
// the draft invented it.
|
||
if k := AttachKWIC([]Candidate{{Key: "нетутакого"}}, chunks, 3, 4)[0].KWIC; len(k) != 0 {
|
||
t.Fatalf("an absent term must have no contexts, got %v", k)
|
||
}
|
||
}
|
||
|
||
func TestScoreVariantsIsNotMajority(t *testing.T) {
|
||
// The frequent rendering is a semantic guess; the rare one is a proper transliteration. §C2-3 says the
|
||
// canon is chosen over ALL the evidence, not by counting — so conformance must be able to beat count.
|
||
c := Candidate{Key: "方源", Src: "方源", Type: "name", Variants: []Variant{
|
||
{Dst: "Источник", Chunks: 5},
|
||
{Dst: "Фан Юань", Chunks: 1},
|
||
}}
|
||
ScoreVariants(&c, ScoreOpts{Conformance: func(dst, typ string) float64 {
|
||
if dst == "Фан Юань" {
|
||
return 1
|
||
}
|
||
return 0
|
||
}})
|
||
if c.Best() != "Фан Юань" {
|
||
t.Fatalf("plain majority won: %+v", c.Variants)
|
||
}
|
||
// …and with NO other signal, frequency does decide (the factor is real, just not sovereign).
|
||
c2 := Candidate{Key: "x", Src: "x", Variants: []Variant{{Dst: "Редкий", Chunks: 1}, {Dst: "Частый", Chunks: 5}}}
|
||
ScoreVariants(&c2, ScoreOpts{})
|
||
if c2.Best() != "Частый" {
|
||
t.Fatalf("with no other evidence the frequent rendering should lead: %+v", c2.Variants)
|
||
}
|
||
}
|
||
|
||
func TestScoreVariantsNeighbourAnchorAndMalformed(t *testing.T) {
|
||
// A rendering that agrees with an APPROVED sibling of the same source series outranks a MORE FREQUENT
|
||
// one that does not. The competing rendering deliberately shares no lexeme with the signed sibling and
|
||
// leads on count — otherwise both variants take the bonus, the winner falls to the alphabetical
|
||
// tie-break, and the assertion pins nothing (audit of 26.07: that is exactly what this test used to do).
|
||
c := Candidate{Key: "元火", Src: "元火", Type: "term", Variants: []Variant{
|
||
{Dst: "пламя духа", Chunks: 2},
|
||
{Dst: "изначальный огонь", Chunks: 1},
|
||
}}
|
||
ns := []Neighbour{{Src: "元水", Dst: "изначальная вода"}}
|
||
ScoreVariants(&c, ScoreOpts{Neighbours: ns})
|
||
if c.Best() != "изначальный огонь" {
|
||
t.Fatalf("agreement with a signed sibling must beat a more frequent stranger: %+v", c.Variants)
|
||
}
|
||
// …and the factor is what did it: with no signed siblings the frequent rendering leads.
|
||
bare := Candidate{Key: "元火", Src: "元火", Type: "term", Variants: []Variant{
|
||
{Dst: "пламя духа", Chunks: 2},
|
||
{Dst: "изначальный огонь", Chunks: 1},
|
||
}}
|
||
ScoreVariants(&bare, ScoreOpts{})
|
||
if bare.Best() != "пламя духа" {
|
||
t.Fatalf("without the neighbour anchor the count must decide, else the test proves nothing: %+v", bare.Variants)
|
||
}
|
||
// D39.47: there is no pair/genre factor to test beside the neighbour one. The formula's ONLY consistency
|
||
// anchor is the signed bank, and a rendering nobody signed is decided by count and the pair's
|
||
// transliteration table — not by an industry default the market does not actually have. The competing
|
||
// «совершенствующийся»/«практик культивации» case that used to live here is now, correctly, a count
|
||
// decision until the owner signs one of them.
|
||
g := Candidate{Key: "修炼者", Src: "修炼者", Type: "term", Variants: []Variant{
|
||
{Dst: "совершенствующийся", Chunks: 3},
|
||
{Dst: "практик культивации", Chunks: 1},
|
||
}}
|
||
ScoreVariants(&g, ScoreOpts{})
|
||
if g.Best() != "совершенствующийся" {
|
||
t.Fatalf("with nothing signed the count decides — no register is imposed on the book: %+v", g.Variants)
|
||
}
|
||
for _, v := range g.Variants {
|
||
if contains(v.Signals, "genre") {
|
||
t.Fatalf("the genre factor is removed (D39.47); a signal naming it means it came back: %+v", v)
|
||
}
|
||
}
|
||
// A SHORT word that merely opens a longer one is not the same lexeme: «гу» (蛊, the book's own term)
|
||
// against «Гуюэ» (a clan), «ад» against «адрес». Without the floor the anchor factors fire on
|
||
// renderings that have nothing to do with the signed row, and their signal string says they legitimately did.
|
||
short := Candidate{Key: "蛊虫", Src: "蛊虫", Type: "term", Variants: []Variant{
|
||
{Dst: "червь гуюэ", Chunks: 1},
|
||
{Dst: "гу-червь", Chunks: 1},
|
||
}}
|
||
ScoreVariants(&short, ScoreOpts{Neighbours: []Neighbour{{Src: "蛊师", Dst: "гу"}}})
|
||
for _, v := range short.Variants {
|
||
if v.Dst == "червь гуюэ" && contains(v.Signals, "neighbour") {
|
||
t.Fatalf("a 2-letter prefix must not count as a shared lexeme: %+v", v)
|
||
}
|
||
}
|
||
// A visibly mangled form is penalised and says so.
|
||
m := Candidate{Key: "y", Src: "y", Variants: []Variant{{Dst: "Хорошо", Chunks: 1}, {Dst: "Плохо-", Chunks: 5}}}
|
||
ScoreVariants(&m, ScoreOpts{})
|
||
if m.Best() != "Хорошо" {
|
||
t.Fatalf("a mangled form must not win on count alone: %+v", m.Variants)
|
||
}
|
||
if !contains(m.Variants[1].Signals, "malformed") {
|
||
t.Fatalf("the penalty must be visible in the signals: %+v", m.Variants[1])
|
||
}
|
||
}
|
||
|
||
// TestCanonForRanksContainmentFirstAndCaps pins the anchor selection: the signed rows a batch actually
|
||
// needs, strongest relation first, bounded. Containment is ranked above a shared character because it is
|
||
// the case that failed live — 元海 inside 元海空窍, where ignoring the signature produced a rendering that
|
||
// contradicted two approved rows at once.
|
||
func TestCanonForRanksContainmentFirstAndCaps(t *testing.T) {
|
||
batch := []Candidate{{Key: "元海空窍", Src: "元海空窍"}}
|
||
ns := []Neighbour{
|
||
{Src: "元火", Dst: "изначальный огонь"}, // shares 元 only
|
||
{Src: "元海", Dst: "море истинной ци"}, // CONTAINED in the candidate
|
||
{Src: "花家", Dst: "клан Хуа"}, // unrelated
|
||
}
|
||
got := CanonFor(batch, ns, 10)
|
||
if len(got) != 2 {
|
||
t.Fatalf("only the related rows belong in the anchor, got %v", got)
|
||
}
|
||
if got[0][0] != "元海" {
|
||
t.Fatalf("containment must outrank a bare shared character: %v", got)
|
||
}
|
||
// The cap is real: an anchor that grows with the bank would eventually cost more than the batch.
|
||
if capped := CanonFor(batch, ns, 1); len(capped) != 1 || capped[0][0] != "元海" {
|
||
t.Fatalf("cap must keep the STRONGEST relation, got %v", capped)
|
||
}
|
||
// A candidate that IS the signed row learns nothing from itself.
|
||
if self := CanonFor([]Candidate{{Key: "元海", Src: "元海"}}, []Neighbour{{Src: "元海", Dst: "море истинной ци"}}, 10); len(self) != 0 {
|
||
t.Fatalf("a row must not anchor itself: %v", self)
|
||
}
|
||
// Nothing signed → no block at all, so a book with no canon takes the pre-existing path byte for byte.
|
||
if a := RenderCanonAnchor(nil); a != "" {
|
||
t.Fatalf("an empty anchor must render to nothing, got %q", a)
|
||
}
|
||
// One marker, one block, and the signed rows verbatim — D39.47 left exactly one anchor, so the rendering
|
||
// is fully pinned here rather than probed for a prefix.
|
||
if a := RenderCanonAnchor([][2]string{{"元海", "море истинной ци"}, {"空窍", "апертура"}}); a != CanonMarker+"\n元海\tморе истинной ци\n空窍\tапертура" {
|
||
t.Fatalf("canon anchor rendering = %q", a)
|
||
}
|
||
}
|
||
|
||
// TestCanonConflictsFlagsOnlyRealContradictions pins the deterministic half: the anchor is a nudge, this
|
||
// is the check. Live on 26.07 the role rendered 一代族长 as «Первый глава рода» with 族长 signed «глава
|
||
// клана» — a contradiction with no signal anywhere. It must flag that, and must NOT flag a rendering that
|
||
// carries the signed word in another case (Russian inflection is not a contradiction).
|
||
func TestCanonConflictsFlagsOnlyRealContradictions(t *testing.T) {
|
||
cands := []Candidate{
|
||
{Key: "一代族长", Src: "一代族长"},
|
||
{Key: "四代族长", Src: "四代族长"},
|
||
{Key: "花家", Src: "花家"},
|
||
}
|
||
ns := []Neighbour{{Src: "族长", Dst: "глава клана"}}
|
||
got := CanonConflicts(cands, map[string]string{
|
||
"一代族长": "Первый глава рода", // contradicts: no lexeme of the signed rendering
|
||
"四代族长": "Четвёртый глава клана", // agrees, in another form
|
||
"花家": "клан Хуа", // contains no signed source at all
|
||
}, ns)
|
||
if len(got) != 1 || got[0].Src != "一代族长" || got[0].CanonSrc != "族长" {
|
||
t.Fatalf("exactly the contradicting rendering must be flagged, got %+v", got)
|
||
}
|
||
// Nothing signed, or nothing consolidated → nothing to say.
|
||
if c := CanonConflicts(cands, map[string]string{"一代族长": "Первый глава рода"}, nil); len(c) != 0 {
|
||
t.Fatalf("with no signed bank there is no canon to contradict: %+v", c)
|
||
}
|
||
// The live probe's other half: a compound that DOES carry the signature, in an oblique case, must not
|
||
// be flagged. «море истинной ци» signed, «апертура моря истинной ци» consolidated — at a flat
|
||
// five-character stem test «море»/«моря» failed to match and this false-flagged.
|
||
ok := CanonConflicts(
|
||
[]Candidate{{Key: "元海空窍", Src: "元海空窍"}},
|
||
map[string]string{"元海空窍": "апертура моря истинной ци"},
|
||
[]Neighbour{{Src: "元海", Dst: "море истинной ци"}})
|
||
if len(ok) != 0 {
|
||
t.Fatalf("a compound carrying the signed rendering in another case is not a contradiction: %+v", ok)
|
||
}
|
||
}
|
||
|
||
// TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds pins two silent-loss holes the audit surfaced: a
|
||
// stray double space inside a rendering must not truncate it (the splitter is tolerant BY DESIGN, and that
|
||
// tolerance cuts both ways), and a batch must actually render within the budget it was split for.
|
||
func TestParseReplyKeepsAWholeRenderingAndBatchBudgetHolds(t *testing.T) {
|
||
got, st := ParseReply("方源\tФан Юань\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
|
||
if st.Bad != 0 || got["方源"] != "Фан Юань" {
|
||
t.Fatalf("a double space inside a rendering must not bank half a name: %v (bad=%d)", got, st.Bad)
|
||
}
|
||
// A rendering that IS the source is not a rendering: a model echoing the term back would otherwise be
|
||
// banked as this book's canon and shown to the editor as «方源 ⟨проверить⟩».
|
||
echo, st := ParseReply("方源\t方源\n花家\tДом Хуа", []string{"方源", "花家"}, func(s string) string { return s }, nil)
|
||
if _, banked := echo["方源"]; banked || st.Bad != 1 {
|
||
t.Fatalf("an echoed source must be refused and counted: %v (bad=%d)", echo, st.Bad)
|
||
}
|
||
cands := make([]Candidate, 200)
|
||
for i := range cands {
|
||
cands[i] = Candidate{Key: "k", Src: "s", Type: "term"}
|
||
}
|
||
for i, b := range Batch(cands, 600, nil) {
|
||
if n := len([]rune(RenderBatch(b))); n > 600 {
|
||
t.Fatalf("batch %d renders %d runes over a 600-rune budget", i, n)
|
||
}
|
||
}
|
||
}
|
||
|
||
func contains(ss []string, want string) bool {
|
||
for _, s := range ss {
|
||
if s == want {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func TestParseReplyAcceptsOnlyAskedTerms(t *testing.T) {
|
||
id := func(s string) string { return s }
|
||
reply := strings.Join([]string{
|
||
"方源\tФан Юань",
|
||
"花家 Дом Хуа", // two spaces instead of a tab
|
||
"青茅山 | гора Цинмао",
|
||
"чужой\tЧужой", // never asked about → must not enter the bank
|
||
"忘却\t" + NoDst,
|
||
"мусор",
|
||
}, "\n")
|
||
got, st := ParseReply(reply, []string{"方源", "花家", "青茅山", "忘却"}, id, nil)
|
||
want := map[string]string{"方源": "Фан Юань", "花家": "Дом Хуа", "青茅山": "гора Цинмао", "忘却": ""}
|
||
if !reflect.DeepEqual(got, want) {
|
||
t.Fatalf("parse = %#v, want %#v", got, want)
|
||
}
|
||
if st.Bad != 2 {
|
||
t.Fatalf("the unusable lines must be COUNTED (silence about them is how a paid call turns into an empty bank), got %d", st.Bad)
|
||
}
|
||
// The declined term is present with an EMPTY rendering — a decision, distinct from "no line at all".
|
||
if v, ok := got["忘却"]; !ok || v != "" {
|
||
t.Fatalf("the decline sentinel must map to an explicit empty rendering, got %q/%v", v, ok)
|
||
}
|
||
}
|
||
|
||
func TestParseReplyNormalizesTheKey(t *testing.T) {
|
||
// The caller passes the engine's source-key normalizer; a reply written in another orthography must
|
||
// still land on its term.
|
||
lower := strings.ToLower
|
||
got, _ := ParseReply("FANG\tФан", []string{"fang"}, lower, nil)
|
||
if got["fang"] != "Фан" {
|
||
t.Fatalf("the reply key must be normalized by the caller's function, got %#v", got)
|
||
}
|
||
}
|
||
|
||
func TestBatchCoversEveryCandidateExactlyOnce(t *testing.T) {
|
||
var cands []Candidate
|
||
for _, k := range []string{"a", "b", "c", "d", "e"} {
|
||
cands = append(cands, Candidate{Key: k, Src: k, KWIC: []string{strings.Repeat("к", 200)}})
|
||
}
|
||
batches := Batch(cands, 400, nil)
|
||
if len(batches) < 2 {
|
||
t.Fatalf("the fixture must actually split, got %d batch(es)", len(batches))
|
||
}
|
||
seen := map[string]int{}
|
||
for _, b := range batches {
|
||
for _, c := range b {
|
||
seen[c.Key]++
|
||
}
|
||
}
|
||
if len(seen) != len(cands) {
|
||
t.Fatalf("batching dropped candidates: %v", seen)
|
||
}
|
||
for k, n := range seen {
|
||
if n != 1 {
|
||
t.Fatalf("candidate %q appears in %d batches", k, n)
|
||
}
|
||
}
|
||
// A single oversized candidate is never silently dropped.
|
||
big := []Candidate{{Key: "big", Src: "big", KWIC: []string{strings.Repeat("ю", 5000)}}}
|
||
if got := Batch(big, 10, nil); len(got) != 1 || len(got[0]) != 1 {
|
||
t.Fatalf("an oversized candidate must still get its own batch, got %v", got)
|
||
}
|
||
}
|
||
|
||
func TestRenderBatchIsDeterministicAndCarriesTheEvidence(t *testing.T) {
|
||
c := Candidate{
|
||
Key: "方源", Src: "方源", Type: "name", Freq: 12, SinceCh: 1, Origin: OriginBoth,
|
||
Aliases: []string{"方小子"}, Related: []string{"古月方源"}, Evidence: []string{"surname:方"},
|
||
Variants: []Variant{{Dst: "Фан Юань", Chunks: 3}, {Dst: "Фань Юань", Chunks: 1}},
|
||
KWIC: []string{"方源来到"},
|
||
}
|
||
first := RenderBatch([]Candidate{c})
|
||
for i := 0; i < 10; i++ {
|
||
if RenderBatch([]Candidate{c}) != first {
|
||
t.Fatal("the rendered block must be byte-stable across renders")
|
||
}
|
||
}
|
||
for _, want := range []string{"方源", "type: name", "origin: both", "freq: 12", "aliases: 方小子",
|
||
"related: 古月方源", "drafts: Фан Юань ×3 | Фань Юань ×1", "ctx: 方源来到"} {
|
||
if !strings.Contains(first, want) {
|
||
t.Fatalf("the block must carry %q:\n%s", want, first)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestAttachKWICCountsOccurrencesUncapped: the KWIC list is CAPPED, the frequency is not. Using the list
|
||
// length as the count would print "freq: 3" for a term that occurs forty times — in exactly the column
|
||
// the owner reads to judge whether a term is worth signing.
|
||
func TestAttachKWICCountsOccurrencesUncapped(t *testing.T) {
|
||
chunks := []Chunk{{Chapter: 1, NSource: strings.Repeat("方源и", 40)}}
|
||
got := AttachKWIC([]Candidate{{Key: "方源"}}, chunks, 3, 4)
|
||
if len(got[0].KWIC) != 3 {
|
||
t.Fatalf("the KWIC list must stay capped, got %d", len(got[0].KWIC))
|
||
}
|
||
if got[0].Freq != 40 {
|
||
t.Fatalf("the frequency must be the real occurrence count, not the capped list length: %d", got[0].Freq)
|
||
}
|
||
// A detector-supplied frequency is authoritative and must not be overwritten.
|
||
kept := AttachKWIC([]Candidate{{Key: "方源", Freq: 7}}, chunks, 3, 4)
|
||
if kept[0].Freq != 7 {
|
||
t.Fatalf("a mined candidate's own frequency must survive, got %d", kept[0].Freq)
|
||
}
|
||
}
|