215 lines
10 KiB
Go
215 lines
10 KiB
Go
package terminology
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// arbitration_test.go: the §G2/§G3/§G5 half of the fix-pack — the run reading its OWN output (self
|
||
// contradictions), the confidence field, and the vote counted per convention instead of per byte string.
|
||
|
||
// lowerFold is a stand-in for the target-form normalizer the pipeline injects: enough of it (case and
|
||
// whitespace) to exercise the fold without importing a target's morphology into this package.
|
||
func lowerFold(s string) string { return strings.Join(strings.Fields(strings.ToLower(s)), " ") }
|
||
|
||
// TestConsolidationConflictsCatchesTheRunContradictingItself is the §G2 case measured on the live bank: the
|
||
// same reply consolidated a part and then rendered the compound without it. The canon check cannot see this
|
||
// — nothing here is signed — and until this pack nothing else looked either.
|
||
func TestConsolidationConflictsCatchesTheRunContradictingItself(t *testing.T) {
|
||
cands := []Candidate{
|
||
{Key: "元海", Src: "元海"},
|
||
{Key: "元海空窍", Src: "元海空窍"},
|
||
{Key: "空窍", Src: "空窍"},
|
||
{Key: "青茅山", Src: "青茅山"},
|
||
}
|
||
out := map[string]string{
|
||
"元海": "море истинной ци",
|
||
"空窍": "апертура",
|
||
"元海空窍": "апертура Первозданного моря", // carries «апертура», DROPS «истинной ци»
|
||
"青茅山": "гора Цинмао",
|
||
}
|
||
got := ConsolidationConflicts(cands, out)
|
||
if len(got) != 1 {
|
||
t.Fatalf("exactly one contradiction (the compound against 元海), got %+v", got)
|
||
}
|
||
if got[0].Src != "元海空窍" || got[0].PartSrc != "元海" {
|
||
t.Fatalf("the conflict must name the compound and the part it dropped: %+v", got[0])
|
||
}
|
||
// A compound that DOES carry both parts is not a conflict, case endings and all.
|
||
out["元海空窍"] = "апертура моря истинной ци"
|
||
if got := ConsolidationConflicts(cands, out); len(got) != 0 {
|
||
t.Fatalf("a rendering carrying every part is consistent, got %+v", got)
|
||
}
|
||
// And a term nobody consolidated cannot contradict anything (silence is not a decision).
|
||
delete(out, "元海")
|
||
out["元海空窍"] = "апертура Первозданного моря"
|
||
if got := ConsolidationConflicts(cands, out); len(got) != 0 {
|
||
t.Fatalf("an unconsolidated part is not a contradiction, got %+v", got)
|
||
}
|
||
}
|
||
|
||
// TestParseReplyTakesTheConfidenceFieldBeforeRejoining is the §G3 parser trap, both directions: a THIRD
|
||
// field of digits is the role's confidence and must come off before the rendering is re-joined, while a
|
||
// rendering that merely ENDS in a number is one field and must survive whole.
|
||
func TestParseReplyTakesTheConfidenceFieldBeforeRejoining(t *testing.T) {
|
||
id := func(s string) string { return s }
|
||
keys := []string{"方源", "花家", "一零八峰", "忘却"}
|
||
reply := strings.Join([]string{
|
||
"方源\tФан Юань\t95",
|
||
"花家\tДом Хуа\t40", // a stray double space AND a confidence: both handled, or the name is halved
|
||
"一零八峰\tПик 108", // a rendering ending in a number, single space → one field, untouched
|
||
"忘却\tзабвение\t900", // a declared column that is not a confidence: dropped and counted, never glued
|
||
}, "\n")
|
||
got, conf, st := ParseReply(reply, keys, id, nil, nil)
|
||
if got["方源"] != "Фан Юань" || conf["方源"] != 95 {
|
||
t.Fatalf("want the rendering without the confidence and the confidence beside it: %q / %d", got["方源"], conf["方源"])
|
||
}
|
||
if got["花家"] != "Дом Хуа" || conf["花家"] != 40 {
|
||
t.Fatalf("a split rendering must be re-joined AFTER the confidence comes off: %q / %d", got["花家"], conf["花家"])
|
||
}
|
||
if got["一零八峰"] != "Пик 108" {
|
||
t.Fatalf("a rendering that legitimately ends in a number must survive whole, got %q", got["一零八峰"])
|
||
}
|
||
if _, has := conf["一零八峰"]; has {
|
||
t.Fatalf("there was no confidence field on that line: %v", conf)
|
||
}
|
||
if got["忘却"] != "забвение" || st.BadConfidence != 1 {
|
||
t.Fatalf("an unreadable confidence column is counted, never glued onto the rendering: %q / %d", got["忘却"], st.BadConfidence)
|
||
}
|
||
if st.Bad != 0 {
|
||
t.Fatalf("a bad confidence does not refuse the rendering, got %d bad", st.Bad)
|
||
}
|
||
// The whole class the third column opened: anything the model writes there that is not a bare 0..100 —
|
||
// «95%», «0.9», «высокая» — used to ride into the rendering, pass wellFormedLemma, pass the answer-language
|
||
// screen, and enter the bank as this book's canon with bad_lines reading 0.
|
||
for _, tail := range []string{"95%", "95 %", "0.9", "~90", "высокая"} {
|
||
g, c, s2 := ParseReply("师父\tнаставник\t"+tail, []string{"师父"}, id, nil, nil)
|
||
if g["师父"] != "наставник" {
|
||
t.Fatalf("tail %q was glued onto the rendering: %q", tail, g["师父"])
|
||
}
|
||
if _, has := c["师父"]; has {
|
||
t.Fatalf("tail %q must not be read as a confidence: %v", tail, c)
|
||
}
|
||
if s2.BadConfidence != 1 {
|
||
t.Fatalf("tail %q must be COUNTED, or the prompt's third column can fail silently: %+v", tail, s2)
|
||
}
|
||
}
|
||
// THE SPACE-RUN PATH, which every case above misses: all of them are TAB-delimited, so the confidence is
|
||
// stripped by the positional column reader and the space-run branch is never executed. Its own stripping
|
||
// could be removed without a single test noticing — and then a model that answers with spaces instead of
|
||
// tabs (the reason that tolerance exists at all) banks «Фан Юань 85» as this book's canon.
|
||
spaced, sconf, sst := ParseReply("方源 Фан Юань 85", []string{"方源"}, id, nil, nil)
|
||
if spaced["方源"] != "Фан Юань" {
|
||
t.Fatalf("a space-delimited line must lose its confidence field, not bank it: %q", spaced["方源"])
|
||
}
|
||
if sconf["方源"] != 85 {
|
||
t.Fatalf("and the confidence must be read: %v", sconf)
|
||
}
|
||
if sst.Bad != 0 {
|
||
t.Fatalf("the line is usable, got %d bad", sst.Bad)
|
||
}
|
||
// A two-field reply — the shape every existing pair prompt asks for — is unchanged and carries no
|
||
// confidence at all, so the field is additive rather than a new requirement.
|
||
plain, conf2, _ := ParseReply("方源\tФан Юань", []string{"方源"}, id, nil, nil)
|
||
if plain["方源"] != "Фан Юань" || len(conf2) != 0 {
|
||
t.Fatalf("a two-field reply must parse as before with no confidence: %q / %v", plain["方源"], conf2)
|
||
}
|
||
}
|
||
|
||
// TestParseReplyAcceptsAMangledDeclineSentinel: declining is a DECISION (§C2-7), and a decline the parser
|
||
// refuses is counted as a broken line instead — the term then reads as "the model produced garbage" rather
|
||
// than "the model said it could not tell".
|
||
func TestParseReplyAcceptsAMangledDeclineSentinel(t *testing.T) {
|
||
id := func(s string) string { return s }
|
||
keys := []string{"方源", "花家", "青茅山"}
|
||
reply := "方源\t" + NoDst + "\n花家\t«" + NoDst + "»\n青茅山\t" + NoDst + "."
|
||
got, _, st := ParseReply(reply, keys, id, nil, nil)
|
||
for _, k := range keys {
|
||
v, answered := got[k]
|
||
if !answered || v != "" {
|
||
t.Fatalf("%s must read as an explicit decline, got %q (answered=%v)", k, v, answered)
|
||
}
|
||
}
|
||
if st.Bad != 0 {
|
||
t.Fatalf("a decline is not a parse failure, got %d bad lines", st.Bad)
|
||
}
|
||
// Narrow on purpose: a rendering that merely MENTIONS something is not a decline.
|
||
other, _, _ := ParseReply("方源\tвариант "+NoDst, []string{"方源"}, id, nil, nil)
|
||
if other["方源"] == "" {
|
||
t.Fatal("only a rendering that IS the sentinel declines; a prefix match would swallow real answers")
|
||
}
|
||
}
|
||
|
||
// TestFoldVariantsCountsConventionsAndShowsRawForms is §G5. Before it, «Море истинной ци» and «море
|
||
// истинной ци» were two variants: the §C2-3 frequency factor scored the consensus on half its evidence, and
|
||
// the spread column reported a contest where there was a spelling difference.
|
||
func TestFoldVariantsCountsConventionsAndShowsRawForms(t *testing.T) {
|
||
observed := []Observed{{Key: "元海", Src: "元海", Proposals: []Proposal{
|
||
{Dst: "море истинной ци", Chunks: 3},
|
||
{Dst: "Море истинной ци", Chunks: 4}, // same convention, different case
|
||
{Dst: "Первозданное море", Chunks: 5},
|
||
}}}
|
||
got := Merge(nil, observed, lowerFold)
|
||
if len(got) != 1 {
|
||
t.Fatalf("one surface, one candidate: %+v", got)
|
||
}
|
||
c := got[0]
|
||
if c.Conventions() != 2 {
|
||
t.Fatalf("two decisions were made, not three: %+v", c.Variants)
|
||
}
|
||
if c.Spread() != 3 {
|
||
t.Fatalf("the drafts still wrote it three ways and the owner must see that: spread=%d", c.Spread())
|
||
}
|
||
var folded *Variant
|
||
for i := range c.Variants {
|
||
if strings.EqualFold(c.Variants[i].Dst, "море истинной ци") {
|
||
folded = &c.Variants[i]
|
||
}
|
||
}
|
||
if folded == nil {
|
||
t.Fatalf("the folded class must be present as a RAW form, not a normalized one: %+v", c.Variants)
|
||
}
|
||
if folded.Chunks != 7 {
|
||
t.Fatalf("the vote is the sum of the class, got %d", folded.Chunks)
|
||
}
|
||
if folded.Dst != "Море истинной ци" {
|
||
t.Fatalf("the representative is the most-proposed RAW form, got %q", folded.Dst)
|
||
}
|
||
// And the fold decides the ranking: 7 folded chunks now outweigh the 5 that used to win on 4-vs-5.
|
||
ScoreVariants(&c, ScoreOpts{})
|
||
if !strings.EqualFold(c.Best(), "море истинной ци") {
|
||
t.Fatalf("the consensus must stop losing to its own spelling variant, got %q", c.Best())
|
||
}
|
||
}
|
||
|
||
// TestFoldVariantsKeepsViaProvenanceHonest: a rendering that ALSO arrived on the candidate's own key is
|
||
// direct. First-wins used to label such a consensus «proposed for <alias>» purely because the alias row was
|
||
// seen first — presenting an agreed rendering as a mis-clustered alias's guess.
|
||
func TestFoldVariantsKeepsViaProvenanceHonest(t *testing.T) {
|
||
mined := []Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子", "小方"}}}
|
||
observed := []Observed{
|
||
{Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 1}}}, // through an alias, FIRST
|
||
{Key: "方源", Src: "方源", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 6}}}, // and directly
|
||
{Key: "小方", Src: "小方", Proposals: []Proposal{{Dst: "малыш Фан", Chunks: 2}}}, // alias only
|
||
}
|
||
for _, c := range Merge(mined, observed, lowerFold) {
|
||
if c.Key != "方源" {
|
||
continue
|
||
}
|
||
for _, v := range c.Variants {
|
||
switch v.Dst {
|
||
case "Фан Юань":
|
||
if v.Via != "" {
|
||
t.Fatalf("a rendering also proposed on the term's own key is DIRECT, got via=%q", v.Via)
|
||
}
|
||
if v.Chunks != 7 {
|
||
t.Fatalf("the alias vote still counts toward the total, got %d", v.Chunks)
|
||
}
|
||
case "малыш Фан":
|
||
if v.Via != "小方" {
|
||
t.Fatalf("a rendering that ONLY ever came through an alias must keep saying so, got %q", v.Via)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|