package terminology import ( "strings" "testing" ) // arbitration_test.go: the §G2/§G3/§G5 half of the fix-pack — the run reading its OWN output (self // contradictions), the confidence field, and the vote counted per convention instead of per byte string. // lowerFold is a stand-in for the target-form normalizer the pipeline injects: enough of it (case and // whitespace) to exercise the fold without importing a target's morphology into this package. func lowerFold(s string) string { return strings.Join(strings.Fields(strings.ToLower(s)), " ") } // TestConsolidationConflictsCatchesTheRunContradictingItself is the §G2 case measured on the live bank: the // same reply consolidated a part and then rendered the compound without it. The canon check cannot see this // — nothing here is signed — and until this pack nothing else looked either. func TestConsolidationConflictsCatchesTheRunContradictingItself(t *testing.T) { cands := []Candidate{ {Key: "元海", Src: "元海"}, {Key: "元海空窍", Src: "元海空窍"}, {Key: "空窍", Src: "空窍"}, {Key: "青茅山", Src: "青茅山"}, } out := map[string]string{ "元海": "море истинной ци", "空窍": "апертура", "元海空窍": "апертура Первозданного моря", // carries «апертура», DROPS «истинной ци» "青茅山": "гора Цинмао", } got := ConsolidationConflicts(cands, out) if len(got) != 1 { t.Fatalf("exactly one contradiction (the compound against 元海), got %+v", got) } if got[0].Src != "元海空窍" || got[0].PartSrc != "元海" { t.Fatalf("the conflict must name the compound and the part it dropped: %+v", got[0]) } // A compound that DOES carry both parts is not a conflict, case endings and all. out["元海空窍"] = "апертура моря истинной ци" if got := ConsolidationConflicts(cands, out); len(got) != 0 { t.Fatalf("a rendering carrying every part is consistent, got %+v", got) } // And a term nobody consolidated cannot contradict anything (silence is not a decision). delete(out, "元海") out["元海空窍"] = "апертура Первозданного моря" if got := ConsolidationConflicts(cands, out); len(got) != 0 { t.Fatalf("an unconsolidated part is not a contradiction, got %+v", got) } } // TestParseReplyTakesTheConfidenceFieldBeforeRejoining is the §G3 parser trap, both directions: a THIRD // field of digits is the role's confidence and must come off before the rendering is re-joined, while a // rendering that merely ENDS in a number is one field and must survive whole. func TestParseReplyTakesTheConfidenceFieldBeforeRejoining(t *testing.T) { id := func(s string) string { return s } keys := []string{"方源", "花家", "一零八峰", "忘却"} reply := strings.Join([]string{ "方源\tФан Юань\t95", "花家\tДом Хуа\t40", // a stray double space AND a confidence: both handled, or the name is halved "一零八峰\tПик 108", // a rendering ending in a number, single space → one field, untouched "忘却\tзабвение\t900", // a declared column that is not a confidence: dropped and counted, never glued }, "\n") got, conf, st := ParseReply(reply, keys, id, nil) if got["方源"] != "Фан Юань" || conf["方源"] != 95 { t.Fatalf("want the rendering without the confidence and the confidence beside it: %q / %d", got["方源"], conf["方源"]) } if got["花家"] != "Дом Хуа" || conf["花家"] != 40 { t.Fatalf("a split rendering must be re-joined AFTER the confidence comes off: %q / %d", got["花家"], conf["花家"]) } if got["一零八峰"] != "Пик 108" { t.Fatalf("a rendering that legitimately ends in a number must survive whole, got %q", got["一零八峰"]) } if _, has := conf["一零八峰"]; has { t.Fatalf("there was no confidence field on that line: %v", conf) } if got["忘却"] != "забвение" || st.BadConfidence != 1 { t.Fatalf("an unreadable confidence column is counted, never glued onto the rendering: %q / %d", got["忘却"], st.BadConfidence) } if st.Bad != 0 { t.Fatalf("a bad confidence does not refuse the rendering, got %d bad", st.Bad) } // The whole class the third column opened: anything the model writes there that is not a bare 0..100 — // «95%», «0.9», «высокая» — used to ride into the rendering, pass wellFormedLemma, pass the answer-language // screen, and enter the bank as this book's canon with bad_lines reading 0. for _, tail := range []string{"95%", "95 %", "0.9", "~90", "высокая"} { g, c, s2 := ParseReply("师父\tнаставник\t"+tail, []string{"师父"}, id, nil) if g["师父"] != "наставник" { t.Fatalf("tail %q was glued onto the rendering: %q", tail, g["师父"]) } if _, has := c["师父"]; has { t.Fatalf("tail %q must not be read as a confidence: %v", tail, c) } if s2.BadConfidence != 1 { t.Fatalf("tail %q must be COUNTED, or the prompt's third column can fail silently: %+v", tail, s2) } } // THE SPACE-RUN PATH, which every case above misses: all of them are TAB-delimited, so the confidence is // stripped by the positional column reader and the space-run branch is never executed. Its own stripping // could be removed without a single test noticing — and then a model that answers with spaces instead of // tabs (the reason that tolerance exists at all) banks «Фан Юань 85» as this book's canon. spaced, sconf, sst := ParseReply("方源 Фан Юань 85", []string{"方源"}, id, nil) if spaced["方源"] != "Фан Юань" { t.Fatalf("a space-delimited line must lose its confidence field, not bank it: %q", spaced["方源"]) } if sconf["方源"] != 85 { t.Fatalf("and the confidence must be read: %v", sconf) } if sst.Bad != 0 { t.Fatalf("the line is usable, got %d bad", sst.Bad) } // A two-field reply — the shape every existing pair prompt asks for — is unchanged and carries no // confidence at all, so the field is additive rather than a new requirement. plain, conf2, _ := ParseReply("方源\tФан Юань", []string{"方源"}, id, nil) if plain["方源"] != "Фан Юань" || len(conf2) != 0 { t.Fatalf("a two-field reply must parse as before with no confidence: %q / %v", plain["方源"], conf2) } } // TestParseReplyAcceptsAMangledDeclineSentinel: declining is a DECISION (§C2-7), and a decline the parser // refuses is counted as a broken line instead — the term then reads as "the model produced garbage" rather // than "the model said it could not tell". func TestParseReplyAcceptsAMangledDeclineSentinel(t *testing.T) { id := func(s string) string { return s } keys := []string{"方源", "花家", "青茅山"} reply := "方源\t" + NoDst + "\n花家\t«" + NoDst + "»\n青茅山\t" + NoDst + "." got, _, st := ParseReply(reply, keys, id, nil) for _, k := range keys { v, answered := got[k] if !answered || v != "" { t.Fatalf("%s must read as an explicit decline, got %q (answered=%v)", k, v, answered) } } if st.Bad != 0 { t.Fatalf("a decline is not a parse failure, got %d bad lines", st.Bad) } // Narrow on purpose: a rendering that merely MENTIONS something is not a decline. other, _, _ := ParseReply("方源\tвариант "+NoDst, []string{"方源"}, id, nil) if other["方源"] == "" { t.Fatal("only a rendering that IS the sentinel declines; a prefix match would swallow real answers") } } // TestFoldVariantsCountsConventionsAndShowsRawForms is §G5. Before it, «Море истинной ци» and «море // истинной ци» were two variants: the §C2-3 frequency factor scored the consensus on half its evidence, and // the spread column reported a contest where there was a spelling difference. func TestFoldVariantsCountsConventionsAndShowsRawForms(t *testing.T) { observed := []Observed{{Key: "元海", Src: "元海", Proposals: []Proposal{ {Dst: "море истинной ци", Chunks: 3}, {Dst: "Море истинной ци", Chunks: 4}, // same convention, different case {Dst: "Первозданное море", Chunks: 5}, }}} got := Merge(nil, observed, lowerFold) if len(got) != 1 { t.Fatalf("one surface, one candidate: %+v", got) } c := got[0] if c.Conventions() != 2 { t.Fatalf("two decisions were made, not three: %+v", c.Variants) } if c.Spread() != 3 { t.Fatalf("the drafts still wrote it three ways and the owner must see that: spread=%d", c.Spread()) } var folded *Variant for i := range c.Variants { if strings.EqualFold(c.Variants[i].Dst, "море истинной ци") { folded = &c.Variants[i] } } if folded == nil { t.Fatalf("the folded class must be present as a RAW form, not a normalized one: %+v", c.Variants) } if folded.Chunks != 7 { t.Fatalf("the vote is the sum of the class, got %d", folded.Chunks) } if folded.Dst != "Море истинной ци" { t.Fatalf("the representative is the most-proposed RAW form, got %q", folded.Dst) } // And the fold decides the ranking: 7 folded chunks now outweigh the 5 that used to win on 4-vs-5. ScoreVariants(&c, ScoreOpts{}) if !strings.EqualFold(c.Best(), "море истинной ци") { t.Fatalf("the consensus must stop losing to its own spelling variant, got %q", c.Best()) } } // TestFoldVariantsKeepsViaProvenanceHonest: a rendering that ALSO arrived on the candidate's own key is // direct. First-wins used to label such a consensus «proposed for » purely because the alias row was // seen first — presenting an agreed rendering as a mis-clustered alias's guess. func TestFoldVariantsKeepsViaProvenanceHonest(t *testing.T) { mined := []Mined{{Key: "方源", Src: "方源", Type: "name", Aliases: []string{"方小子", "小方"}}} observed := []Observed{ {Key: "方小子", Src: "方小子", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 1}}}, // through an alias, FIRST {Key: "方源", Src: "方源", Proposals: []Proposal{{Dst: "Фан Юань", Chunks: 6}}}, // and directly {Key: "小方", Src: "小方", Proposals: []Proposal{{Dst: "малыш Фан", Chunks: 2}}}, // alias only } for _, c := range Merge(mined, observed, lowerFold) { if c.Key != "方源" { continue } for _, v := range c.Variants { switch v.Dst { case "Фан Юань": if v.Via != "" { t.Fatalf("a rendering also proposed on the term's own key is DIRECT, got via=%q", v.Via) } if v.Chunks != 7 { t.Fatalf("the alias vote still counts toward the total, got %d", v.Chunks) } case "малыш Фан": if v.Via != "小方" { t.Fatalf("a rendering that ONLY ever came through an alias must keep saying so, got %q", v.Via) } } } } }