package terminology import ( "reflect" "strings" "testing" ) func TestParseTypesKeepsOnlyAskedAndVocabulary(t *testing.T) { id := func(s string) string { return s } reply := strings.Join([]string{ "元石\tterm\tnone", "方源 name male", // spaces instead of a tab — tolerant split "青茅山 | place | none", // padded pipe "家老\tPERSON\tnone", // off-vocabulary CLASS → refused and counted "陌生\tterm\tnone", // never asked → refused and counted "元石\tname\tmale", // duplicate key → first answer wins "мусор", // one field → refused and counted }, "\n") got, genders, st := ParseTypes(reply, []string{"元石", "方源", "青茅山", "家老"}, id) want := map[string]string{"元石": "term", "方源": "name", "青茅山": "place"} if !reflect.DeepEqual(got, want) { t.Fatalf("parse = %#v, want %#v", got, want) } // `none` is an ANSWER and reaches the bank as nothing at all, so only 方源 carries a datum. if !reflect.DeepEqual(genders, map[string]string{"方源": "male"}) { t.Fatalf("genders = %#v, want only 方源:male", genders) } // off-vocabulary PERSON + unasked 陌生 + malformed мусор = 3 bad lines (the duplicate is silently ignored, // not counted, exactly like ParseReply). if st.Bad != 3 { t.Fatalf("unusable/off-vocabulary lines must be COUNTED, got %d", st.Bad) } if st.BadGender != 0 || st.NoGender != 0 { t.Fatalf("every usable line answered the gender column: %+v", st) } } // TestTheGenderColumnNeverCostsTheClass is the guarantee that lets a new column be added to a reply the // engine already depends on: the two answers are independent. A gender written in a word nobody asked for // is dropped and counted; a line that omits the column is counted too — otherwise a model quietly ceasing // to answer it looks exactly like a book full of places. Neither loses the classification. func TestTheGenderColumnNeverCostsTheClass(t *testing.T) { id := func(s string) string { return s } reply := strings.Join([]string{ "元石\tterm\tнеодушевлённое", // off-vocabulary gender → class kept, gender dropped and counted "方源\tname", // no third column at all → class kept, counted apart "青鸟\tname\tHIDDEN", // `hidden` is deliberately NOT in the vocabulary (a human's decision) "小蝉\tname\tFEMALE", // case-insensitive, like the class }, "\n") types, genders, st := ParseTypes(reply, []string{"元石", "方源", "青鸟", "小蝉"}, id) if len(types) != 4 { t.Fatalf("every line classified a term the caller asked about: %#v", types) } if !reflect.DeepEqual(genders, map[string]string{"小蝉": "female"}) { t.Fatalf("only the in-vocabulary gender may reach the bank, got %#v", genders) } if st.Bad != 0 { t.Fatalf("a gender fault must not be charged to the CLASS counter: %+v", st) } if st.BadGender != 2 || st.NoGender != 1 { t.Fatalf("the two gender failures are counted apart: %+v", st) } } func TestParseTypesNormalizesKey(t *testing.T) { got, _, _ := ParseTypes("FANG\tname\tmale", []string{"fang"}, strings.ToLower) if got["fang"] != "name" { t.Fatalf("the reply key must be normalized by the caller's function, got %#v", got) } } // TestTypeLabelMismatchesIsHonest pins the screen's DELIBERATE limit: it flags a name/place whose rendering // was clearly translated (multi-word), but it is NOT a safety net — a mistyped name rendered as one token, // lower-case (元石→юаньши) or capitalised (元海→Юаньхай), passes it clean. The classifier phase is what // prevents that harm; asserting the miss keeps a future reader from mistaking this for the guard. func TestTypeLabelMismatchesIsHonest(t *testing.T) { rows := []LabelRow{ {Src: "花家", Type: "name", Dst: "Дом Хуа"}, // translated name → FLAG {Src: "青茅山", Type: "place", Dst: "гора Цинмао"}, // translated place → FLAG {Src: "元石", Type: "name", Dst: "юаньши"}, // the harm, single lower-case token → MISSED {Src: "元海", Type: "name", Dst: "Юаньхай"}, // the harm, capitalised token → MISSED {Src: "灵泉", Type: "term", Dst: "духовный источник"}, // term is not screened → not flagged } got := TypeLabelMismatches(rows) var srcs []string for _, r := range got { srcs = append(srcs, r.Src) } if !reflect.DeepEqual(srcs, []string{"花家", "青茅山"}) { t.Fatalf("only the translated name/place rows must flag, got %v", srcs) } }