132 lines
6.4 KiB
Go
132 lines
6.4 KiB
Go
package terminology
|
||
|
||
import (
|
||
"sort"
|
||
"strings"
|
||
)
|
||
|
||
// classify.go: the TYPE re-derivation channel (bank-quality §2, D39.68). The draft type heuristic is wrong
|
||
// 12–22% (D39.65 row 36a), and type ∈ {name,place} routes a candidate to transliteration — so a realia
|
||
// surface mistyped as a name (元石) is FORCED to «юаньши» instead of being translated. A focused classifier
|
||
// pass fixes the type BEFORE the render (the live probe fixed 6/6 branch-harm units; an inline type returned
|
||
// WITH the rendering arrives too late to de-bias it). This file is the pure half — the reply parser and the
|
||
// honest $0 label screen; the class DEFINITIONS live in the pair's classifier prompt, no pair literal here.
|
||
|
||
// Types is the engine's closed type vocabulary for the CLASSIFIER's answer. The names are engine identifiers
|
||
// (the miner emits them, the glossary stores them, emissionEligible gates on them), so they are Go constants
|
||
// here, not pair data; the pair's prompt explains each class in its own language with its own examples.
|
||
var Types = map[string]bool{"name": true, "place": true, "title": true, "term": true}
|
||
|
||
// CandidateTypes is the wider set a CANDIDATE can actually carry, and it is not the same set — which is a
|
||
// live seam, not pedantry. The banknote channel accepts `nickname` from a draft (pipeline.bankTypeOK), that
|
||
// type rides into Observed and onto a draft-side-only candidate, and the classifier never overwrites a term
|
||
// it was not asked about. So anything keyed on a candidate's type — the family rules, first — must be keyed
|
||
// on THIS set: validating against Types alone rejects a legitimate pair rule for `nickname` while the type
|
||
// keeps arriving, i.e. refuses the fix and keeps the defect.
|
||
var CandidateTypes = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true}
|
||
|
||
// TypeNames lists a type set in a stable order, so an error message can say what it actually accepts instead
|
||
// of a hardcoded list that drifts from the map beside it.
|
||
func TypeNames(set map[string]bool) []string {
|
||
out := make([]string, 0, len(set))
|
||
for t := range set {
|
||
out = append(out, t)
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
// Genders is the closed vocabulary of the classifier's THIRD column — the gender datum backlog row 210
|
||
// exists for. It is deliberately a SUBSET of the bank's own vocabulary (membank.knownGenders, which also
|
||
// has `hidden`): `hidden` means «this character's gender is concealed until a reveal, so avoid gendering
|
||
// them at all» (D19.3), which is a decision about a book's plot and its spoiler policy, not an observation
|
||
// a reader of KWIC lines can make. A model guessing it would silently de-gender an ordinary character for
|
||
// the whole book. It stays a human datum, entered in the seed.
|
||
//
|
||
// GenderNone is the answer for everything else — a place, an object, a title, and a character whose gender
|
||
// the contexts simply do not establish — and it is an ANSWER rather than an omission: without a word for
|
||
// «no gender here» the only way to say it is to leave the column out, which is indistinguishable from a
|
||
// model that ignored the question.
|
||
var Genders = map[string]bool{"male": true, "female": true, "neuter": true, GenderNone: true}
|
||
|
||
// GenderNone is the vocabulary's «no gender datum» word. It maps to the empty string on the way to the
|
||
// bank — the engine's own spelling for «this row has no gender» everywhere else.
|
||
const GenderNone = "none"
|
||
|
||
// ParseTypes turns a classifier reply into key → corrected type and key → gender, keeping only the terms we
|
||
// asked about and only the closed vocabularies. Same tolerant field format and the same accounting
|
||
// discipline as ParseReply: an unusable or off-vocabulary line is COUNTED, never silently dropped, so a paid
|
||
// call that bought no classification is loud rather than an empty map that reads as "nothing to correct".
|
||
//
|
||
// ⚠ THE GENDER COLUMN NEVER COSTS THE CLASS. A line whose third field is unreadable still yields its type:
|
||
// the two answers are independent, and dropping a correct classification because the gender word was
|
||
// misspelled would make the new column able to break the old one. It also never GLUES — the failure the
|
||
// terminologist's own third column was built against («наставник» + «95%» becoming one rendering) — because
|
||
// the vocabulary is closed and a value outside it is refused rather than written.
|
||
func ParseTypes(reply string, expected []string, normalize func(string) string) (types, genders map[string]string, st ReplyStats) {
|
||
want := make(map[string]bool, len(expected))
|
||
for _, k := range expected {
|
||
want[k] = true
|
||
}
|
||
types, genders = map[string]string{}, map[string]string{}
|
||
for _, ln := range strings.Split(reply, "\n") {
|
||
if t := strings.TrimSpace(ln); t == "" || strings.HasPrefix(t, "#") {
|
||
continue
|
||
}
|
||
f := splitFields(ln)
|
||
if len(f) < 2 {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
key := normalize(f[0])
|
||
if !want[key] {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
if _, dup := types[key]; dup {
|
||
continue // first answer wins, as in ParseReply
|
||
}
|
||
typ := strings.ToLower(strings.TrimSpace(f[1]))
|
||
if !Types[typ] {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
types[key] = typ
|
||
switch {
|
||
case len(f) < 3:
|
||
st.NoGender++
|
||
default:
|
||
g := strings.ToLower(strings.TrimSpace(f[2]))
|
||
switch {
|
||
case !Genders[g]:
|
||
st.BadGender++
|
||
case g != GenderNone:
|
||
genders[key] = g
|
||
}
|
||
}
|
||
}
|
||
return types, genders, st
|
||
}
|
||
|
||
// LabelRow is one row the $0 label screen inspects: the corrected type and the consolidated rendering.
|
||
type LabelRow struct {
|
||
Src string
|
||
Type string
|
||
Dst string
|
||
}
|
||
|
||
// TypeLabelMismatches is the HONEST $0 screen (§2, warm-run hygiene). It flags a name/place row whose
|
||
// rendering was clearly TRANSLATED (multi-word) — a label/rendering disagreement worth a human's eye. It is
|
||
// explicitly NOT a safety net for the transliteration harm: a mistyped name rendered as a single lower-case
|
||
// token (元石→юаньши) or a capitalised one (元海→Юаньхай) passes it clean, because the source class the harm
|
||
// needs is exactly what a $0 pass cannot recover. The classifier pass is what prevents the harm; this only
|
||
// surfaces leftover label noise for review. Deterministic, input order preserved.
|
||
func TypeLabelMismatches(rows []LabelRow) []LabelRow {
|
||
var out []LabelRow
|
||
for _, r := range rows {
|
||
if (r.Type == "name" || r.Type == "place") && strings.ContainsRune(strings.TrimSpace(r.Dst), ' ') {
|
||
out = append(out, r)
|
||
}
|
||
}
|
||
return out
|
||
}
|