98 lines
4.3 KiB
Go
98 lines
4.3 KiB
Go
package terminology
|
||
|
||
import (
|
||
"sort"
|
||
"strings"
|
||
)
|
||
|
||
// classify.go: the TYPE re-derivation channel (bank-quality §2, D39.68). The draft type heuristic is wrong
|
||
// 12–22% (D39.65 row 36a), and type ∈ {name,place} routes a candidate to transliteration — so a realia
|
||
// surface mistyped as a name (元石) is FORCED to «юаньши» instead of being translated. A focused classifier
|
||
// pass fixes the type BEFORE the render (the live probe fixed 6/6 branch-harm units; an inline type returned
|
||
// WITH the rendering arrives too late to de-bias it). This file is the pure half — the reply parser and the
|
||
// honest $0 label screen; the class DEFINITIONS live in the pair's classifier prompt, no pair literal here.
|
||
|
||
// Types is the engine's closed type vocabulary for the CLASSIFIER's answer. The names are engine identifiers
|
||
// (the miner emits them, the glossary stores them, emissionEligible gates on them), so they are Go constants
|
||
// here, not pair data; the pair's prompt explains each class in its own language with its own examples.
|
||
var Types = map[string]bool{"name": true, "place": true, "title": true, "term": true}
|
||
|
||
// CandidateTypes is the wider set a CANDIDATE can actually carry, and it is not the same set — which is a
|
||
// live seam, not pedantry. The banknote channel accepts `nickname` from a draft (pipeline.bankTypeOK), that
|
||
// type rides into Observed and onto a draft-side-only candidate, and the classifier never overwrites a term
|
||
// it was not asked about. So anything keyed on a candidate's type — the family rules, first — must be keyed
|
||
// on THIS set: validating against Types alone rejects a legitimate pair rule for `nickname` while the type
|
||
// keeps arriving, i.e. refuses the fix and keeps the defect.
|
||
var CandidateTypes = map[string]bool{"name": true, "place": true, "title": true, "term": true, "nickname": true}
|
||
|
||
// TypeNames lists a type set in a stable order, so an error message can say what it actually accepts instead
|
||
// of a hardcoded list that drifts from the map beside it.
|
||
func TypeNames(set map[string]bool) []string {
|
||
out := make([]string, 0, len(set))
|
||
for t := range set {
|
||
out = append(out, t)
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
// ParseTypes turns a classifier reply into key → corrected type, keeping only the terms we asked about and
|
||
// only the closed vocabulary. Same tolerant two-field line format and the same accounting discipline as
|
||
// ParseReply: an unusable or off-vocabulary line is COUNTED (st.Bad), never silently dropped, so a paid call
|
||
// that bought no classification is loud rather than an empty map that reads as "nothing to correct".
|
||
func ParseTypes(reply string, expected []string, normalize func(string) string) (map[string]string, ReplyStats) {
|
||
want := make(map[string]bool, len(expected))
|
||
for _, k := range expected {
|
||
want[k] = true
|
||
}
|
||
out := map[string]string{}
|
||
var st ReplyStats
|
||
for _, ln := range strings.Split(reply, "\n") {
|
||
if t := strings.TrimSpace(ln); t == "" || strings.HasPrefix(t, "#") {
|
||
continue
|
||
}
|
||
f := splitFields(ln)
|
||
if len(f) < 2 {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
key := normalize(f[0])
|
||
if !want[key] {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
if _, dup := out[key]; dup {
|
||
continue // first answer wins, as in ParseReply
|
||
}
|
||
typ := strings.ToLower(strings.TrimSpace(f[1]))
|
||
if !Types[typ] {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
out[key] = typ
|
||
}
|
||
return out, st
|
||
}
|
||
|
||
// LabelRow is one row the $0 label screen inspects: the corrected type and the consolidated rendering.
|
||
type LabelRow struct {
|
||
Src string
|
||
Type string
|
||
Dst string
|
||
}
|
||
|
||
// TypeLabelMismatches is the HONEST $0 screen (§2, warm-run hygiene). It flags a name/place row whose
|
||
// rendering was clearly TRANSLATED (multi-word) — a label/rendering disagreement worth a human's eye. It is
|
||
// explicitly NOT a safety net for the transliteration harm: a mistyped name rendered as a single lower-case
|
||
// token (元石→юаньши) or a capitalised one (元海→Юаньхай) passes it clean, because the source class the harm
|
||
// needs is exactly what a $0 pass cannot recover. The classifier pass is what prevents the harm; this only
|
||
// surfaces leftover label noise for review. Deterministic, input order preserved.
|
||
func TypeLabelMismatches(rows []LabelRow) []LabelRow {
|
||
var out []LabelRow
|
||
for _, r := range rows {
|
||
if (r.Type == "name" || r.Type == "place") && strings.ContainsRune(strings.TrimSpace(r.Dst), ' ') {
|
||
out = append(out, r)
|
||
}
|
||
}
|
||
return out
|
||
}
|