76 lines
3.3 KiB
Go
76 lines
3.3 KiB
Go
package terminology
|
||
|
||
import "strings"
|
||
|
||
// classify.go: the TYPE re-derivation channel (bank-quality §2, D39.68). The draft type heuristic is wrong
|
||
// 12–22% (D39.65 row 36a), and type ∈ {name,place} routes a candidate to transliteration — so a realia
|
||
// surface mistyped as a name (元石) is FORCED to «юаньши» instead of being translated. A focused classifier
|
||
// pass fixes the type BEFORE the render (the live probe fixed 6/6 branch-harm units; an inline type returned
|
||
// WITH the rendering arrives too late to de-bias it). This file is the pure half — the reply parser and the
|
||
// honest $0 label screen; the class DEFINITIONS live in the pair's classifier prompt, no pair literal here.
|
||
|
||
// Types is the engine's closed type vocabulary. The names are engine identifiers (the miner emits them, the
|
||
// glossary stores them, emissionEligible gates on them), so they are Go constants here, not pair data; the
|
||
// pair's prompt explains each class in its own language with its own examples.
|
||
var Types = map[string]bool{"name": true, "place": true, "title": true, "term": true}
|
||
|
||
// ParseTypes turns a classifier reply into key → corrected type, keeping only the terms we asked about and
|
||
// only the closed vocabulary. Same tolerant two-field line format and the same accounting discipline as
|
||
// ParseReply: an unusable or off-vocabulary line is COUNTED (st.Bad), never silently dropped, so a paid call
|
||
// that bought no classification is loud rather than an empty map that reads as "nothing to correct".
|
||
func ParseTypes(reply string, expected []string, normalize func(string) string) (map[string]string, ReplyStats) {
|
||
want := make(map[string]bool, len(expected))
|
||
for _, k := range expected {
|
||
want[k] = true
|
||
}
|
||
out := map[string]string{}
|
||
var st ReplyStats
|
||
for _, ln := range strings.Split(reply, "\n") {
|
||
if t := strings.TrimSpace(ln); t == "" || strings.HasPrefix(t, "#") {
|
||
continue
|
||
}
|
||
f := splitFields(ln)
|
||
if len(f) < 2 {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
key := normalize(f[0])
|
||
if !want[key] {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
if _, dup := out[key]; dup {
|
||
continue // first answer wins, as in ParseReply
|
||
}
|
||
typ := strings.ToLower(strings.TrimSpace(f[1]))
|
||
if !Types[typ] {
|
||
st.Bad++
|
||
continue
|
||
}
|
||
out[key] = typ
|
||
}
|
||
return out, st
|
||
}
|
||
|
||
// LabelRow is one row the $0 label screen inspects: the corrected type and the consolidated rendering.
|
||
type LabelRow struct {
|
||
Src string
|
||
Type string
|
||
Dst string
|
||
}
|
||
|
||
// TypeLabelMismatches is the HONEST $0 screen (§2, warm-run hygiene). It flags a name/place row whose
|
||
// rendering was clearly TRANSLATED (multi-word) — a label/rendering disagreement worth a human's eye. It is
|
||
// explicitly NOT a safety net for the transliteration harm: a mistyped name rendered as a single lower-case
|
||
// token (元石→юаньши) or a capitalised one (元海→Юаньхай) passes it clean, because the source class the harm
|
||
// needs is exactly what a $0 pass cannot recover. The classifier pass is what prevents the harm; this only
|
||
// surfaces leftover label noise for review. Deterministic, input order preserved.
|
||
func TypeLabelMismatches(rows []LabelRow) []LabelRow {
|
||
var out []LabelRow
|
||
for _, r := range rows {
|
||
if (r.Type == "name" || r.Type == "place") && strings.ContainsRune(strings.TrimSpace(r.Dst), ' ') {
|
||
out = append(out, r)
|
||
}
|
||
}
|
||
return out
|
||
}
|