327 lines
14 KiB
Go
327 lines
14 KiB
Go
package membank
|
||
|
||
// labelharness_test.go: the K6 (glossary post-check) half of the durable labels re-measurement — the
|
||
// companion to internal/checks/labelharness_test.go. It lives here because dstFormPresent / entry are
|
||
// unexported. Env-gated exactly the same way (TM_CHECKER_LABELS=1, corpus OUT of git).
|
||
//
|
||
// For every (unit, glossary term) row of the package-6 pool it asks the REAL dstFormPresent whether an
|
||
// accepted dst form appears in the unit's final, and joins that to the human label (defect = a real miss,
|
||
// ok = the form is present). predicted_defect = NOT dstFormPresent. The package-6 baseline (metrics.json
|
||
// k6) predates the bank-quality stemmer (D39.75), so the harness measures BOTH the pre-stemmer path (inert
|
||
// stemmer == the ratified baseline) and the current path (stemmer on) — the delta is the stemmer / #10 work.
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/lang"
|
||
"textmachine/backend/internal/standdata"
|
||
"textmachine/backend/internal/text"
|
||
)
|
||
|
||
// The same corpus root internal/checks reads, through the same package: the copy those two halves used
|
||
// to carry (plus miner's own) was the third user the old comment said would justify extracting one.
|
||
var memLabelsDir = standdata.EnvOr("TM_CHECKER_LABELS_DIR", standdata.StandFile("gu-zhenren", "labels"))
|
||
|
||
type k6Pool struct {
|
||
ID string `json:"id"`
|
||
Run string `json:"run"`
|
||
Chapter int `json:"chapter"`
|
||
Chunk int `json:"chunk_idx"`
|
||
Src string `json:"src"`
|
||
Dst string `json:"dst"`
|
||
Forms []string `json:"forms"`
|
||
Status string `json:"status"`
|
||
Type string `json:"type"`
|
||
}
|
||
|
||
type k6Label struct {
|
||
ID string `json:"id"`
|
||
Label string `json:"label"`
|
||
}
|
||
|
||
type memUnit struct {
|
||
Run string `json:"run"`
|
||
Chapter int `json:"chapter"`
|
||
Chunk int `json:"chunk_idx"`
|
||
Final string `json:"final"`
|
||
}
|
||
|
||
// stripManifestHeadingMem mirrors the checks harness: the post-check also sees final BEFORE ApplyHeading.
|
||
func stripManifestHeadingMem(final string) string {
|
||
lines := strings.Split(final, "\n")
|
||
if len(lines) == 0 {
|
||
return final
|
||
}
|
||
rest, ok := strings.CutPrefix(strings.TrimSpace(lines[0]), "Глава ")
|
||
if !ok {
|
||
return final
|
||
}
|
||
for _, r := range rest {
|
||
if r < '0' || r > '9' {
|
||
return final
|
||
}
|
||
}
|
||
i := 1
|
||
for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
|
||
i++
|
||
}
|
||
return strings.Join(lines[i:], "\n")
|
||
}
|
||
|
||
// k6metric is the confusion count with the same defect>disputable>ok convention (here labels are per-row).
|
||
type k6metric struct{ TP, FP, FN, TN, Disp int }
|
||
|
||
func (m *k6metric) add(pred, defect bool) {
|
||
switch {
|
||
case pred && defect:
|
||
m.TP++
|
||
case pred && !defect:
|
||
m.FP++
|
||
case !pred && defect:
|
||
m.FN++
|
||
default:
|
||
m.TN++
|
||
}
|
||
}
|
||
|
||
// measureK6 runs dstFormPresent over every pool row against its unit's final and joins to the labels.
|
||
// When demoteSingleHan is set it models the #10 fix: a single-Han-rune key miss is demoted to
|
||
// observability (not a predicted CONFIRMED defect), exactly as production Postcheck now routes it.
|
||
func measureK6(t *testing.T, stemmer lang.TargetStemmer, demoteSingleHan bool) (k6metric, []string) {
|
||
return measureK6With(t, stemmer, demoteSingleHan, matchStrict)
|
||
}
|
||
|
||
// measureK6With is measureK6 with the anchored stem tolerance switchable, so the SAME labels answer both
|
||
// "what does the shipping path score" and "what would the tolerance score" — the before/after the pack
|
||
// requires on one label set rather than two.
|
||
func measureK6With(t *testing.T, stemmer lang.TargetStemmer, demoteSingleHan bool, rule int) (k6metric, []string) {
|
||
t.Helper()
|
||
// corpus finals keyed by unit id.
|
||
finals := map[string]string{}
|
||
cb, err := os.ReadFile(filepath.Join(memLabelsDir, "raw", "corpus.jsonl"))
|
||
if err != nil {
|
||
t.Fatalf("read corpus: %v", err)
|
||
}
|
||
for _, line := range strings.Split(strings.TrimSpace(string(cb)), "\n") {
|
||
if strings.TrimSpace(line) == "" {
|
||
continue
|
||
}
|
||
var u memUnit
|
||
if err := json.Unmarshal([]byte(line), &u); err != nil {
|
||
t.Fatalf("parse corpus: %v", err)
|
||
}
|
||
finals[fmt.Sprintf("%s:%d:%d", u.Run, u.Chapter, u.Chunk)] = u.Final
|
||
}
|
||
// labels.
|
||
labels := map[string]string{}
|
||
lb, err := os.ReadFile(filepath.Join(memLabelsDir, "labels", "k6.jsonl"))
|
||
if err != nil {
|
||
t.Fatalf("read k6 labels: %v", err)
|
||
}
|
||
for _, line := range strings.Split(strings.TrimSpace(string(lb)), "\n") {
|
||
if strings.TrimSpace(line) == "" {
|
||
continue
|
||
}
|
||
var l k6Label
|
||
if err := json.Unmarshal([]byte(line), &l); err != nil {
|
||
t.Fatalf("parse k6 label: %v", err)
|
||
}
|
||
labels[l.ID] = l.Label
|
||
}
|
||
// pool → dstFormPresent per row, joined to labels.
|
||
pb, err := os.ReadFile(filepath.Join(memLabelsDir, "pools", "k6_glossary.jsonl"))
|
||
if err != nil {
|
||
t.Fatalf("read k6 pool: %v", err)
|
||
}
|
||
var m k6metric
|
||
var flaggedOK []string // fp ids (flagged as miss but label=ok) — the #10 / inflection-gap surface
|
||
for _, line := range strings.Split(strings.TrimSpace(string(pb)), "\n") {
|
||
if strings.TrimSpace(line) == "" {
|
||
continue
|
||
}
|
||
var p k6Pool
|
||
if err := json.Unmarshal([]byte(line), &p); err != nil {
|
||
t.Fatalf("parse k6 pool: %v", err)
|
||
}
|
||
lab, ok := labels[p.ID]
|
||
if !ok {
|
||
continue // pool wider than the labelled sample
|
||
}
|
||
final := stripManifestHeadingMem(finals[fmt.Sprintf("%s:%d:%d", p.Run, p.Chapter, p.Chunk)])
|
||
nout := text.NormalizeTargetForm(final)
|
||
noutRunes := []rune(nout)
|
||
outWords := lang.TokenizeWords(nout)
|
||
var forms []string
|
||
for _, f := range p.Forms {
|
||
if nf := text.NormalizeTargetForm(f); nf != "" {
|
||
forms = append(forms, nf)
|
||
}
|
||
}
|
||
e := &entry{src: p.Src, dst: p.Dst, declForms: forms}
|
||
present := dstFormPresent(e, noutRunes, outWords, stemmer)
|
||
if rule != matchStrict {
|
||
present = present || dstFormPresentAnchored(&Bank{stemmer: stemmer}, e, final, rule)
|
||
}
|
||
// The firing key for a pool row is its src (the term whose key matched). Model the #10 demote.
|
||
predDefect := !present && !(demoteSingleHan && singleHanKeyFired(p.Src))
|
||
switch lab {
|
||
case "disputable":
|
||
m.Disp++
|
||
case "defect":
|
||
m.add(predDefect, true)
|
||
default:
|
||
if predDefect {
|
||
flaggedOK = append(flaggedOK, p.ID)
|
||
}
|
||
m.add(predDefect, false)
|
||
}
|
||
}
|
||
return m, flaggedOK
|
||
}
|
||
|
||
func TestK6LabelsBaseline(t *testing.T) {
|
||
if _, err := os.Stat(filepath.Join(memLabelsDir, "raw", "corpus.jsonl")); err != nil {
|
||
if os.Getenv("TM_CHECKER_LABELS") == "1" {
|
||
t.Fatalf("TM_CHECKER_LABELS=1 but corpus is missing: %v", err)
|
||
}
|
||
t.Skipf("labelled corpus absent (%v) — set TM_CHECKER_LABELS=1 to force", err)
|
||
}
|
||
|
||
// Pre-stemmer, pre-#10 path = the ratified package-6 baseline (metrics.json k6).
|
||
inert := lang.TargetStemmer{}
|
||
base, baseFP := measureK6(t, inert, false)
|
||
t.Logf("K6 BASELINE (pre-stemmer, pre-#10 = metrics.json): tp=%d fp=%d fn=%d tn=%d disp=%d", base.TP, base.FP, base.FN, base.TN, base.Disp)
|
||
t.Logf("K6 baseline fp ids: %v", baseFP)
|
||
if (base != k6metric{TP: 1, FP: 14, FN: 0, TN: 243, Disp: 2}) {
|
||
t.Errorf("k6 baseline drift: got %+v want tp1 fp14 fn0 tn243 disp2", base)
|
||
}
|
||
|
||
// Stemmer on (D39.75), no #10 — the intermediate landed state. Delta vs baseline = inflection-gap fps
|
||
// the stemmer now accepts (凡人→смертных, 甲等→разряд А).
|
||
stem := lang.NewTargetStemmer(lang.TargetChecksFor("ru"))
|
||
mid, midFP := measureK6(t, stem, false)
|
||
t.Logf("K6 stemmer-on, pre-#10: fp=%d (down from 14 — %v) precision=%.4f", mid.FP, diff(baseFP, midFP), prec(mid))
|
||
|
||
// Stemmer on + #10 demote — THIS pack's after-state. The 6 single-Han-rune (转) misses demote to
|
||
// observability, leaving the 元石/蛊虫 inflection residual (a SEED-completeness gap, not a code one).
|
||
after, afterFP := measureK6(t, stem, true)
|
||
t.Logf("K6 AFTER (stemmer + #10): tp=%d fp=%d fn=%d tn=%d disp=%d precision=%.4f", after.TP, after.FP, after.FN, after.TN, after.Disp, prec(after))
|
||
t.Logf("K6 after fp ids (residual = 元石/蛊虫 inflection-gap, seed concern): %v", afterFP)
|
||
if (after != k6metric{TP: 1, FP: 6, FN: 0, TN: 251, Disp: 2}) {
|
||
t.Errorf("k6 after-#10 drift: got %+v want tp1 fp6 fn0 tn251 disp2", after)
|
||
}
|
||
}
|
||
|
||
func prec(m k6metric) float64 {
|
||
if m.TP+m.FP == 0 {
|
||
return 0
|
||
}
|
||
return float64(m.TP) / float64(m.TP+m.FP)
|
||
}
|
||
|
||
// diff returns ids in a not in b (the fps removed by a change).
|
||
func diff(a, b []string) []string {
|
||
in := map[string]bool{}
|
||
for _, s := range b {
|
||
in[s] = true
|
||
}
|
||
var out []string
|
||
for _, s := range a {
|
||
if !in[s] {
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// TestK6TheTwoRelaxationsPricedSeparatelyOnTheSameLabels prices each relaxation of the post-check's
|
||
// matching rule on the ratified label set, SEPARATELY — because measured together they credit the wrong one.
|
||
//
|
||
// The residual false positives of the shipping path are all one shape: the seed listed only SINGULAR decl
|
||
// forms and the text used a plural («гу-червь» seeded as «гу-червя/гу-червю/гу-червём», «гу-червей» shipped;
|
||
// «первокамень» seeded five ways, «первокамней» shipped). dstFormPresent stems the base dst but matches a
|
||
// STORED decl form literally, so none of the five seeded forms can reach the sixth. Stemming the decl forms
|
||
// too removes every one of them, and it carries no hazard: a decl form is an author-supplied rendering of
|
||
// this very term, not a guess about the language.
|
||
//
|
||
// The anchored near-stem tolerance — the one with the «Синь»/«синий» hazard — removes NOTHING further here.
|
||
// It earns its place on a different population: a book whose rows carry no decl forms at all, where the base
|
||
// dst is the only thing to match and the stemmer's one-rune limit is the whole cost (the cold run of 11.09,
|
||
// 69 rows, zero decl).
|
||
//
|
||
// ⚠ WHAT THIS DOES NOT SHOW. The pool holds 258 labelled rows but only ONE labelled DEFECT, so "precision
|
||
// 1.0" after is a statement about one true positive and says nothing about the rate on a larger defect
|
||
// population. The load-bearing numbers are «six false flags become zero» and «recall did not move», not the
|
||
// ratio. Both relaxations are measured here and NEITHER is on the shipping path: changing which chunks get
|
||
// flagged is money-path behaviour and a separate decision, which this test exists to let someone make on a
|
||
// number instead of an argument.
|
||
func TestK6TheTwoRelaxationsPricedSeparatelyOnTheSameLabels(t *testing.T) {
|
||
if _, err := os.Stat(filepath.Join(memLabelsDir, "raw", "corpus.jsonl")); err != nil {
|
||
if os.Getenv("TM_CHECKER_LABELS") == "1" {
|
||
t.Fatalf("TM_CHECKER_LABELS=1 but corpus is missing: %v", err)
|
||
}
|
||
t.Skipf("labelled corpus absent (%v) — set TM_CHECKER_LABELS=1 to force", err)
|
||
}
|
||
stem := lang.NewTargetStemmer(lang.TargetChecksFor("ru"))
|
||
|
||
// The shipping path, re-measured here so the comparison is against a number this test took itself and
|
||
// not one quoted from another test's log.
|
||
before, beforeFP := measureK6With(t, stem, true, matchStrict)
|
||
stemAll, stemAllFP := measureK6With(t, stem, true, matchStemAll)
|
||
after, afterFP := measureK6With(t, stem, true, matchRelaxed)
|
||
t.Logf("K6 shipping path : tp=%d fp=%d fn=%d tn=%d precision=%.4f fps=%v", before.TP, before.FP, before.FN, before.TN, prec(before), beforeFP)
|
||
t.Logf("K6 + decl forms stemmed too : tp=%d fp=%d precision=%.4f — removes %v", stemAll.TP, stemAll.FP, prec(stemAll), diff(beforeFP, stemAllFP))
|
||
t.Logf("K6 + anchored near-stem : tp=%d fp=%d fn=%d tn=%d precision=%.4f — removes a further %v", after.TP, after.FP, after.FN, after.TN, prec(after), diff(stemAllFP, afterFP))
|
||
|
||
// RECALL MUST NOT MOVE. Both relaxations only ever turn a predicted defect into a pass, so each can lose
|
||
// a true positive and neither can gain one — and a relaxation that bought its precision by dropping the
|
||
// single real defect would be worthless. tp and fn are pinned for that reason, not for tidiness.
|
||
for _, c := range []struct {
|
||
name string
|
||
m k6metric
|
||
}{{"decl-stemmed", stemAll}, {"anchored", after}} {
|
||
if c.m.TP != before.TP || c.m.FN != before.FN {
|
||
t.Errorf("%s moved RECALL: tp %d→%d, fn %d→%d — a relaxation must only remove false flags",
|
||
c.name, before.TP, c.m.TP, before.FN, c.m.FN)
|
||
}
|
||
}
|
||
// The ATTRIBUTION, pinned: the six residual false flags are the seeded-singular gap, and it is stemming
|
||
// the decl forms that closes them — not the near-stem tolerance, which closes none of them. A later
|
||
// change that shifts the credit between the two steps must fail here, because the whole value of the
|
||
// measurement is knowing which relaxation is the one worth its risk.
|
||
if (before != k6metric{TP: 1, FP: 6, FN: 0, TN: 251, Disp: 2}) {
|
||
t.Errorf("shipping-path drift: got %+v want tp1 fp6 fn0 tn251 disp2", before)
|
||
}
|
||
if (stemAll != k6metric{TP: 1, FP: 0, FN: 0, TN: 257, Disp: 2}) {
|
||
t.Errorf("decl-stemming drift: got %+v want tp1 fp0 fn0 tn257 disp2 — it is this step that closes all six", stemAll)
|
||
}
|
||
if n := len(diff(stemAllFP, afterFP)); n != 0 {
|
||
t.Errorf("the near-stem tolerance removed %d further false flags on this corpus; it removed none when measured, and the report attributes the gain to decl-stemming", n)
|
||
}
|
||
}
|
||
|
||
// TestTheAnchoredToleranceRefusesASingleWordName pins the gate that keeps the tolerance out of the collision
|
||
// data/target-ru.txt refuses the bare soft sign over: «Синь» and «синий» satisfy the one-rune stem relation,
|
||
// so a single-word rendering must never get it. Two words with a strict anchor DO, which is the whole
|
||
// difference between the tolerance and the global suffix that note rejects.
|
||
func TestTheAnchoredToleranceRefusesASingleWordName(t *testing.T) {
|
||
stem := lang.NewTargetStemmer(lang.TargetChecksFor("ru"))
|
||
// Premise: the relation itself DOES equate the dangerous pair — without this the test would pass
|
||
// because the stemmer never saw a collision, not because the anchor gate held it.
|
||
if !stem.NearStem("Синь", "синий") {
|
||
t.Fatalf("premise broken: NearStem must equate «Синь»/«синий» — that is the hazard being gated")
|
||
}
|
||
one := &entry{src: "辛", dst: "Синь"}
|
||
if dstFormPresentAnchored(&Bank{stemmer: stem}, one, "Небо было синим, совсем синий день.", matchRelaxed) {
|
||
t.Errorf("a one-word name was accepted through the stem tolerance: «Синь» must not be found in «синий»")
|
||
}
|
||
// The same tolerance, anchored by a second word that matches strictly, is exactly what it is for.
|
||
two := &entry{src: "辛家", dst: "род Синь"}
|
||
if !dstFormPresentAnchored(&Bank{stemmer: stem}, two, "Он вышел из рода Синя на рассвете.", matchRelaxed) {
|
||
t.Errorf("an anchored two-word rendering was NOT accepted: «род Синь» should match «рода Синя»")
|
||
}
|
||
}
|