textmachine/backend/internal/checks/labelharness_test.go

package checks

// labelharness_test.go: the DURABLE re-measurement of the deterministic checkers against the
// polygon-6 labelled corpus (<repo>/books/gu-zhenren/labels/, package-6 baseline metrics.json, D39.39).
// The previous harness lived in a scratch module and was lost ("персист, не scratch"); this is its
// in-git replacement. It is env-gated exactly like the miner-parity test: the labelled corpus carries
// book text and is OUT of git (CLAUDE.md), so the test SKIPS when the corpus is absent (CI / fresh
// checkout) and runs on the stand, or when TM_CHECKER_LABELS=1 forces it (then a missing path fails loud).
//
// It calls the REAL internal/checks functions over the real (source, draft, final) of every unit and
// reports precision/recall per defect class, joining the checker verdict to the human labels by the exact
// convention package-6 ratified (reverse-engineered from the stored verdicts and locked against
// metrics.json): unit-level classes (K1/K5a/K5b/K5c) aggregate occurrence labels to their unit with the
// priority defect>disputable>ok and fire on the sub-counter; K2 is per-token over the labelled field; K3
// is per-word-type; K4b is per chevron line, measured as the per-line PREDICATE chevronSpeechShape &&
// isSpokenChevronLine (spoken=defect, ambiguous=disputable) — NOT the unit em-dash-MIXING count; the two
// coincide on this corpus (0 divergence) but the predicate is what tech-debt anchor #3 concerns; k4_inverse
// is per em-dash line (thought=defect) via the attribution inner-marker rule (the D39.39 build — no longer
// "always silent").
//
// All 12 metrics.json keys are reproduced (11 here + K6 in the membank harness). k4a (n=348, all-TN — every
// dialogue line across the k4.jsonl buckets carries no straight-quote/hyphen MARKER defect) is measured
// MARKER-ONLY (measureK4a), deliberately NOT via the folded DialogueDash counter: the k4_inverse build folds
// inverseMarker into that counter, so a raw-count k4a would misread the 3 inverse fires as marker defects. The
// marker-only measure — a leading « " » / « - » / « – » defect, never the correct em-dash — keeps k4a locked
// at 0/0/0/348 regardless of the fold (verified against metrics.json).
//
// $0, pure, deterministic. The K6 glossary post-check has its own harness in internal/membank
// (labelharness_test.go there) because dstFormPresent is unexported.
import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/standdata"
)

var (
	// The labelled corpus lives in the stand, which since 24.08 is <repo>/books under its own git
	// repository (D39.157 п.2) — resolved by the repository MARKER, never from $HOME, whose value says
	// nothing about where this clone is. TM_CHECKER_LABELS_DIR stays senior to the default.
	labelsDir    = standdata.EnvOr("TM_CHECKER_LABELS_DIR", standdata.StandFile("gu-zhenren", "labels"))
	langpackRoot = standdata.EnvOr("TM_CHECKER_LANGPACK_ROOT", "../../configs/langpacks")
)

// --- corpus / label record shapes -------------------------------------------------

type corpusUnit struct {
	Run      string `json:"run"`
	Chapter  int    `json:"chapter"`
	ChunkIdx int    `json:"chunk_idx"`
	Source   string `json:"source"`
	Draft    string `json:"draft"`
	Final    string `json:"final"`
}

func (u corpusUnit) id() string { return fmt.Sprintf("%s:%d:%d", u.Run, u.Chapter, u.ChunkIdx) }

// unitOf strips a label id down to run:chapter:chunk_idx.
func unitOf(labelID string) string {
	p := strings.Split(labelID, ":")
	if len(p) < 3 {
		return labelID
	}
	return strings.Join(p[:3], ":")
}

type labelRow struct {
	ID       string `json:"id"`
	Label    string `json:"label"`
	LabelCC  string `json:"label_code_convention"`
	Token    string `json:"token"`
	Field    string `json:"field"`
	Word     string `json:"word"`
	Bucket   string `json:"bucket"`
	Speech   string `json:"speech"`
	SrcField string `json:"src_field"`
}

type metric struct{ TP, FP, FN, TN, Disp int }

func (m metric) n() int { return m.TP + m.FP + m.FN + m.TN }
func (m *metric) add(pred, defect bool) {
	switch {
	case pred && defect:
		m.TP++
	case pred && !defect:
		m.FP++
	case !pred && defect:
		m.FN++
	default:
		m.TN++
	}
}
func (m metric) precision() string {
	if m.TP+m.FP == 0 {
		return "n/a"
	}
	return fmt.Sprintf("%.4f", float64(m.TP)/float64(m.TP+m.FP))
}
func (m metric) recall() string {
	if m.TP+m.FN == 0 {
		return "n/a"
	}
	return fmt.Sprintf("%.4f", float64(m.TP)/float64(m.TP+m.FN))
}

func loadJSONL(t *testing.T, path string) []labelRow {
	t.Helper()
	b, err := os.ReadFile(path)
	if err != nil {
		t.Fatalf("read %s: %v", path, err)
	}
	var out []labelRow
	for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
		if strings.TrimSpace(line) == "" {
			continue
		}
		var r labelRow
		if err := json.Unmarshal([]byte(line), &r); err != nil {
			t.Fatalf("parse %s: %v (%q)", path, err, line)
		}
		out = append(out, r)
	}
	return out
}

// buildCheckers compiles the real zh-ru checker spec exactly as the runner does.
func buildCheckers(t *testing.T) *Checkers {
	t.Helper()
	pack, err := lang.Load(langpackRoot, "zh", "ru")
	if err != nil {
		t.Fatalf("load langpack: %v", err)
	}
	c := CompileCheckersFor(pack, lang.TargetChecksFor("ru"))
	c.SetSourceScripts(lang.LangScripts("zh"))
	return c
}

func loadCorpus(t *testing.T) map[string]corpusUnit {
	t.Helper()
	b, err := os.ReadFile(filepath.Join(labelsDir, "raw", "corpus.jsonl"))
	if err != nil {
		t.Fatalf("read corpus: %v", err)
	}
	units := map[string]corpusUnit{}
	for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
		if strings.TrimSpace(line) == "" {
			continue
		}
		var u corpusUnit
		if err := json.Unmarshal([]byte(line), &u); err != nil {
			t.Fatalf("parse corpus: %v", err)
		}
		units[u.id()] = u
	}
	return units
}

// unitFlag maps every unit to whether a per-unit checker sub-counter fired, given a picker.
func unitFlags(units map[string]corpusUnit, c *Checkers, pick func(CheapGateResult) int, regression bool) map[string]bool {
	out := make(map[string]bool, len(units))
	cfg := CheapGateConfig{Checkers: c, RegressionEnabled: regression}
	for id, u := range units {
		// Production runs the cheap gates on finalText BEFORE chunk.ApplyHeading prepends the
		// deterministic «Глава N» manifest heading (waverun.go:547 / export.go:215 run AFTER
		// RunCheapGates on waverun.go:501). The corpus baked that heading in, so strip it back off —
		// otherwise the chapter number reads as an "appeared" number to the regression guard (the
		// README's documented trap). Old-stack acceptance headings ("Раздел N: title") are baked BODY
		// text, not the bare manifest heading, so they are left in place — exactly the package-6 baseline.
		r := RunCheapGates(u.Source, u.Draft, stripManifestHeading(u.Final), cfg)
		out[id] = pick(r) > 0
	}
	return out
}

// stripManifestHeading removes a leading bare «Глава N» line (+ the following blank line) — the
// deterministic manifest heading current code applies after the gates. A titled heading
// («Глава N: …» / «Раздел N: …») is left untouched (old-stack baked body text).
func stripManifestHeading(final string) string {
	lines := strings.Split(final, "\n")
	if len(lines) == 0 {
		return final
	}
	l0 := strings.TrimSpace(lines[0])
	rest, ok := strings.CutPrefix(l0, "Глава ")
	if !ok {
		return final
	}
	for _, r := range rest {
		if r < '0' || r > '9' {
			return final // has a title after the number → not the bare manifest heading
		}
	}
	i := 1
	for i < len(lines) && strings.TrimSpace(lines[i]) == "" {
		i++
	}
	return strings.Join(lines[i:], "\n")
}

// unitLevelMetric aggregates occurrence labels to their unit (priority defect>disputable>ok) and joins to
// the unit checker flag. labelPick selects the label field (label / label_code_convention).
func unitLevelMetric(labels []labelRow, flags map[string]bool, labelPick func(labelRow) string) metric {
	type acc struct{ hasDefect, hasDisp bool }
	byUnit := map[string]*acc{}
	for _, l := range labels {
		u := unitOf(l.ID)
		a := byUnit[u]
		if a == nil {
			a = &acc{}
			byUnit[u] = a
		}
		switch labelPick(l) {
		case "defect":
			a.hasDefect = true
		case "disputable":
			a.hasDisp = true
		}
	}
	var m metric
	for u, a := range byUnit {
		switch {
		case a.hasDefect:
			m.add(flags[u], true)
		case a.hasDisp:
			m.Disp++
		default:
			m.add(flags[u], false)
		}
	}
	return m
}

func TestCheckerLabelsBaseline(t *testing.T) {
	if _, err := os.Stat(filepath.Join(labelsDir, "raw", "corpus.jsonl")); err != nil {
		if os.Getenv("TM_CHECKER_LABELS") == "1" {
			t.Fatalf("TM_CHECKER_LABELS=1 but corpus is missing: %v", err)
		}
		t.Skipf("labelled corpus absent (%v) — set TM_CHECKER_LABELS=1 to force", err)
	}

	c := buildCheckers(t)
	units := loadCorpus(t)
	lbl := func(name string) []labelRow { return loadJSONL(t, filepath.Join(labelsDir, "labels", name)) }
	pickLabel := func(l labelRow) string { return l.Label }

	results := map[string]metric{}

	// Unit-level src↔target classes.
	results["K1_unit"] = unitLevelMetric(lbl("k1.jsonl"),
		unitFlags(units, c, func(r CheapGateResult) int { return r.DC1TimeUnits }, false), pickLabel)
	results["k5a_unit"] = unitLevelMetric(lbl("k5a.jsonl"),
		unitFlags(units, c, func(r CheapGateResult) int { return r.NumberMagnitude }, false), pickLabel)
	results["k5b_unit"] = unitLevelMetric(lbl("k5b.jsonl"),
		unitFlags(units, c, func(r CheapGateResult) int { return r.DC2Magnitude }, false), pickLabel)
	percentFlags := unitFlags(units, c, func(r CheapGateResult) int { return r.PercentScale }, false)
	results["k5c_unit"] = unitLevelMetric(lbl("k5c.jsonl"), percentFlags, pickLabel)
	results["k5c_unit_codeconv"] = unitLevelMetric(lbl("k5c.jsonl"), percentFlags,
		func(l labelRow) string {
			if l.LabelCC != "" {
				return l.LabelCC
			}
			return l.Label
		})

	// K2 — per token over the labelled field.
	results["k2"] = measureK2(lbl("k2.jsonl"), units, c)
	// K3 — per word-type.
	results["k3"] = measureK3(lbl("k3.jsonl"), c)
	// K4b / k4_inverse — per line.
	results["k4b"], results["k4_inverse"] = measureK4(lbl("k4.jsonl"), units, c)
	// K4a — the marker-defect class over every k4.jsonl line (all-TN), measured marker-only so the inverseMarker
	// fold cannot leak into it.
	results["k4a"] = measureK4a(lbl("k4.jsonl"), units)
	// K5d — number drift (regression guard ON).
	results["k5d"] = measureK5d(lbl("k5d.jsonl"), units, c)

	printMetrics(t, results)
	writeMetricsJSON(t, results)
	assertBaseline(t, results)
}

// measureK2 runs lintLatinResidue on the labelled field's text and flags a token if its surface is a hit.
func measureK2(labels []labelRow, units map[string]corpusUnit, c *Checkers) metric {
	var m metric
	for _, l := range labels {
		u, ok := units[unitOf(l.ID)]
		if !ok {
			continue
		}
		text := u.Final
		if l.Field == "draft" {
			text = u.Draft
		}
		_, det := lintLatinResidue(text, nil)
		hit := false
		for _, d := range det {
			// detail line: "Latin word left untranslated in the target output: a, b, c"
			if idx := strings.Index(d, ": "); idx >= 0 {
				for _, s := range strings.Split(d[idx+2:], ", ") {
					if s == l.Token {
						hit = true
					}
				}
			}
		}
		joinAdd(&m, l.Label, hit)
	}
	return m
}

// measureK3 runs the broken-word lint on each labelled word-type.
func measureK3(labels []labelRow, c *Checkers) metric {
	var m metric
	for _, l := range labels {
		n, _ := c.lintBrokenWord(l.Word)
		joinAdd(&m, l.Label, n > 0)
	}
	return m
}

// measureK4 reproduces the per-line chevron-speech verdict (K4b) and the inverse dash-thought recall
// (k4_inverse). The chevron flag is the per-line predicate chevronSpeechShape && isSpokenChevronLine —
// the "detector" measurement, in isolation from the unit-level em-dash mixing count (tech-debt anchor #3
// is exactly about this per-line shape). k4_inverse has no rule (a thought typeset with a dash is
// structurally invisible) so it is never flagged — a pure recall gap.
func measureK4(labels []labelRow, units map[string]corpusUnit, c *Checkers) (metric, metric) {
	var k4b, k4inv metric
	for _, l := range labels {
		switch l.Bucket {
		case "chevron":
			line := strings.TrimLeft(lineForLabel(l, units), " \t　")
			flagged := chevronSpeechShape([]rune(line)) && c.isSpokenChevronLine(line)
			joinK4(&k4b, l.Speech, "spoken", flagged)
		case "em-dash":
			// k4_inverse (D39.39): a dash line whose ATTRIBUTION carries an inner-speech marker is a thought
			// typeset as spoken — scoped to the attribution (matches production speechAttribution), not the
			// whole line, so a spoken reply that merely mentions «про себя» is not flagged.
			line := strings.TrimLeft(lineForLabel(l, units), " \t　")
			rs := []rune(line)
			isDash := len(rs) > 0 && (rs[0] == '—' || rs[0] == '–' || rs[0] == '-')
			attr := speechAttribution(line)
			joinK4(&k4inv, l.Speech, "thought", isDash && attr != "" && c.lineHasInnerMarker(attr))
		}
	}
	return k4b, k4inv
}

// measureK4a reproduces the metrics.json k4a class: the dialogue-dash MARKER defect (a line opened with a
// straight quote or a hyphen/en-dash where the em-dash belongs) over EVERY k4.jsonl line (all buckets, n=348).
// It reuses the production leading-marker predicates (quoteShape/dialogueShape) but is MARKER-ONLY: the correct
// em-dash «—» is never a defect, so the k4_inverse build (inverseMarker folded into the DialogueDash counter)
// cannot flip an em-dash thought line into this class. The corpus carries no such defect → 0/0/0/348.
func measureK4a(labels []labelRow, units map[string]corpusUnit) metric {
	var m metric
	for _, l := range labels {
		rs := []rune(strings.TrimLeft(lineForLabel(l, units), " \t　"))
		defect := false
		if len(rs) > 0 {
			switch rs[0] {
			case '"':
				defect = quoteShape(rs[1:])
			case '-', '–':
				defect = dialogueShape(rs[1:])
			}
		}
		m.add(defect, false) // k4a labels every corpus dialogue line as non-defect for the marker class
	}
	return m
}

// lineForLabel extracts the labelled line from the unit by its :field:line_no id tail.
func lineForLabel(l labelRow, units map[string]corpusUnit) string {
	p := strings.Split(l.ID, ":")
	if len(p) < 6 {
		return ""
	}
	field := p[4]
	var lineNo int
	fmt.Sscanf(p[5], "%d", &lineNo)
	u, ok := units[unitOf(l.ID)]
	if !ok {
		return ""
	}
	text := u.Final
	if field == "draft" {
		text = u.Draft
	}
	lines := strings.Split(text, "\n")
	if lineNo >= 0 && lineNo < len(lines) {
		return lines[lineNo]
	}
	return ""
}

// measureK5d runs the regression guard (draft→final number drift) per unit and joins to position labels.
func measureK5d(labels []labelRow, units map[string]corpusUnit, c *Checkers) metric {
	flags := unitFlags(units, c, func(r CheapGateResult) int { return r.NumberDrift }, true)
	var m metric
	for _, l := range labels {
		joinAdd(&m, l.Label, flags[unitOf(l.ID)])
	}
	return m
}

func joinAdd(m *metric, label string, pred bool) {
	switch label {
	case "disputable":
		m.Disp++
	case "defect":
		m.add(pred, true)
	default:
		m.add(pred, false)
	}
}

// joinK4 uses the K4 label vocabulary: ambiguous=disputable; the defectSpeech value (spoken for K4b, thought
// for the inverse) is the defect; every other resolvable speech is ok.
func joinK4(m *metric, speech, defectSpeech string, pred bool) {
	switch speech {
	case "ambiguous":
		m.Disp++
	default:
		m.add(pred, speech == defectSpeech)
	}
}

func printMetrics(t *testing.T, results map[string]metric) {
	keys := make([]string, 0, len(results))
	for k := range results {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	t.Logf("%-20s %4s %4s %4s %4s %4s %5s | %-8s %-8s", "class", "n", "tp", "fp", "fn", "tn", "disp", "prec", "recall")
	for _, k := range keys {
		m := results[k]
		t.Logf("%-20s %4d %4d %4d %4d %4d %5d | %-8s %-8s", k, m.n(), m.TP, m.FP, m.FN, m.TN, m.Disp, m.precision(), m.recall())
	}
}

func writeMetricsJSON(t *testing.T, results map[string]metric) {
	out := os.Getenv("TM_CHECKER_LABELS_OUT")
	if out == "" {
		return
	}
	m := map[string]any{}
	for k, v := range results {
		m[k] = map[string]int{"n": v.n(), "tp": v.TP, "fp": v.FP, "fn": v.FN, "tn": v.TN, "disputable": v.Disp}
	}
	b, _ := json.MarshalIndent(m, "", " ")
	if err := os.WriteFile(out, b, 0o644); err != nil {
		t.Logf("write metrics out: %v", err)
	}
}

// assertBaseline pins each class to its expected value on the CURRENT code. Classes untouched by the
// checker package hold their ratified package-6 metrics.json values; the two classes this package moved
// (k2, k4b) pin the new measured after-values with the delta recorded below. k6 lives in the membank harness.
func assertBaseline(t *testing.T, r map[string]metric) {
	want := map[string]metric{
		"K1_unit":           {TP: 6, FP: 0, FN: 0, TN: 9},           // unchanged (metrics.json)
		"k3":                {TP: 1, FP: 0, FN: 9, TN: 2167},        // unchanged (one «-йть» pattern; combmark did not move it)
		"k5a_unit":          {TP: 0, FP: 0, FN: 1, TN: 26, Disp: 3}, // #5 CLOSED: the 1 miss is DC2-covered (k5b)
		"k5b_unit":          {TP: 1, FP: 0, FN: 0, TN: 7, Disp: 3},  // unchanged
		"k5c_unit":          {TP: 1, FP: 5, FN: 1, TN: 8, Disp: 1},  // #1: PercentScale reclassified SOFT (Р2); detection unchanged
		"k5c_unit_codeconv": {TP: 7, FP: 0, FN: 1, TN: 8},           // unchanged (#8 两 masked by suppressor)
		// #6 caps-aware Latin: recall 0.75→0.85 (BANK×2, Cultivation×2 now caught). MEASURED precision 0.971:
		// the 1 fp is «Qidian» (a brand). 1.0 is achievable only IF the book config allowlists it — no committed
		// gu-zhenren config currently does, so the honest measured number is 0.971, not 1.0.
		"k2": {TP: 34, FP: 1, FN: 6, TN: 51},
		// #3 attribution join extended to «X!»/«X?»/«X…»: recall 0.109→0.217, precision 0.833→0.909, +5 tp,
		// ZERO new fp. Residual 36 misses = 24 unattributed replies (no dash join this shape can see) + 12
		// attributed with a verb OUTSIDE speech_verb (isSpokenChevronLine silent) — Р4 ceiling ~0.22, inline
		// in chevronSpeechShape.
		"k4b": {TP: 10, FP: 1, FN: 36, TN: 99, Disp: 4},
		// k4_inverse (D39.39 build, inner_marker-veto-on-dash): dash line + inner-speech marker → thought
		// typeset as spoken. recall 0→0.75 (+3 tp), precision 1.0, 0 fp. The 1 residual miss uses a marker
		// not in the target list (recall ceiling — extend the data, not the rule).
		"k4_inverse": {TP: 3, FP: 0, FN: 1, TN: 71, Disp: 5},
		"k5d":        {TP: 0, FP: 4, FN: 0, TN: 1}, // unchanged (manifest-heading strip)
		// k4a: the dialogue-dash MARKER class over all 348 k4.jsonl lines, all-TN. Measured marker-only, so the
		// inverseMarker fold into DialogueDash cannot leak the 3 inverse fires into it (metrics.json: 0/0/0/348).
		"k4a": {TP: 0, FP: 0, FN: 0, TN: 348},
	}
	for k, w := range want {
		got := r[k]
		if got != w {
			t.Errorf("expected drift %s: got %+v want %+v", k, got, w)
		}
	}
}