textmachine/backend/internal/pipeline/live_reprobe_test.go

//go:build live

package pipeline

// live_reprobe_test.go: the PAID acceptance probe of the §2 type-classifier (D39.69 §2 / row 74).
// Build-tagged `live` like live_conformance_test.go, so `go test ./...` never compiles it.
//
//	TM_LIVE=1 TM_CLASSIFY6_CONFIG=~/books/gu-zhenren/coldrun-b/reprobe/classify6/book.yaml \
//	  go test -tags live -run TestLiveClassifierHarmSet -v ./internal/pipeline/
//
// It drives the PRODUCTION classifier path end to end — the pair's own resolved prompt template, the
// production `classifierMessages` render, the production `bankCallBudget` sizing and, crucially, the
// production `runBankAttempt` money sequence (reserve → call → settle+checkpoint). It calls no provider
// itself: the single-egress guard (TestProviderEgressSeamIsSingle) is a real invariant, and a probe that
// bypassed the seam would also bypass the ledger — its cost would then be invisible to `tmctl report`,
// which is the opposite of what a paid probe owes. Everything it reports is therefore re-readable from
// the probe project's own store.
//
// The candidates are primed with the WRONG draft type on purpose: the harm this phase exists to fix is a
// realia mis-typed `name` and then transliterated into a meaningless syllable (元石 → «юаньши»), so a probe
// that primed the RIGHT type would also be passed by a model that merely echoes its input.

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/llm"
	"textmachine/backend/internal/obs"
	"textmachine/backend/internal/terminology"
	"textmachine/backend/internal/text"
)

// harmSet is the six terms of the ratified acceptance threshold (BANK_QUALITY_DESIGN §2): every one of
// them is a THING or a facility that must be translated by sense, and every one of them was mis-typed by
// the draft heuristic. Each carries a real ctx line from the stand book, because the classifier prompt
// tells the model to decide by context rather than by the `type:` guess.
//
// ⚠ HONEST LIMIT of the 6/6 number: 元石 and 灵泉 appear VERBATIM as `term` examples inside
// prompts/zh-ru/classifier.md, so two of the six are recall, not judgement. The discriminating subset is
// 元海 / 蛊室 / 池塘 / 酒肆 — read the result as 4 decided + 2 recalled.
var harmSet = []terminology.Candidate{
	{Key: "元石", Src: "元石", Type: "name", Freq: 12, Origin: terminology.OriginBanknote,
		KWIC: []string{"谁能拔得头筹，就会有二十块元石的丰厚奖励"}},
	{Key: "元海", Src: "元海", Type: "name", Freq: 9, Origin: terminology.OriginBanknote,
		KWIC: []string{"察看自身的空窍元海"}},
	{Key: "蛊室", Src: "蛊室", Type: "place", Freq: 5, Origin: terminology.OriginBanknote,
		KWIC: []string{"你们就去学堂里的蛊室，挑选蛊虫"}},
	{Key: "池塘", Src: "池塘", Type: "place", Freq: 4, Origin: terminology.OriginBanknote,
		KWIC: []string{"村外的池塘边上，长满了青草"}},
	{Key: "灵泉", Src: "灵泉", Type: "name", Freq: 3, Origin: terminology.OriginBanknote,
		KWIC: []string{"灵泉之水，可以洗练身躯"}},
	{Key: "酒肆", Src: "酒肆", Type: "place", Freq: 3, Origin: terminology.OriginBanknote,
		KWIC: []string{"镇上的酒肆里，说书人正讲着蛊师的故事"}},
}

func TestLiveClassifierHarmSet(t *testing.T) {
	if os.Getenv("TM_LIVE") != "1" {
		t.Skip("paid probe: set TM_LIVE=1")
	}
	cfg := os.Getenv("TM_CLASSIFY6_CONFIG")
	if cfg == "" {
		t.Skip("set TM_CLASSIFY6_CONFIG to a book.yaml whose gates.terminology.classify_types is on")
	}
	r, err := NewRunner(cfg, obs.NewLogger())
	if err != nil {
		t.Fatalf("build runner: %v", err)
	}
	defer r.Close()
	if r.classifierTemplate == nil {
		t.Fatalf("classify_types is off in %s — the probe would measure nothing", cfg)
	}
	// The eager client map is filled by the precompute pass inside TranslateBook, which this probe does
	// not run; build it the same way rather than teaching clientFor about a probe.
	if err := r.buildClients(); err != nil {
		t.Fatalf("build clients: %v", err)
	}

	msgs, err := r.classifierMessages(harmSet)
	if err != nil {
		t.Fatalf("render production classifier messages: %v", err)
	}
	// The stage comes from the PRODUCTION derivation, not a literal: a probe that hand-built its stage would
	// measure a shape the engine never sends — and once the effort knob landed, its green 6/6 would say
	// nothing about the level the bank roles actually ride (D39.87 §0.2, the fourth synthetic-stage site).
	st := r.bankStage(roleClassifier)
	_, maxTokens := r.bankCallBudget(st.Model, msgs)

	// N SAMPLES, not one. The verdict this probe produces is a judgement call by a stochastic model, and a
	// single call cannot tell a level difference from ordinary variance — reading n=1 as a measurement is
	// exactly the manufactured-convergence failure this repo has paid for before. Each run gets its OWN
	// snapshot axis so it is a FRESH call; a repeat of the same index still replays that run for $0.
	// Default 5, not 1. The comment above condemns n=1 and the first version of this rig then DEFAULTED to
	// it — so the next person to run it would have got exactly the reading it warns against. A run costs
	// ~$0.0002-0.0009, so five samples are free at the scale of any probe that would bother running this.
	runs := 5
	if v := os.Getenv("TM_CLASSIFY6_N"); v != "" {
		if n, cerr := strconv.Atoi(v); cerr == nil && n > 0 {
			runs = n
		}
	}
	effort := st.Reasoning
	if effort == "" {
		effort = "unset"
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
	defer cancel()

	type sample struct {
		Run                int               `json:"run"`
		Finish             string            `json:"finish"`
		Verdict            string            `json:"verdict"`
		Usage              llm.Usage         `json:"usage"`
		CostUSD            float64           `json:"cost_usd"`
		LatencyMS          int64             `json:"latency_ms"`
		Reply              string            `json:"reply"`
		Parsed             map[string]string `json:"parsed"`
		BadLines           int               `json:"bad_lines"`
		Hits               int               `json:"hits_term"`
		DiscriminatingHits int               `json:"discriminating_hits"`
	}
	samples := make([]sample, 0, runs)
	full, totalCost := 0, 0.0
	for i := 0; i < runs; i++ {
		snapID := fmt.Sprintf("reprobe-classifier-6of6-%s-%d", effort, i)
		// jobs.snapshot_id is a FOREIGN KEY into snapshots — the production path upserts the wave snapshot
		// before opening jobs, so the probe registers its own axis the same way instead of borrowing one.
		if err := r.Store.UpsertSnapshot(snapID, "", `{"probe":"classifier-harm-set-6of6","effort":"`+effort+`"}`); err != nil {
			t.Fatalf("register the probe snapshot axis: %v", err)
		}
		job, jerr := r.Store.EnsureJob(r.Book.BookID, 0, terminologyStageName, snapID)
		if jerr != nil {
			t.Fatalf("ensure job: %v", jerr)
		}
		start := time.Now()
		att, aerr := r.runBankAttempt(ctx, st, snapID, chunk.Chunk{Chapter: 0, ChunkIdx: 0}, job, msgs)
		if aerr != nil {
			t.Fatalf("live classifier call %d failed: %v", i, aerr)
		}
		got, stats := terminology.ParseTypes(att.text, candKeys(harmSet), text.NormalizeSourceKey)
		hits, decided := 0, 0
		for _, c := range harmSet {
			if got[c.Key] != "term" {
				continue
			}
			hits++
			if c.Key != "元石" && c.Key != "灵泉" { // the two the prompt names verbatim
				decided++
			}
		}
		if hits == len(harmSet) {
			full++
		}
		totalCost += att.runCost
		samples = append(samples, sample{i, att.finish, string(att.cls.Reason), att.usage, att.runCost,
			time.Since(start).Milliseconds(), att.text, got, stats.Bad, hits, decided})
		t.Logf("CLASSIFIER 6/6 [effort=%s run=%d/%d]: model=%s max_tokens=%d finish=%q verdict=%q latency=%dms usage=%+v cost_usd=%.6f bad_lines=%d hits=%d/6 (discriminating %d/4) got=%v",
			effort, i+1, runs, att.modelActual, maxTokens, att.finish, att.cls.Reason,
			samples[i].LatencyMS, att.usage, att.runCost, stats.Bad, hits, decided, got)
	}
	t.Logf("CLASSIFIER 6/6 SUMMARY [effort=%s]: %d/%d runs reached the 6/6 threshold · total_cost_usd=%.6f", effort, full, runs, totalCost)

	// Persist the raw wire evidence next to the probe project — "persist, not scratch": every number in
	// the report must be re-readable from the artifact that produced it. The file is per-EFFORT, so a
	// comparison between levels does not overwrite its own other half.
	blob, _ := json.MarshalIndent(map[string]any{
		"model": st.Model, "effort": effort, "max_tokens": maxTokens,
		"runs": runs, "runs_at_threshold": full, "total_cost_usd": totalCost, "samples": samples,
	}, "", "  ")
	if werr := os.WriteFile(filepath.Join(filepath.Dir(cfg), "classifier-6of6-"+effort+".json"), blob, 0o644); werr != nil {
		t.Errorf("persist probe evidence: %v", werr)
	}

	// ⚠ THE THRESHOLD AND THE SAMPLE DISAGREE, and that is an OWNER decision, not something to soften here.
	// D39.69 §2 ratified "6/6" against a SINGLE call; nobody defined it over a sample. Measured 02.08 on
	// live 0731 weights: effort `low` reaches 6/6 on 4 of 5 runs, `high` on 5 of 5 — a difference n=5 cannot
	// separate — while `high` costs 2.30x per call. So the level this engine now recommends for the bank
	// roles makes this gate RED. Reporting that honestly is the point; relaxing the gate to fit the
	// recommendation would be fitting the acceptance criterion to the result.
	if full != runs {
		t.Fatalf("the ratified acceptance threshold is 6/6 term (D39.69 §2) and it was reached on %d of %d runs "+
			"at effort %q — see classifier-6of6-%s.json. ⚠ The threshold was defined for ONE call, not for a "+
			"sample: decide with the owner whether it means \"every run\" or \"the median run\" before treating "+
			"this as a regression.", full, runs, effort, effort)
	}
}