204 lines
11 KiB
Go
204 lines
11 KiB
Go
//go:build live
|
||
|
||
package pipeline
|
||
|
||
// live_reprobe_test.go: the PAID acceptance probe of the §2 type-classifier (D39.69 §2 / row 74).
|
||
// Build-tagged `live` like live_conformance_test.go, so `go test ./...` never compiles it.
|
||
//
|
||
// TM_LIVE=1 TM_CLASSIFY6_CONFIG=~/books/gu-zhenren/coldrun-b/reprobe/classify6/book.yaml \
|
||
// go test -tags live -run TestLiveClassifierHarmSet -v ./internal/pipeline/
|
||
//
|
||
// It drives the PRODUCTION classifier path end to end — the pair's own resolved prompt template, the
|
||
// production `classifierMessages` render, the production `bankCallBudget` sizing and, crucially, the
|
||
// production `runBankAttempt` money sequence (reserve → call → settle+checkpoint). It calls no provider
|
||
// itself: the single-egress guard (TestProviderEgressSeamIsSingle) is a real invariant, and a probe that
|
||
// bypassed the seam would also bypass the ledger — its cost would then be invisible to `tmctl report`,
|
||
// which is the opposite of what a paid probe owes. Everything it reports is therefore re-readable from
|
||
// the probe project's own store.
|
||
//
|
||
// The candidates are primed with the WRONG draft type on purpose: the harm this phase exists to fix is a
|
||
// realia mis-typed `name` and then transliterated into a meaningless syllable (元石 → «юаньши»), so a probe
|
||
// that primed the RIGHT type would also be passed by a model that merely echoes its input.
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strconv"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/llm"
|
||
"textmachine/backend/internal/obs"
|
||
"textmachine/backend/internal/terminology"
|
||
"textmachine/backend/internal/text"
|
||
)
|
||
|
||
// harmSet is the six terms of the ratified acceptance threshold (BANK_QUALITY_DESIGN §2): every one of
|
||
// them is a THING or a facility that must be translated by sense, and every one of them was mis-typed by
|
||
// the draft heuristic. Each carries a real ctx line from the stand book, because the classifier prompt
|
||
// tells the model to decide by context rather than by the `type:` guess.
|
||
//
|
||
// ⚠ HONEST LIMIT of the 6/6 number: 元石 and 灵泉 appear VERBATIM as `term` examples inside
|
||
// prompts/zh-ru/classifier.md, so two of the six are recall, not judgement. The discriminating subset is
|
||
// 元海 / 蛊室 / 池塘 / 酒肆 — read the result as 4 decided + 2 recalled.
|
||
var harmSet = []terminology.Candidate{
|
||
{Key: "元石", Src: "元石", Type: "name", Freq: 12, Origin: terminology.OriginBanknote,
|
||
KWIC: []string{"谁能拔得头筹，就会有二十块元石的丰厚奖励"}},
|
||
{Key: "元海", Src: "元海", Type: "name", Freq: 9, Origin: terminology.OriginBanknote,
|
||
KWIC: []string{"察看自身的空窍元海"}},
|
||
{Key: "蛊室", Src: "蛊室", Type: "place", Freq: 5, Origin: terminology.OriginBanknote,
|
||
KWIC: []string{"你们就去学堂里的蛊室，挑选蛊虫"}},
|
||
{Key: "池塘", Src: "池塘", Type: "place", Freq: 4, Origin: terminology.OriginBanknote,
|
||
KWIC: []string{"村外的池塘边上，长满了青草"}},
|
||
{Key: "灵泉", Src: "灵泉", Type: "name", Freq: 3, Origin: terminology.OriginBanknote,
|
||
KWIC: []string{"灵泉之水，可以洗练身躯"}},
|
||
{Key: "酒肆", Src: "酒肆", Type: "place", Freq: 3, Origin: terminology.OriginBanknote,
|
||
KWIC: []string{"镇上的酒肆里，说书人正讲着蛊师的故事"}},
|
||
}
|
||
|
||
func TestLiveClassifierHarmSet(t *testing.T) {
|
||
if os.Getenv("TM_LIVE") != "1" {
|
||
t.Skip("paid probe: set TM_LIVE=1")
|
||
}
|
||
cfg := os.Getenv("TM_CLASSIFY6_CONFIG")
|
||
if cfg == "" {
|
||
t.Skip("set TM_CLASSIFY6_CONFIG to a book.yaml whose gates.terminology.classify_types is on")
|
||
}
|
||
r, err := NewRunner(cfg, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatalf("build runner: %v", err)
|
||
}
|
||
defer r.Close()
|
||
if r.classifierTemplate == nil {
|
||
t.Fatalf("classify_types is off in %s — the probe would measure nothing", cfg)
|
||
}
|
||
// The eager client map is filled by the precompute pass inside TranslateBook, which this probe does
|
||
// not run; build it the same way rather than teaching clientFor about a probe.
|
||
if err := r.buildClients(); err != nil {
|
||
t.Fatalf("build clients: %v", err)
|
||
}
|
||
|
||
msgs, err := r.classifierMessages(harmSet)
|
||
if err != nil {
|
||
t.Fatalf("render production classifier messages: %v", err)
|
||
}
|
||
// The stage comes from the PRODUCTION derivation, not a literal: a probe that hand-built its stage would
|
||
// measure a shape the engine never sends — and once the effort knob landed, its green 6/6 would say
|
||
// nothing about the level the bank roles actually ride (D39.87 §0.2, the fourth synthetic-stage site).
|
||
st := r.bankStage(roleClassifier)
|
||
_, maxTokens := r.bankCallBudget(st.Model, msgs)
|
||
|
||
// N SAMPLES, not one. The verdict this probe produces is a judgement call by a stochastic model, and a
|
||
// single call cannot tell a level difference from ordinary variance — reading n=1 as a measurement is
|
||
// exactly the manufactured-convergence failure this repo has paid for before. Each run gets its OWN
|
||
// snapshot axis so it is a FRESH call; a repeat of the same index still replays that run for $0.
|
||
// Default 5, not 1. The comment above condemns n=1 and the first version of this rig then DEFAULTED to
|
||
// it — so the next person to run it would have got exactly the reading it warns against. A run costs
|
||
// ~$0.0002-0.0009, so five samples are free at the scale of any probe that would bother running this.
|
||
runs := classify6SampleN
|
||
if v := os.Getenv("TM_CLASSIFY6_N"); v != "" {
|
||
// A malformed override is REFUSED, not ignored: silently falling back to the ratified 5 would buy
|
||
// five paid calls for an operator who believes they asked for something else, one line above the
|
||
// guard that exists to stop exactly that.
|
||
n, cerr := strconv.Atoi(v)
|
||
if cerr != nil || n <= 0 {
|
||
t.Fatalf("TM_CLASSIFY6_N=%q is not a positive integer — refusing rather than quietly running the default N=%d and billing for it", v, classify6SampleN)
|
||
}
|
||
runs = n
|
||
}
|
||
// The size guard fires BEFORE the first paid call, not after: an override the threshold does not
|
||
// cover must cost nothing. The refusal is LOUD rather than a skip — a watch run that quietly did not
|
||
// measure is indistinguishable from one that passed.
|
||
if ok, why := classify6SampleRatified(runs); !ok {
|
||
t.Fatalf("TM_CLASSIFY6_N=%d: %s. Nothing was called and nothing was billed. Run the ratified N=%d, or ask the owner for a threshold at your N",
|
||
runs, why, classify6SampleN)
|
||
}
|
||
effort := st.Reasoning
|
||
if effort == "" {
|
||
effort = "unset"
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||
defer cancel()
|
||
|
||
type sample struct {
|
||
Run int `json:"run"`
|
||
Finish string `json:"finish"`
|
||
Verdict string `json:"verdict"`
|
||
Usage llm.Usage `json:"usage"`
|
||
CostUSD float64 `json:"cost_usd"`
|
||
LatencyMS int64 `json:"latency_ms"`
|
||
Reply string `json:"reply"`
|
||
Parsed map[string]string `json:"parsed"`
|
||
BadLines int `json:"bad_lines"`
|
||
Hits int `json:"hits_term"`
|
||
DiscriminatingHits int `json:"discriminating_hits"`
|
||
}
|
||
samples := make([]sample, 0, runs)
|
||
full, totalCost := 0, 0.0
|
||
for i := 0; i < runs; i++ {
|
||
snapID := fmt.Sprintf("reprobe-classifier-6of6-%s-%d", effort, i)
|
||
// jobs.snapshot_id is a FOREIGN KEY into snapshots — the production path upserts the wave snapshot
|
||
// before opening jobs, so the probe registers its own axis the same way instead of borrowing one.
|
||
if err := r.Store.UpsertSnapshot(snapID, "", `{"probe":"classifier-harm-set-6of6","effort":"`+effort+`"}`); err != nil {
|
||
t.Fatalf("register the probe snapshot axis: %v", err)
|
||
}
|
||
job, jerr := r.Store.EnsureJob(r.Book.BookID, 0, terminologyStageName, snapID)
|
||
if jerr != nil {
|
||
t.Fatalf("ensure job: %v", jerr)
|
||
}
|
||
start := time.Now()
|
||
att, aerr := r.runBankAttempt(ctx, st, snapID, chunk.Chunk{Chapter: 0, ChunkIdx: 0}, job, msgs)
|
||
if aerr != nil {
|
||
t.Fatalf("live classifier call %d failed: %v", i, aerr)
|
||
}
|
||
got, stats := terminology.ParseTypes(att.text, candKeys(harmSet), text.NormalizeSourceKey)
|
||
hits, decided := 0, 0
|
||
for _, c := range harmSet {
|
||
if got[c.Key] != "term" {
|
||
continue
|
||
}
|
||
hits++
|
||
if c.Key != "元石" && c.Key != "灵泉" { // the two the prompt names verbatim
|
||
decided++
|
||
}
|
||
}
|
||
if hits == len(harmSet) {
|
||
full++
|
||
}
|
||
totalCost += att.runCost
|
||
samples = append(samples, sample{i, att.finish, string(att.cls.Reason), att.usage, att.runCost,
|
||
time.Since(start).Milliseconds(), att.text, got, stats.Bad, hits, decided})
|
||
t.Logf("CLASSIFIER 6/6 [effort=%s run=%d/%d]: model=%s max_tokens=%d finish=%q verdict=%q latency=%dms usage=%+v cost_usd=%.6f bad_lines=%d hits=%d/6 (discriminating %d/4) got=%v",
|
||
effort, i+1, runs, att.modelActual, maxTokens, att.finish, att.cls.Reason,
|
||
samples[i].LatencyMS, att.usage, att.runCost, stats.Bad, hits, decided, got)
|
||
}
|
||
t.Logf("CLASSIFIER 6/6 SUMMARY [effort=%s]: %d/%d runs reached the 6/6 threshold · total_cost_usd=%.6f", effort, full, runs, totalCost)
|
||
|
||
// Persist the raw wire evidence next to the probe project — "persist, not scratch": every number in
|
||
// the report must be re-readable from the artifact that produced it. The file is per-EFFORT, so a
|
||
// comparison between levels does not overwrite its own other half.
|
||
blob, _ := json.MarshalIndent(map[string]any{
|
||
"model": st.Model, "effort": effort, "max_tokens": maxTokens,
|
||
"runs": runs, "runs_at_threshold": full, "total_cost_usd": totalCost, "samples": samples,
|
||
}, "", " ")
|
||
if werr := os.WriteFile(filepath.Join(filepath.Dir(cfg), "classifier-6of6-"+effort+".json"), blob, 0o644); werr != nil {
|
||
t.Errorf("persist probe evidence: %v", werr)
|
||
}
|
||
|
||
// The threshold is the SAMPLED form ratified by D39.136 п.6б (row 116 closed by delegation): «≥4 of 5
|
||
// runs at N=5 reach 6/6». It replaced D39.69 §2's "6/6", which had been defined against a SINGLE call
|
||
// and which this rig used to apply per run — the open question the rig declared here is CLOSED, and
|
||
// the closure was the owner's, not a softening by whoever ran the probe. What the sampled form buys:
|
||
// the 02.08 measurement on live 0731 weights (effort `low` 4 of 5, `high` 5 of 5, a difference n=5
|
||
// cannot separate, `high` 2.30x per call) is a PASS for the level the engine actually recommends for
|
||
// the bank roles, instead of a red gate that says nothing about the model.
|
||
if ok, why := classify6Accepted(runs, full); !ok {
|
||
t.Fatalf("classifier acceptance FAILED: %s — 6/6 was reached on %d of %d runs at effort %q "+
|
||
"(ratified threshold D39.136 п.6б: ≥%d of %d). Raw evidence: classifier-6of6-%s.json",
|
||
why, full, runs, effort, classify6RunsAtFullOK, classify6SampleN, effort)
|
||
}
|
||
}
|