Add banknote and voice observability lines to tmctl report, fix live-tagged rig build and add classifier threshold probe
This commit is contained in:
Claude (backend session)
parent
e9a6bb2711
commit
1ecf5c2f1a
4 changed files with 286 additions and 9 deletions
|
|
@ -250,6 +250,29 @@ func renderReport(w io.Writer,
|
|||
}
|
||||
fmt.Fprintf(w, "%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NPostcheckMiss, rs.PostcheckDetail)
|
||||
}
|
||||
// Banknote channel (pack-20 / D39.42 п.4): the draft-side WHAT proposals are counted per chunk in
|
||||
// retrieval_state, but until now NO surface printed them — the cold run of 31.07 had to read them
|
||||
// out of the DB by hand to answer "did the channel produce anything, and what did the parser
|
||||
// reject". Chunk coverage matters as much as the total: 0 lines on most chunks is a channel
|
||||
// failure that a healthy grand total can hide. Printed only when the channel produced or rejected
|
||||
// something, so a banknote-off book's report stays byte-identical.
|
||||
var bankLines, bankChunks, bankParseFail, bankTruncated int
|
||||
for _, rs := range states {
|
||||
bankLines += rs.NBanknoteLines
|
||||
if rs.NBanknoteLines > 0 {
|
||||
bankChunks++
|
||||
}
|
||||
if rs.BanknoteParseFail != 0 {
|
||||
bankParseFail++
|
||||
}
|
||||
if rs.BanknoteTruncated != 0 {
|
||||
bankTruncated++
|
||||
}
|
||||
}
|
||||
if bankLines > 0 || bankParseFail > 0 || bankTruncated > 0 {
|
||||
fmt.Fprintf(w, "\n=== BANKNOTE (draft-side WHAT channel, observability) === lines=%d over %d/%d chunk(s) · parse-fail chunks=%d · truncated chunks=%d\n",
|
||||
bankLines, bankChunks, len(states), bankParseFail, bankTruncated)
|
||||
}
|
||||
// Cheap style/number flaggers (observability, not gates): total + per-chunk detail.
|
||||
fmt.Fprintf(w, "\n=== STYLE GATES (observability: dialogue-dashes, ё, translit-interjections, 万/億 magnitudes, reflow-regression) — total %d ===\n", style)
|
||||
stylePrinted := false
|
||||
|
|
@ -333,6 +356,17 @@ func renderQuality(w io.Writer, q *pipeline.QualityReport) error {
|
|||
fmt.Fprintf(w, "UNSIGNED BANK: terms=%d · row-showings=%d · model followed=%d%s — proposals, not canon; sign them with `tmctl translate --verify-bank`\n",
|
||||
q.UnsignedBankTerms, q.UnverifiedShown, q.UnverifiedFollowed, followed)
|
||||
}
|
||||
// The pack-19 flaggers (D39.55). They were aggregated into the report struct from the start but no
|
||||
// surface ever PRINTED them, so the only run that had them on could not read its own axes. Counts are
|
||||
// meaningless without their denominators (2 flags out of 3 attributed replies is a different fact from
|
||||
// 2 out of 300), so the line always carries them. Axis D is reported apart from the A–C total because
|
||||
// its addressee comes from a heuristic, and the rule version travels with the numbers (the voice gate
|
||||
// is deliberately not snapshot-folded). Spoiler leaks sit here too: the one safety signal of the four.
|
||||
// Printed only when the flagger actually ran, so a book without voice content is byte-identical.
|
||||
if q.VoiceCheckVersion != "" || q.VoiceFlags > 0 || q.SpoilerLeaks > 0 {
|
||||
fmt.Fprintf(w, "VOICE (axes A–C, observability): flags=%d over attributed=%d of replies=%d · axis-D pair-register=%d · spoiler-leaks=%d · rules=%s\n",
|
||||
q.VoiceFlags, q.VoiceAttributed, q.VoiceReplies, q.VoicePairRegister, q.SpoilerLeaks, dashIfEmpty(q.VoiceCheckVersion))
|
||||
}
|
||||
// Addressable-defect RESIDUAL (pack-16): how many defects a repair loop could actually attack in what
|
||||
// this run shipped — the $0 measurement that decides whether enabling the paid loop is worth it. The
|
||||
// line is printed only when something fired, so a clean book's report is unchanged.
|
||||
|
|
|
|||
95
backend/cmd/tmctl/render_observability_test.go
Normal file
95
backend/cmd/tmctl/render_observability_test.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/pipeline"
|
||||
"textmachine/backend/internal/store"
|
||||
)
|
||||
|
||||
// render_observability_test.go pins the two counters the engine had always computed but no surface
|
||||
// printed (found by the coldrun-b readiness pass): the pack-19 voice flagger and the pack-20 banknote
|
||||
// channel. Both are pure observability — no gate, no snapshot fold, no verdict — so each test also
|
||||
// pins the SILENCE case: a run without the channel keeps its byte-identical output.
|
||||
|
||||
func TestRenderQualityPrintsVoiceAxesWithDenominators(t *testing.T) {
|
||||
var b strings.Builder
|
||||
q := &pipeline.QualityReport{
|
||||
VoiceFlags: 2, VoiceReplies: 40, VoiceAttributed: 31, VoicePairRegister: 5,
|
||||
SpoilerLeaks: 1, VoiceCheckVersion: "voice-v1",
|
||||
}
|
||||
if err := renderQuality(&b, q); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := b.String()
|
||||
// The count alone is unreadable — 2 of 31 attributed replies is a different fact from 2 of 3.
|
||||
for _, want := range []string{"VOICE (axes A–C", "flags=2", "attributed=31", "replies=40",
|
||||
"axis-D pair-register=5", "spoiler-leaks=1", "rules=voice-v1"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("the voice line must carry %q, got:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderQualityVoiceLinePrintsZeroWhenTheGateRan(t *testing.T) {
|
||||
// «Not measured» and «measured, clean» are different answers (coldrun-a §7 reported the first as
|
||||
// the second). The version string is the discriminator: it is set iff gates.voice was on.
|
||||
var b strings.Builder
|
||||
if err := renderQuality(&b, &pipeline.QualityReport{VoiceCheckVersion: "voice-v1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(b.String(), "flags=0 over attributed=0 of replies=0") {
|
||||
t.Fatalf("a gate that ran and found nothing must say so with its denominators:\n%s", b.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderQualityStaysSilentWithoutTheVoiceGate(t *testing.T) {
|
||||
var b strings.Builder
|
||||
if err := renderQuality(&b, &pipeline.QualityReport{TotalUnits: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(b.String(), "VOICE") {
|
||||
t.Fatalf("a book whose voice gate never ran must not grow a voice line:\n%s", b.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderReportPrintsBanknoteCoverageNotJustTheTotal(t *testing.T) {
|
||||
// Chunk coverage is the signal a grand total hides: 12 lines can be 12 chunks × 1 or 1 chunk × 12,
|
||||
// and only the second is a broken channel.
|
||||
var b strings.Builder
|
||||
states := []store.RetrievalState{
|
||||
{Chapter: 1, ChunkIdx: 0, NBanknoteLines: 7},
|
||||
{Chapter: 1, ChunkIdx: 1, NBanknoteLines: 0, BanknoteParseFail: 1},
|
||||
{Chapter: 2, ChunkIdx: 0, NBanknoteLines: 5, BanknoteTruncated: 1},
|
||||
}
|
||||
if err := renderReport(&b,
|
||||
func() ([]store.RequestLogView, error) { return nil, nil },
|
||||
func() ([]store.ChunkStatus, error) { return nil, nil },
|
||||
func() ([]store.RetrievalState, error) { return states, nil },
|
||||
okLedger); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := b.String()
|
||||
for _, want := range []string{"=== BANKNOTE", "lines=12", "over 2/3 chunk(s)",
|
||||
"parse-fail chunks=1", "truncated chunks=1"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("the banknote line must carry %q, got:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderReportStaysSilentWithoutTheBanknoteChannel(t *testing.T) {
|
||||
var b strings.Builder
|
||||
states := []store.RetrievalState{{Chapter: 1, ChunkIdx: 0, NExactHits: 2}}
|
||||
if err := renderReport(&b,
|
||||
func() ([]store.RequestLogView, error) { return nil, nil },
|
||||
func() ([]store.ChunkStatus, error) { return nil, nil },
|
||||
func() ([]store.RetrievalState, error) { return states, nil },
|
||||
okLedger); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(b.String(), "BANKNOTE") {
|
||||
t.Fatalf("a banknote-off book must not grow a banknote section:\n%s", b.String())
|
||||
}
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@ import (
|
|||
"time"
|
||||
|
||||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/lang"
|
||||
"textmachine/backend/internal/llm"
|
||||
"textmachine/backend/internal/obs"
|
||||
)
|
||||
|
|
@ -37,14 +38,18 @@ import (
|
|||
// that stresses exactly the silent-refusal failure modes (echo on dense CJK, empty
|
||||
// on a starved reasoning budget). grok is commented out by default: it bills the
|
||||
// clean-prod xAI account (the hygiene contract in models.yaml) — enable deliberately.
|
||||
// srcLang is what the echo share is measured AGAINST: D39.64 replaced the Han+kana-hardcoded cjkShare with
|
||||
// sourceScriptShare over the SOURCE language's declared scripts, so the probe must now say which language
|
||||
// its fragment is in (a ja fragment measured against zh scripts under-reports its own kana echo).
|
||||
var liveModels = []struct {
|
||||
model string
|
||||
source string
|
||||
model string
|
||||
srcLang string
|
||||
source string
|
||||
}{
|
||||
{"deepseek-v4-flash", "祝福是鲁迅创作的短篇小说，收录于《彷徨》。故事以第一人称叙述。"}, // the echo-mine repro family (Lu Xun 祝福)
|
||||
{"glm-5", "静かな図書館の朝、彼女は古い本を開いて、ゆっくりと読み始めた。"},
|
||||
{"kimi-k2.6", "这是一个用于验证适配器的测试句子，内容需要被完整翻译成俄语。"},
|
||||
// {"grok-4.20-0309-non-reasoning", "这是一个测试句子。"}, // clean-prod xAI only
|
||||
{"deepseek-v4-flash", "zh", "祝福是鲁迅创作的短篇小说，收录于《彷徨》。故事以第一人称叙述。"}, // the echo-mine repro family (Lu Xun 祝福)
|
||||
{"glm-5", "ja", "静かな図書館の朝、彼女は古い本を開いて、ゆっくりと読み始めた。"},
|
||||
{"kimi-k2.6", "zh", "这是一个用于验证适配器的测试句子，内容需要被完整翻译成俄语。"},
|
||||
// {"grok-4.20-0309-non-reasoning", "zh", "这是一个测试句子。"}, // clean-prod xAI only
|
||||
}
|
||||
|
||||
func TestLiveAdapterConformance(t *testing.T) {
|
||||
|
|
@ -93,9 +98,11 @@ func TestLiveAdapterConformance(t *testing.T) {
|
|||
// translation is ok; an echo → cjk_artifact, empty → empty, refusal →
|
||||
// soft_refusal. The response slug is logged for canonicalization review
|
||||
// (Gemini returns models/…, xAI redirects retired slugs, etc.).
|
||||
cls := classify(classifyInput{Source: tc.source, Output: resp.Text, Finish: resp.FinishReason, TargetLang: "ru"})
|
||||
t.Logf("model=%q finish=%q usage=%+v cjk_share=%.2f verdict=%q text=%.120q",
|
||||
resp.Model, resp.FinishReason, resp.Usage, cjkShare(resp.Text), cls.Reason, resp.Text)
|
||||
scripts := lang.LangScripts(tc.srcLang)
|
||||
cls := classify(classifyInput{Source: tc.source, Output: resp.Text, Finish: resp.FinishReason,
|
||||
TargetLang: "ru", SourceScripts: scripts})
|
||||
t.Logf("model=%q finish=%q usage=%+v source_script_share=%.2f verdict=%q text=%.120q",
|
||||
resp.Model, resp.FinishReason, resp.Usage, sourceScriptShare(resp.Text, scripts), cls.Reason, resp.Text)
|
||||
|
||||
if strings.TrimSpace(resp.Text) == "" {
|
||||
t.Fatalf("%s returned EMPTY content (read `content`, raise max_tokens — reasoning likely ate the budget)", tc.model)
|
||||
|
|
|
|||
141
backend/internal/pipeline/live_reprobe_test.go
Normal file
141
backend/internal/pipeline/live_reprobe_test.go
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
//go:build live
|
||||
|
||||
package pipeline
|
||||
|
||||
// live_reprobe_test.go: the PAID acceptance probe of the §2 type-classifier (D39.69 §2 / row 74).
|
||||
// Build-tagged `live` like live_conformance_test.go, so `go test ./...` never compiles it.
|
||||
//
|
||||
// TM_LIVE=1 TM_CLASSIFY6_CONFIG=~/books/gu-zhenren/coldrun-b/reprobe/classify6/book.yaml \
|
||||
// go test -tags live -run TestLiveClassifierHarmSet -v ./internal/pipeline/
|
||||
//
|
||||
// It drives the PRODUCTION classifier path end to end — the pair's own resolved prompt template, the
|
||||
// production `classifierMessages` render, the production `bankCallBudget` sizing and, crucially, the
|
||||
// production `runBankAttempt` money sequence (reserve → call → settle+checkpoint). It calls no provider
|
||||
// itself: the single-egress guard (TestProviderEgressSeamIsSingle) is a real invariant, and a probe that
|
||||
// bypassed the seam would also bypass the ledger — its cost would then be invisible to `tmctl report`,
|
||||
// which is the opposite of what a paid probe owes. Everything it reports is therefore re-readable from
|
||||
// the probe project's own store.
|
||||
//
|
||||
// The candidates are primed with the WRONG draft type on purpose: the harm this phase exists to fix is a
|
||||
// realia mis-typed `name` and then transliterated into a meaningless syllable (元石 → «юаньши»), so a probe
|
||||
// that primed the RIGHT type would also be passed by a model that merely echoes its input.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"textmachine/backend/internal/chunk"
|
||||
"textmachine/backend/internal/config"
|
||||
"textmachine/backend/internal/obs"
|
||||
"textmachine/backend/internal/terminology"
|
||||
"textmachine/backend/internal/text"
|
||||
)
|
||||
|
||||
// harmSet is the six terms of the ratified acceptance threshold (BANK_QUALITY_DESIGN §2): every one of
|
||||
// them is a THING or a facility that must be translated by sense, and every one of them was mis-typed by
|
||||
// the draft heuristic. Each carries a real ctx line from the stand book, because the classifier prompt
|
||||
// tells the model to decide by context rather than by the `type:` guess.
|
||||
//
|
||||
// ⚠ HONEST LIMIT of the 6/6 number: 元石 and 灵泉 appear VERBATIM as `term` examples inside
|
||||
// prompts/zh-ru/classifier.md, so two of the six are recall, not judgement. The discriminating subset is
|
||||
// 元海 / 蛊室 / 池塘 / 酒肆 — read the result as 4 decided + 2 recalled.
|
||||
var harmSet = []terminology.Candidate{
|
||||
{Key: "元石", Src: "元石", Type: "name", Freq: 12, Origin: terminology.OriginBanknote,
|
||||
KWIC: []string{"谁能拔得头筹，就会有二十块元石的丰厚奖励"}},
|
||||
{Key: "元海", Src: "元海", Type: "name", Freq: 9, Origin: terminology.OriginBanknote,
|
||||
KWIC: []string{"察看自身的空窍元海"}},
|
||||
{Key: "蛊室", Src: "蛊室", Type: "place", Freq: 5, Origin: terminology.OriginBanknote,
|
||||
KWIC: []string{"你们就去学堂里的蛊室，挑选蛊虫"}},
|
||||
{Key: "池塘", Src: "池塘", Type: "place", Freq: 4, Origin: terminology.OriginBanknote,
|
||||
KWIC: []string{"村外的池塘边上，长满了青草"}},
|
||||
{Key: "灵泉", Src: "灵泉", Type: "name", Freq: 3, Origin: terminology.OriginBanknote,
|
||||
KWIC: []string{"灵泉之水，可以洗练身躯"}},
|
||||
{Key: "酒肆", Src: "酒肆", Type: "place", Freq: 3, Origin: terminology.OriginBanknote,
|
||||
KWIC: []string{"镇上的酒肆里，说书人正讲着蛊师的故事"}},
|
||||
}
|
||||
|
||||
func TestLiveClassifierHarmSet(t *testing.T) {
|
||||
if os.Getenv("TM_LIVE") != "1" {
|
||||
t.Skip("paid probe: set TM_LIVE=1")
|
||||
}
|
||||
cfg := os.Getenv("TM_CLASSIFY6_CONFIG")
|
||||
if cfg == "" {
|
||||
t.Skip("set TM_CLASSIFY6_CONFIG to a book.yaml whose gates.terminology.classify_types is on")
|
||||
}
|
||||
r, err := NewRunner(cfg, obs.NewLogger())
|
||||
if err != nil {
|
||||
t.Fatalf("build runner: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
if r.classifierTemplate == nil {
|
||||
t.Fatalf("classify_types is off in %s — the probe would measure nothing", cfg)
|
||||
}
|
||||
// The eager client map is filled by the precompute pass inside TranslateBook, which this probe does
|
||||
// not run; build it the same way rather than teaching clientFor about a probe.
|
||||
if err := r.buildClients(); err != nil {
|
||||
t.Fatalf("build clients: %v", err)
|
||||
}
|
||||
|
||||
msgs, err := r.classifierMessages(harmSet)
|
||||
if err != nil {
|
||||
t.Fatalf("render production classifier messages: %v", err)
|
||||
}
|
||||
model := r.Pipeline.Gates.Terminology.ClassifierModel()
|
||||
_, maxTokens := r.bankCallBudget(model, msgs)
|
||||
|
||||
// The probe's own snapshot axis: a literal, so re-running it replays the paid checkpoint for $0
|
||||
// instead of re-buying the answer — the same property the production bank roles have.
|
||||
const snapID = "reprobe-classifier-6of6"
|
||||
// jobs.snapshot_id is a FOREIGN KEY into snapshots — the production path upserts the wave snapshot
|
||||
// before opening jobs, so the probe registers its own axis the same way instead of borrowing one.
|
||||
if err := r.Store.UpsertSnapshot(snapID, "", `{"probe":"classifier-harm-set-6of6"}`); err != nil {
|
||||
t.Fatalf("register the probe snapshot axis: %v", err)
|
||||
}
|
||||
st := config.Stage{Name: terminologyStageName, Role: roleClassifier, Model: model}
|
||||
job, err := r.Store.EnsureJob(r.Book.BookID, 0, terminologyStageName, snapID)
|
||||
if err != nil {
|
||||
t.Fatalf("ensure job: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
att, err := r.runBankAttempt(ctx, roleClassifier, model, st, snapID, chunk.Chunk{Chapter: 0, ChunkIdx: 0}, job, msgs)
|
||||
if err != nil {
|
||||
t.Fatalf("live classifier call failed: %v", err)
|
||||
}
|
||||
|
||||
got, stats := terminology.ParseTypes(att.text, candKeys(harmSet), text.NormalizeSourceKey)
|
||||
hits, decided := 0, 0
|
||||
for _, c := range harmSet {
|
||||
if got[c.Key] != "term" {
|
||||
continue
|
||||
}
|
||||
hits++
|
||||
if c.Key != "元石" && c.Key != "灵泉" { // the two the prompt names verbatim
|
||||
decided++
|
||||
}
|
||||
}
|
||||
t.Logf("CLASSIFIER 6/6 PROBE: model=%s max_tokens=%d finish=%q verdict=%q latency=%dms usage=%+v cost_usd=%.6f bad_lines=%d hits=%d/6 (discriminating %d/4) got=%v",
|
||||
att.modelActual, maxTokens, att.finish, att.cls.Reason, time.Since(start).Milliseconds(), att.usage, att.runCost, stats.Bad, hits, decided, got)
|
||||
|
||||
// Persist the raw wire evidence next to the probe project — "persist, not scratch": every number in
|
||||
// the report must be re-readable from the artifact that produced it.
|
||||
blob, _ := json.MarshalIndent(map[string]any{
|
||||
"model": att.modelActual, "max_tokens": maxTokens, "finish": att.finish,
|
||||
"verdict": string(att.cls.Reason), "usage": att.usage, "cost_usd": att.runCost,
|
||||
"reply": att.text, "parsed": got, "bad_lines": stats.Bad,
|
||||
"hits_term": hits, "discriminating_hits": decided,
|
||||
}, "", " ")
|
||||
if werr := os.WriteFile(filepath.Join(filepath.Dir(cfg), "classifier-6of6.json"), blob, 0o644); werr != nil {
|
||||
t.Errorf("persist probe evidence: %v", werr)
|
||||
}
|
||||
|
||||
if hits != len(harmSet) {
|
||||
t.Fatalf("acceptance threshold is 6/6 term (D39.69 §2); got %d/6: %v — reply:\n%s", hits, got, att.text)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue