From 1ecf5c2f1a119a54a0571f0ce2b36a2cc9199011 Mon Sep 17 00:00:00 2001 From: "Claude (backend session)" Date: Sun, 2 Aug 2026 14:05:29 +0300 Subject: [PATCH] Add banknote and voice observability lines to tmctl report, fix live-tagged rig build and add classifier threshold probe --- backend/cmd/tmctl/render.go | 34 +++++ .../cmd/tmctl/render_observability_test.go | 95 ++++++++++++ .../pipeline/live_conformance_test.go | 25 ++-- .../internal/pipeline/live_reprobe_test.go | 141 ++++++++++++++++++ 4 files changed, 286 insertions(+), 9 deletions(-) create mode 100644 backend/cmd/tmctl/render_observability_test.go create mode 100644 backend/internal/pipeline/live_reprobe_test.go diff --git a/backend/cmd/tmctl/render.go b/backend/cmd/tmctl/render.go index ce87f6bb..3c18117d 100644 --- a/backend/cmd/tmctl/render.go +++ b/backend/cmd/tmctl/render.go @@ -250,6 +250,29 @@ func renderReport(w io.Writer, } fmt.Fprintf(w, "%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NPostcheckMiss, rs.PostcheckDetail) } + // Banknote channel (pack-20 / D39.42 п.4): the draft-side WHAT proposals are counted per chunk in + // retrieval_state, but until now NO surface printed them — the cold run of 31.07 had to read them + // out of the DB by hand to answer "did the channel produce anything, and what did the parser + // reject". Chunk coverage matters as much as the total: 0 lines on most chunks is a channel + // failure that a healthy grand total can hide. Printed only when the channel produced or rejected + // something, so a banknote-off book's report stays byte-identical. + var bankLines, bankChunks, bankParseFail, bankTruncated int + for _, rs := range states { + bankLines += rs.NBanknoteLines + if rs.NBanknoteLines > 0 { + bankChunks++ + } + if rs.BanknoteParseFail != 0 { + bankParseFail++ + } + if rs.BanknoteTruncated != 0 { + bankTruncated++ + } + } + if bankLines > 0 || bankParseFail > 0 || bankTruncated > 0 { + fmt.Fprintf(w, "\n=== BANKNOTE (draft-side WHAT channel, observability) === lines=%d over %d/%d chunk(s) · parse-fail chunks=%d · truncated chunks=%d\n", + bankLines, bankChunks, len(states), bankParseFail, bankTruncated) + } // Cheap style/number flaggers (observability, not gates): total + per-chunk detail. fmt.Fprintf(w, "\n=== STYLE GATES (observability: dialogue-dashes, ё, translit-interjections, 万/億 magnitudes, reflow-regression) — total %d ===\n", style) stylePrinted := false @@ -333,6 +356,17 @@ func renderQuality(w io.Writer, q *pipeline.QualityReport) error { fmt.Fprintf(w, "UNSIGNED BANK: terms=%d · row-showings=%d · model followed=%d%s — proposals, not canon; sign them with `tmctl translate --verify-bank`\n", q.UnsignedBankTerms, q.UnverifiedShown, q.UnverifiedFollowed, followed) } + // The pack-19 flaggers (D39.55). They were aggregated into the report struct from the start but no + // surface ever PRINTED them, so the only run that had them on could not read its own axes. Counts are + // meaningless without their denominators (2 flags out of 3 attributed replies is a different fact from + // 2 out of 300), so the line always carries them. Axis D is reported apart from the A–C total because + // its addressee comes from a heuristic, and the rule version travels with the numbers (the voice gate + // is deliberately not snapshot-folded). Spoiler leaks sit here too: the one safety signal of the four. + // Printed only when the flagger actually ran, so a book without voice content is byte-identical. + if q.VoiceCheckVersion != "" || q.VoiceFlags > 0 || q.SpoilerLeaks > 0 { + fmt.Fprintf(w, "VOICE (axes A–C, observability): flags=%d over attributed=%d of replies=%d · axis-D pair-register=%d · spoiler-leaks=%d · rules=%s\n", + q.VoiceFlags, q.VoiceAttributed, q.VoiceReplies, q.VoicePairRegister, q.SpoilerLeaks, dashIfEmpty(q.VoiceCheckVersion)) + } // Addressable-defect RESIDUAL (pack-16): how many defects a repair loop could actually attack in what // this run shipped — the $0 measurement that decides whether enabling the paid loop is worth it. The // line is printed only when something fired, so a clean book's report is unchanged. diff --git a/backend/cmd/tmctl/render_observability_test.go b/backend/cmd/tmctl/render_observability_test.go new file mode 100644 index 00000000..da15dba8 --- /dev/null +++ b/backend/cmd/tmctl/render_observability_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "strings" + "testing" + + "textmachine/backend/internal/pipeline" + "textmachine/backend/internal/store" +) + +// render_observability_test.go pins the two counters the engine had always computed but no surface +// printed (found by the coldrun-b readiness pass): the pack-19 voice flagger and the pack-20 banknote +// channel. Both are pure observability — no gate, no snapshot fold, no verdict — so each test also +// pins the SILENCE case: a run without the channel keeps its byte-identical output. + +func TestRenderQualityPrintsVoiceAxesWithDenominators(t *testing.T) { + var b strings.Builder + q := &pipeline.QualityReport{ + VoiceFlags: 2, VoiceReplies: 40, VoiceAttributed: 31, VoicePairRegister: 5, + SpoilerLeaks: 1, VoiceCheckVersion: "voice-v1", + } + if err := renderQuality(&b, q); err != nil { + t.Fatal(err) + } + out := b.String() + // The count alone is unreadable — 2 of 31 attributed replies is a different fact from 2 of 3. + for _, want := range []string{"VOICE (axes A–C", "flags=2", "attributed=31", "replies=40", + "axis-D pair-register=5", "spoiler-leaks=1", "rules=voice-v1"} { + if !strings.Contains(out, want) { + t.Fatalf("the voice line must carry %q, got:\n%s", want, out) + } + } +} + +func TestRenderQualityVoiceLinePrintsZeroWhenTheGateRan(t *testing.T) { + // «Not measured» and «measured, clean» are different answers (coldrun-a §7 reported the first as + // the second). The version string is the discriminator: it is set iff gates.voice was on. + var b strings.Builder + if err := renderQuality(&b, &pipeline.QualityReport{VoiceCheckVersion: "voice-v1"}); err != nil { + t.Fatal(err) + } + if !strings.Contains(b.String(), "flags=0 over attributed=0 of replies=0") { + t.Fatalf("a gate that ran and found nothing must say so with its denominators:\n%s", b.String()) + } +} + +func TestRenderQualityStaysSilentWithoutTheVoiceGate(t *testing.T) { + var b strings.Builder + if err := renderQuality(&b, &pipeline.QualityReport{TotalUnits: 3}); err != nil { + t.Fatal(err) + } + if strings.Contains(b.String(), "VOICE") { + t.Fatalf("a book whose voice gate never ran must not grow a voice line:\n%s", b.String()) + } +} + +func TestRenderReportPrintsBanknoteCoverageNotJustTheTotal(t *testing.T) { + // Chunk coverage is the signal a grand total hides: 12 lines can be 12 chunks × 1 or 1 chunk × 12, + // and only the second is a broken channel. + var b strings.Builder + states := []store.RetrievalState{ + {Chapter: 1, ChunkIdx: 0, NBanknoteLines: 7}, + {Chapter: 1, ChunkIdx: 1, NBanknoteLines: 0, BanknoteParseFail: 1}, + {Chapter: 2, ChunkIdx: 0, NBanknoteLines: 5, BanknoteTruncated: 1}, + } + if err := renderReport(&b, + func() ([]store.RequestLogView, error) { return nil, nil }, + func() ([]store.ChunkStatus, error) { return nil, nil }, + func() ([]store.RetrievalState, error) { return states, nil }, + okLedger); err != nil { + t.Fatal(err) + } + out := b.String() + for _, want := range []string{"=== BANKNOTE", "lines=12", "over 2/3 chunk(s)", + "parse-fail chunks=1", "truncated chunks=1"} { + if !strings.Contains(out, want) { + t.Fatalf("the banknote line must carry %q, got:\n%s", want, out) + } + } +} + +func TestRenderReportStaysSilentWithoutTheBanknoteChannel(t *testing.T) { + var b strings.Builder + states := []store.RetrievalState{{Chapter: 1, ChunkIdx: 0, NExactHits: 2}} + if err := renderReport(&b, + func() ([]store.RequestLogView, error) { return nil, nil }, + func() ([]store.ChunkStatus, error) { return nil, nil }, + func() ([]store.RetrievalState, error) { return states, nil }, + okLedger); err != nil { + t.Fatal(err) + } + if strings.Contains(b.String(), "BANKNOTE") { + t.Fatalf("a banknote-off book must not grow a banknote section:\n%s", b.String()) + } +} diff --git a/backend/internal/pipeline/live_conformance_test.go b/backend/internal/pipeline/live_conformance_test.go index 87302298..d98f87d2 100644 --- a/backend/internal/pipeline/live_conformance_test.go +++ b/backend/internal/pipeline/live_conformance_test.go @@ -29,6 +29,7 @@ import ( "time" "textmachine/backend/internal/config" + "textmachine/backend/internal/lang" "textmachine/backend/internal/llm" "textmachine/backend/internal/obs" ) @@ -37,14 +38,18 @@ import ( // that stresses exactly the silent-refusal failure modes (echo on dense CJK, empty // on a starved reasoning budget). grok is commented out by default: it bills the // clean-prod xAI account (the hygiene contract in models.yaml) — enable deliberately. +// srcLang is what the echo share is measured AGAINST: D39.64 replaced the Han+kana-hardcoded cjkShare with +// sourceScriptShare over the SOURCE language's declared scripts, so the probe must now say which language +// its fragment is in (a ja fragment measured against zh scripts under-reports its own kana echo). var liveModels = []struct { - model string - source string + model string + srcLang string + source string }{ - {"deepseek-v4-flash", "祝福是鲁迅创作的短篇小说，收录于《彷徨》。故事以第一人称叙述。"}, // the echo-mine repro family (Lu Xun 祝福) - {"glm-5", "静かな図書館の朝、彼女は古い本を開いて、ゆっくりと読み始めた。"}, - {"kimi-k2.6", "这是一个用于验证适配器的测试句子，内容需要被完整翻译成俄语。"}, - // {"grok-4.20-0309-non-reasoning", "这是一个测试句子。"}, // clean-prod xAI only + {"deepseek-v4-flash", "zh", "祝福是鲁迅创作的短篇小说，收录于《彷徨》。故事以第一人称叙述。"}, // the echo-mine repro family (Lu Xun 祝福) + {"glm-5", "ja", "静かな図書館の朝、彼女は古い本を開いて、ゆっくりと読み始めた。"}, + {"kimi-k2.6", "zh", "这是一个用于验证适配器的测试句子，内容需要被完整翻译成俄语。"}, + // {"grok-4.20-0309-non-reasoning", "zh", "这是一个测试句子。"}, // clean-prod xAI only } func TestLiveAdapterConformance(t *testing.T) { @@ -93,9 +98,11 @@ func TestLiveAdapterConformance(t *testing.T) { // translation is ok; an echo → cjk_artifact, empty → empty, refusal → // soft_refusal. The response slug is logged for canonicalization review // (Gemini returns models/…, xAI redirects retired slugs, etc.). - cls := classify(classifyInput{Source: tc.source, Output: resp.Text, Finish: resp.FinishReason, TargetLang: "ru"}) - t.Logf("model=%q finish=%q usage=%+v cjk_share=%.2f verdict=%q text=%.120q", - resp.Model, resp.FinishReason, resp.Usage, cjkShare(resp.Text), cls.Reason, resp.Text) + scripts := lang.LangScripts(tc.srcLang) + cls := classify(classifyInput{Source: tc.source, Output: resp.Text, Finish: resp.FinishReason, + TargetLang: "ru", SourceScripts: scripts}) + t.Logf("model=%q finish=%q usage=%+v source_script_share=%.2f verdict=%q text=%.120q", + resp.Model, resp.FinishReason, resp.Usage, sourceScriptShare(resp.Text, scripts), cls.Reason, resp.Text) if strings.TrimSpace(resp.Text) == "" { t.Fatalf("%s returned EMPTY content (read `content`, raise max_tokens — reasoning likely ate the budget)", tc.model) diff --git a/backend/internal/pipeline/live_reprobe_test.go b/backend/internal/pipeline/live_reprobe_test.go new file mode 100644 index 00000000..0b641fb0 --- /dev/null +++ b/backend/internal/pipeline/live_reprobe_test.go @@ -0,0 +1,141 @@ +//go:build live + +package pipeline + +// live_reprobe_test.go: the PAID acceptance probe of the §2 type-classifier (D39.69 §2 / row 74). +// Build-tagged `live` like live_conformance_test.go, so `go test ./...` never compiles it. +// +// TM_LIVE=1 TM_CLASSIFY6_CONFIG=~/books/gu-zhenren/coldrun-b/reprobe/classify6/book.yaml \ +// go test -tags live -run TestLiveClassifierHarmSet -v ./internal/pipeline/ +// +// It drives the PRODUCTION classifier path end to end — the pair's own resolved prompt template, the +// production `classifierMessages` render, the production `bankCallBudget` sizing and, crucially, the +// production `runBankAttempt` money sequence (reserve → call → settle+checkpoint). It calls no provider +// itself: the single-egress guard (TestProviderEgressSeamIsSingle) is a real invariant, and a probe that +// bypassed the seam would also bypass the ledger — its cost would then be invisible to `tmctl report`, +// which is the opposite of what a paid probe owes. Everything it reports is therefore re-readable from +// the probe project's own store. +// +// The candidates are primed with the WRONG draft type on purpose: the harm this phase exists to fix is a +// realia mis-typed `name` and then transliterated into a meaningless syllable (元石 → «юаньши»), so a probe +// that primed the RIGHT type would also be passed by a model that merely echoes its input. + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "textmachine/backend/internal/chunk" + "textmachine/backend/internal/config" + "textmachine/backend/internal/obs" + "textmachine/backend/internal/terminology" + "textmachine/backend/internal/text" +) + +// harmSet is the six terms of the ratified acceptance threshold (BANK_QUALITY_DESIGN §2): every one of +// them is a THING or a facility that must be translated by sense, and every one of them was mis-typed by +// the draft heuristic. Each carries a real ctx line from the stand book, because the classifier prompt +// tells the model to decide by context rather than by the `type:` guess. +// +// ⚠ HONEST LIMIT of the 6/6 number: 元石 and 灵泉 appear VERBATIM as `term` examples inside +// prompts/zh-ru/classifier.md, so two of the six are recall, not judgement. The discriminating subset is +// 元海 / 蛊室 / 池塘 / 酒肆 — read the result as 4 decided + 2 recalled. +var harmSet = []terminology.Candidate{ + {Key: "元石", Src: "元石", Type: "name", Freq: 12, Origin: terminology.OriginBanknote, + KWIC: []string{"谁能拔得头筹，就会有二十块元石的丰厚奖励"}}, + {Key: "元海", Src: "元海", Type: "name", Freq: 9, Origin: terminology.OriginBanknote, + KWIC: []string{"察看自身的空窍元海"}}, + {Key: "蛊室", Src: "蛊室", Type: "place", Freq: 5, Origin: terminology.OriginBanknote, + KWIC: []string{"你们就去学堂里的蛊室，挑选蛊虫"}}, + {Key: "池塘", Src: "池塘", Type: "place", Freq: 4, Origin: terminology.OriginBanknote, + KWIC: []string{"村外的池塘边上，长满了青草"}}, + {Key: "灵泉", Src: "灵泉", Type: "name", Freq: 3, Origin: terminology.OriginBanknote, + KWIC: []string{"灵泉之水，可以洗练身躯"}}, + {Key: "酒肆", Src: "酒肆", Type: "place", Freq: 3, Origin: terminology.OriginBanknote, + KWIC: []string{"镇上的酒肆里，说书人正讲着蛊师的故事"}}, +} + +func TestLiveClassifierHarmSet(t *testing.T) { + if os.Getenv("TM_LIVE") != "1" { + t.Skip("paid probe: set TM_LIVE=1") + } + cfg := os.Getenv("TM_CLASSIFY6_CONFIG") + if cfg == "" { + t.Skip("set TM_CLASSIFY6_CONFIG to a book.yaml whose gates.terminology.classify_types is on") + } + r, err := NewRunner(cfg, obs.NewLogger()) + if err != nil { + t.Fatalf("build runner: %v", err) + } + defer r.Close() + if r.classifierTemplate == nil { + t.Fatalf("classify_types is off in %s — the probe would measure nothing", cfg) + } + // The eager client map is filled by the precompute pass inside TranslateBook, which this probe does + // not run; build it the same way rather than teaching clientFor about a probe. + if err := r.buildClients(); err != nil { + t.Fatalf("build clients: %v", err) + } + + msgs, err := r.classifierMessages(harmSet) + if err != nil { + t.Fatalf("render production classifier messages: %v", err) + } + model := r.Pipeline.Gates.Terminology.ClassifierModel() + _, maxTokens := r.bankCallBudget(model, msgs) + + // The probe's own snapshot axis: a literal, so re-running it replays the paid checkpoint for $0 + // instead of re-buying the answer — the same property the production bank roles have. + const snapID = "reprobe-classifier-6of6" + // jobs.snapshot_id is a FOREIGN KEY into snapshots — the production path upserts the wave snapshot + // before opening jobs, so the probe registers its own axis the same way instead of borrowing one. + if err := r.Store.UpsertSnapshot(snapID, "", `{"probe":"classifier-harm-set-6of6"}`); err != nil { + t.Fatalf("register the probe snapshot axis: %v", err) + } + st := config.Stage{Name: terminologyStageName, Role: roleClassifier, Model: model} + job, err := r.Store.EnsureJob(r.Book.BookID, 0, terminologyStageName, snapID) + if err != nil { + t.Fatalf("ensure job: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + start := time.Now() + att, err := r.runBankAttempt(ctx, roleClassifier, model, st, snapID, chunk.Chunk{Chapter: 0, ChunkIdx: 0}, job, msgs) + if err != nil { + t.Fatalf("live classifier call failed: %v", err) + } + + got, stats := terminology.ParseTypes(att.text, candKeys(harmSet), text.NormalizeSourceKey) + hits, decided := 0, 0 + for _, c := range harmSet { + if got[c.Key] != "term" { + continue + } + hits++ + if c.Key != "元石" && c.Key != "灵泉" { // the two the prompt names verbatim + decided++ + } + } + t.Logf("CLASSIFIER 6/6 PROBE: model=%s max_tokens=%d finish=%q verdict=%q latency=%dms usage=%+v cost_usd=%.6f bad_lines=%d hits=%d/6 (discriminating %d/4) got=%v", + att.modelActual, maxTokens, att.finish, att.cls.Reason, time.Since(start).Milliseconds(), att.usage, att.runCost, stats.Bad, hits, decided, got) + + // Persist the raw wire evidence next to the probe project — "persist, not scratch": every number in + // the report must be re-readable from the artifact that produced it. + blob, _ := json.MarshalIndent(map[string]any{ + "model": att.modelActual, "max_tokens": maxTokens, "finish": att.finish, + "verdict": string(att.cls.Reason), "usage": att.usage, "cost_usd": att.runCost, + "reply": att.text, "parsed": got, "bad_lines": stats.Bad, + "hits_term": hits, "discriminating_hits": decided, + }, "", " ") + if werr := os.WriteFile(filepath.Join(filepath.Dir(cfg), "classifier-6of6.json"), blob, 0o644); werr != nil { + t.Errorf("persist probe evidence: %v", werr) + } + + if hits != len(harmSet) { + t.Fatalf("acceptance threshold is 6/6 term (D39.69 §2); got %d/6: %v — reply:\n%s", hits, got, att.text) + } +}