223 lines
13 KiB
Go
223 lines
13 KiB
Go
package main
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/llm"
|
||
"textmachine/backend/internal/pipeline"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// coldRunShapeRows reproduces the shape of the paid run of 11.09 that this pack is about: twenty-nine
|
||
// usable calls and four that answered 200 with nothing usable — three empty at the ceiling, one a
|
||
// truncated draft — after which every unit shipped on a later attempt. The four cost $0.106472 of the
|
||
// $0.419423 the book paid.
|
||
func coldRunShapeRows() []store.RequestLogView {
|
||
rows := []store.RequestLogView{
|
||
{TS: "t1", Chapter: 1, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", CompletionTokens: 8496, FinishReason: "length", Degraded: "empty", OK: 0, CostUSD: 0.012044},
|
||
{TS: "t2", Chapter: 2, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", CompletionTokens: 8496, FinishReason: "length", Degraded: "length", OK: 0, CostUSD: 0.011637},
|
||
{TS: "t3", Chapter: 3, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", CompletionTokens: 8496, FinishReason: "length", Degraded: "empty", OK: 0, CostUSD: 0.011782},
|
||
{TS: "t4", Chapter: 2, Stage: "edit", Role: "editor", ModelRequested: "m", ModelActual: "m", CompletionTokens: 16000, FinishReason: "length", Degraded: "empty", OK: 0, CostUSD: 0.071009},
|
||
}
|
||
// The paying remainder, as one row, so the share has a real denominator.
|
||
rows = append(rows, store.RequestLogView{TS: "t5", Chapter: 1, Stage: "draft", Role: "translator",
|
||
ModelRequested: "m", ModelActual: "m", FinishReason: "stop", OK: 1, CostUSD: 0.312951})
|
||
// Two replayed rows, and the SECOND is the one that matters. A resume writes tm_hit=1 with ok taken
|
||
// from the stored disposition (pipeline/resume.go), so a resumed FLAGGED unit is `tm_hit=1, ok=0` —
|
||
// a row that looks exactly like a failure and bought nothing. Without it the tm_hit test in the
|
||
// filter is unreachable: an `ok` replay is already excluded by the ok test beside it, and a fixture
|
||
// that carries only that one pins half a condition while reading as if it pinned both.
|
||
rows = append(rows, store.RequestLogView{TS: "t6", Chapter: 1, Stage: "draft", Role: "translator",
|
||
ModelRequested: "m", ModelActual: "m", TMHit: 1, OK: 1, CostUSD: 0})
|
||
// It carries a COST as well, which today's resume path does not write — and the row is built that
|
||
// way on purpose. This renderer is a pure function of the rows it is handed, and a replayed row's
|
||
// money was already booked by the row it replays; counting it again would inflate the denominator
|
||
// the share is argued from. A fixture where every replay costs nothing pins the exclusion in the
|
||
// numerator and leaves the denominator's half of it unreachable.
|
||
rows = append(rows, store.RequestLogView{TS: "t7", Chapter: 3, Stage: "edit", Role: "editor",
|
||
ModelRequested: "m", ModelActual: "m", TMHit: 1, OK: 0, FinishReason: "length", Degraded: "empty", CostUSD: 0.5})
|
||
// Two more shapes that carry ok=0 and are NOT waste, and both are taken from what the engine
|
||
// really writes — a report that counted them would tell the owner a larger number than the truth,
|
||
// which is the one direction a money line must never err in.
|
||
//
|
||
// t8: a call that never reached a billed response. The runner writes ok=0 with cost_usd=0 and the
|
||
// cause in `err`, leaving `degraded` and `finish_reason` EMPTY — so counting it would also print a
|
||
// bucket with no name.
|
||
rows = append(rows, store.RequestLogView{TS: "t8", Chapter: 3, Stage: "edit", Role: "editor",
|
||
ModelRequested: "m", ModelActual: "m", OK: 0, Err: "context canceled", CostUSD: 0})
|
||
// t9: a cosmetic sanitizer strip. The verdict is not ok and the cleaned text SHIPS — the reader
|
||
// got exactly what this call paid for.
|
||
rows = append(rows, store.RequestLogView{TS: "t9", Chapter: 1, Stage: "edit", Role: "editor",
|
||
ModelRequested: "m", ModelActual: "m", OK: 0, FinishReason: "stop", Degraded: "sanitizer_stripped", CostUSD: 0.018654})
|
||
return rows
|
||
}
|
||
|
||
// TestTheReportAddsUpWhatWasPaidForAndThrownAway is the line the run of 11.09 had nowhere to print. The
|
||
// FLAGS section reports CHUNKS whose final disposition is not ok, and all four of those units were
|
||
// recovered by the regeneration and shipped — so a run that spent a quarter of its money on empty
|
||
// replies printed a clean report.
|
||
func TestTheReportAddsUpWhatWasPaidForAndThrownAway(t *testing.T) {
|
||
var b strings.Builder
|
||
if err := renderReport(&b,
|
||
func() ([]store.RequestLogView, error) { return coldRunShapeRows(), nil },
|
||
// No flagged chunk at all: every unit recovered. This is the fixture's whole point.
|
||
func() ([]store.ChunkStatus, error) { return nil, nil },
|
||
func() ([]store.RetrievalState, error) { return nil, nil },
|
||
pricedByAnswerer, okLedger); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
out := b.String()
|
||
if strings.Contains(out, "=== FLAGS") {
|
||
t.Fatalf("the fixture must have NO flagged chunk, else it is not the case this line exists for:\n%s", out)
|
||
}
|
||
// FOUR, and the fixture carries THREE other rows that also read ok=0 and are not waste — a replay,
|
||
// a $0 transport failure, and a cosmetic strip whose text shipped. The message names all three,
|
||
// because whoever breaks this has to learn from it WHICH of them started being counted; a message
|
||
// that names only one is a right verdict delivered with the wrong cause.
|
||
if !strings.Contains(out, "PAID AND THROWN AWAY: 4 call(s), $0.106472") {
|
||
t.Fatalf("the money of the thrown-away calls must be added up and printed, and none of these is a "+
|
||
"paid-for-nothing CALL: a replay (tm_hit), a failure that cost $0, a stripped answer that SHIPPED:\n%s", out)
|
||
}
|
||
// The share is what makes it an argument rather than a number. The DENOMINATOR is every fresh
|
||
// billed row — including the stripped-but-shipped call, which really was paid for — so it is
|
||
// $0.419423 + $0.018654 = $0.438077 and the share is 24.3%. The replayed row's $0.500000 is in
|
||
// NEITHER half: it would drop the share to 11.4% and quietly make the waste look like a seventh of
|
||
// what it is.
|
||
if !strings.Contains(out, "24.3% of the $0.438077") {
|
||
t.Fatalf("the line must say what share of the book's spend this is, and a replay is in neither half:\n%s", out)
|
||
}
|
||
// The breakdown separates the two causes the remedy has to tell apart.
|
||
// Three empty, not four: the resumed flagged row carries `empty` too and must not swell the cause.
|
||
if !strings.Contains(out, "empty×3") || !strings.Contains(out, "length×1") {
|
||
t.Fatalf("the line must break the waste down by cause — three empty, one truncated:\n%s", out)
|
||
}
|
||
// ⚠ THE NEXT TWO ARE SCOPED TO THE LINE, not to the whole report. Both words appear in the ROW
|
||
// TABLE above it — `sanitizer_stripped` as that row's own degraded tag — so a whole-output search
|
||
// answers a question nobody asked and goes red on a correct report.
|
||
line := paidLine(t, out)
|
||
// A call that cost nothing is not money thrown away, and it would arrive with no cause to print.
|
||
if strings.Contains(line, "unnamed") {
|
||
t.Fatalf("a $0 transport failure must not enter a MONEY line, least of all as a nameless bucket: %q", line)
|
||
}
|
||
// A cosmetic strip ships its text. The reader got what it paid for.
|
||
if strings.Contains(line, "sanitizer_stripped") {
|
||
t.Fatalf("a stripped-but-SHIPPED answer is not waste; counting it overstates the loss: %q", line)
|
||
}
|
||
}
|
||
|
||
// paidLine returns the PAID AND THROWN AWAY line itself, so an assertion about what the LINE says
|
||
// cannot be satisfied — or broken — by the row table printed above it.
|
||
func paidLine(t *testing.T, out string) string {
|
||
t.Helper()
|
||
for _, l := range strings.Split(out, "\n") {
|
||
if strings.HasPrefix(l, "PAID AND THROWN AWAY") {
|
||
return l
|
||
}
|
||
}
|
||
t.Fatalf("the report carries no PAID AND THROWN AWAY line at all:\n%s", out)
|
||
return ""
|
||
}
|
||
|
||
// TestACleanRunSaysNothingAboutWaste is the control. A section that printed on every run would be
|
||
// noise, and a zero would be indistinguishable from a run nobody measured.
|
||
func TestACleanRunSaysNothingAboutWaste(t *testing.T) {
|
||
rows := []store.RequestLogView{
|
||
{TS: "t1", Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", FinishReason: "stop", OK: 1, CostUSD: 0.02},
|
||
{TS: "t2", Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", TMHit: 1, OK: 1, CostUSD: 0},
|
||
}
|
||
var b strings.Builder
|
||
if err := renderReport(&b,
|
||
func() ([]store.RequestLogView, error) { return rows, nil },
|
||
func() ([]store.ChunkStatus, error) { return nil, nil },
|
||
func() ([]store.RetrievalState, error) { return nil, nil },
|
||
pricedByAnswerer, okLedger); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if out := b.String(); strings.Contains(out, "PAID AND THROWN AWAY") {
|
||
t.Fatalf("a run where every call answered must not print the line at all:\n%s", out)
|
||
}
|
||
}
|
||
|
||
// TestTheRunningReportSaysWhatTheBudgetWentTo covers the OTHER renderer. `tmctl translate` prints a
|
||
// line per stage while a paid run is happening, and that is the reader who can still act on it — the
|
||
// operator watching a book cost money. A thinking share visible only in the post-mortem arrives after
|
||
// the budget is spent.
|
||
//
|
||
// The three cells are the same three answers the report table has to keep apart, and they are read by
|
||
// their own marker (`think=`) rather than by a substring, because every other number on this line is
|
||
// small and a bare "0" matches several of them.
|
||
func TestTheRunningReportSaysWhatTheBudgetWentTo(t *testing.T) {
|
||
ate, none := 8496, 0
|
||
res := &pipeline.BookResult{BookID: "b1", TotalUSD: 0.1, Chunks: []pipeline.ChunkOutcome{{
|
||
Chapter: 1, ChunkIdx: 0, Disposition: pipeline.DispOK, FinalText: "ТЕКСТ",
|
||
Stages: []pipeline.StageResult{
|
||
{Stage: "draft", Model: "m", Disposition: pipeline.DispOK, Attempts: 2, FinishReason: "stop",
|
||
Usage: llm.Usage{PromptTokens: 10, CompletionTokens: 8496, ReasoningInCompletion: &ate}},
|
||
{Stage: "edit", Model: "m", Disposition: pipeline.DispOK, Attempts: 1, FinishReason: "stop",
|
||
Usage: llm.Usage{PromptTokens: 10, CompletionTokens: 2624, ReasoningInCompletion: &none}},
|
||
{Stage: "judge", Model: "m", Disposition: pipeline.DispOK, Attempts: 1, FinishReason: "stop",
|
||
Usage: llm.Usage{PromptTokens: 10, CompletionTokens: 100}},
|
||
},
|
||
}}}
|
||
var b strings.Builder
|
||
if err := renderTranslate(&b, res, okLedger); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
out := b.String()
|
||
for stage, want := range map[string]string{"draft": "think=8496", "edit": "think=0", "judge": "think=?"} {
|
||
line := stageLine(t, out, stage)
|
||
if !strings.Contains(line, want) {
|
||
t.Fatalf("the %s line must carry %q — a call that spent its budget thinking, one that did not, and one whose provider said nothing are three different answers: %q", stage, want, line)
|
||
}
|
||
}
|
||
}
|
||
|
||
// stageLine returns the running report's line for one stage, so an assertion about that stage cannot be
|
||
// satisfied by a neighbouring one.
|
||
func stageLine(t *testing.T, out, stage string) string {
|
||
t.Helper()
|
||
for _, l := range strings.Split(out, "\n") {
|
||
if f := strings.Fields(l); len(f) > 0 && f[0] == stage {
|
||
return l
|
||
}
|
||
}
|
||
t.Fatalf("the running report has no line for stage %q:\n%s", stage, out)
|
||
return ""
|
||
}
|
||
|
||
// TestAPaidFailureWithNoCauseIsCountedUnderAName is the branch that exists so a money line never prints
|
||
// a bucket nobody can ask about — my own remedy for the nameless `×1` the first review found.
|
||
//
|
||
// ⚠ IT IS EXERCISED DIRECTLY, AND THE REASON IS STATED RATHER THAN GLOSSED. No writer in the engine
|
||
// produces this row TODAY: every billed failure path fills `degraded` (the classifier's tag, the cut's
|
||
// finish, `billed_2xx_decode_failed`), and the one shape that leaves both empty — a call that never
|
||
// reached a billed response — costs $0 and is excluded a line earlier. So the branch is defensive over
|
||
// the renderer's INPUT, and this renderer is a pure function of the rows it is handed: it must be right
|
||
// for a row a future writer produces, not only for the ones today's writers do. Measured before this
|
||
// test existed: a `panic` planted in that branch left the package green, while the same panic in a
|
||
// reachable branch went red — the branch was decoration.
|
||
func TestAPaidFailureWithNoCauseIsCountedUnderAName(t *testing.T) {
|
||
rows := []store.RequestLogView{
|
||
{TS: "t1", Chapter: 1, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m",
|
||
OK: 0, CostUSD: 0.01, Degraded: "", FinishReason: ""},
|
||
{TS: "t2", Chapter: 1, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m",
|
||
OK: 1, CostUSD: 0.09, FinishReason: "stop"},
|
||
}
|
||
var b strings.Builder
|
||
if err := renderReport(&b,
|
||
func() ([]store.RequestLogView, error) { return rows, nil },
|
||
func() ([]store.ChunkStatus, error) { return nil, nil },
|
||
func() ([]store.RetrievalState, error) { return nil, nil },
|
||
pricedByAnswerer, okLedger); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
line := paidLine(t, b.String())
|
||
if !strings.Contains(line, "unnamed×1") {
|
||
t.Fatalf("a BILLED failure that names no cause must still be counted under a name — an unlabelled "+
|
||
"bucket in a money line is a number the reader cannot ask about: %q", line)
|
||
}
|
||
if !strings.Contains(line, "1 call(s), $0.010000") {
|
||
t.Fatalf("the billed failure must be counted and priced: %q", line)
|
||
}
|
||
}
|