package main import ( "strings" "testing" "textmachine/backend/internal/llm" "textmachine/backend/internal/pipeline" "textmachine/backend/internal/store" ) // coldRunShapeRows reproduces the shape of the paid run of 11.09 that this pack is about: twenty-nine // usable calls and four that answered 200 with nothing usable — three empty at the ceiling, one a // truncated draft — after which every unit shipped on a later attempt. The four cost $0.106472 of the // $0.419423 the book paid. func coldRunShapeRows() []store.RequestLogView { rows := []store.RequestLogView{ {TS: "t1", Chapter: 1, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", CompletionTokens: 8496, FinishReason: "length", Degraded: "empty", OK: 0, CostUSD: 0.012044}, {TS: "t2", Chapter: 2, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", CompletionTokens: 8496, FinishReason: "length", Degraded: "length", OK: 0, CostUSD: 0.011637}, {TS: "t3", Chapter: 3, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", CompletionTokens: 8496, FinishReason: "length", Degraded: "empty", OK: 0, CostUSD: 0.011782}, {TS: "t4", Chapter: 2, Stage: "edit", Role: "editor", ModelRequested: "m", ModelActual: "m", CompletionTokens: 16000, FinishReason: "length", Degraded: "empty", OK: 0, CostUSD: 0.071009}, } // The paying remainder, as one row, so the share has a real denominator. rows = append(rows, store.RequestLogView{TS: "t5", Chapter: 1, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", FinishReason: "stop", OK: 1, CostUSD: 0.312951}) // Two replayed rows, and the SECOND is the one that matters. A resume writes tm_hit=1 with ok taken // from the stored disposition (pipeline/resume.go), so a resumed FLAGGED unit is `tm_hit=1, ok=0` — // a row that looks exactly like a failure and bought nothing. Without it the tm_hit test in the // filter is unreachable: an `ok` replay is already excluded by the ok test beside it, and a fixture // that carries only that one pins half a condition while reading as if it pinned both. rows = append(rows, store.RequestLogView{TS: "t6", Chapter: 1, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", TMHit: 1, OK: 1, CostUSD: 0}) // It carries a COST as well, which today's resume path does not write — and the row is built that // way on purpose. This renderer is a pure function of the rows it is handed, and a replayed row's // money was already booked by the row it replays; counting it again would inflate the denominator // the share is argued from. A fixture where every replay costs nothing pins the exclusion in the // numerator and leaves the denominator's half of it unreachable. rows = append(rows, store.RequestLogView{TS: "t7", Chapter: 3, Stage: "edit", Role: "editor", ModelRequested: "m", ModelActual: "m", TMHit: 1, OK: 0, FinishReason: "length", Degraded: "empty", CostUSD: 0.5}) // Two more shapes that carry ok=0 and are NOT waste, and both are taken from what the engine // really writes — a report that counted them would tell the owner a larger number than the truth, // which is the one direction a money line must never err in. // // t8: a call that never reached a billed response. The runner writes ok=0 with cost_usd=0 and the // cause in `err`, leaving `degraded` and `finish_reason` EMPTY — so counting it would also print a // bucket with no name. rows = append(rows, store.RequestLogView{TS: "t8", Chapter: 3, Stage: "edit", Role: "editor", ModelRequested: "m", ModelActual: "m", OK: 0, Err: "context canceled", CostUSD: 0}) // t9: a cosmetic sanitizer strip. The verdict is not ok and the cleaned text SHIPS — the reader // got exactly what this call paid for. rows = append(rows, store.RequestLogView{TS: "t9", Chapter: 1, Stage: "edit", Role: "editor", ModelRequested: "m", ModelActual: "m", OK: 0, FinishReason: "stop", Degraded: "sanitizer_stripped", CostUSD: 0.018654}) return rows } // TestTheReportAddsUpWhatWasPaidForAndThrownAway is the line the run of 11.09 had nowhere to print. The // FLAGS section reports CHUNKS whose final disposition is not ok, and all four of those units were // recovered by the regeneration and shipped — so a run that spent a quarter of its money on empty // replies printed a clean report. func TestTheReportAddsUpWhatWasPaidForAndThrownAway(t *testing.T) { var b strings.Builder if err := renderReport(&b, func() ([]store.RequestLogView, error) { return coldRunShapeRows(), nil }, // No flagged chunk at all: every unit recovered. This is the fixture's whole point. func() ([]store.ChunkStatus, error) { return nil, nil }, func() ([]store.RetrievalState, error) { return nil, nil }, pricedByAnswerer, okLedger); err != nil { t.Fatal(err) } out := b.String() if strings.Contains(out, "=== FLAGS") { t.Fatalf("the fixture must have NO flagged chunk, else it is not the case this line exists for:\n%s", out) } // FOUR, and the fixture carries THREE other rows that also read ok=0 and are not waste — a replay, // a $0 transport failure, and a cosmetic strip whose text shipped. The message names all three, // because whoever breaks this has to learn from it WHICH of them started being counted; a message // that names only one is a right verdict delivered with the wrong cause. if !strings.Contains(out, "PAID AND THROWN AWAY: 4 call(s), $0.106472") { t.Fatalf("the money of the thrown-away calls must be added up and printed, and none of these is a "+ "paid-for-nothing CALL: a replay (tm_hit), a failure that cost $0, a stripped answer that SHIPPED:\n%s", out) } // The share is what makes it an argument rather than a number. The DENOMINATOR is every fresh // billed row — including the stripped-but-shipped call, which really was paid for — so it is // $0.419423 + $0.018654 = $0.438077 and the share is 24.3%. The replayed row's $0.500000 is in // NEITHER half: it would drop the share to 11.4% and quietly make the waste look like a seventh of // what it is. if !strings.Contains(out, "24.3% of the $0.438077") { t.Fatalf("the line must say what share of the book's spend this is, and a replay is in neither half:\n%s", out) } // The breakdown separates the two causes the remedy has to tell apart. // Three empty, not four: the resumed flagged row carries `empty` too and must not swell the cause. if !strings.Contains(out, "empty×3") || !strings.Contains(out, "length×1") { t.Fatalf("the line must break the waste down by cause — three empty, one truncated:\n%s", out) } // ⚠ THE NEXT TWO ARE SCOPED TO THE LINE, not to the whole report. Both words appear in the ROW // TABLE above it — `sanitizer_stripped` as that row's own degraded tag — so a whole-output search // answers a question nobody asked and goes red on a correct report. line := paidLine(t, out) // A call that cost nothing is not money thrown away, and it would arrive with no cause to print. if strings.Contains(line, "unnamed") { t.Fatalf("a $0 transport failure must not enter a MONEY line, least of all as a nameless bucket: %q", line) } // A cosmetic strip ships its text. The reader got what it paid for. if strings.Contains(line, "sanitizer_stripped") { t.Fatalf("a stripped-but-SHIPPED answer is not waste; counting it overstates the loss: %q", line) } } // paidLine returns the PAID AND THROWN AWAY line itself, so an assertion about what the LINE says // cannot be satisfied — or broken — by the row table printed above it. func paidLine(t *testing.T, out string) string { t.Helper() for _, l := range strings.Split(out, "\n") { if strings.HasPrefix(l, "PAID AND THROWN AWAY") { return l } } t.Fatalf("the report carries no PAID AND THROWN AWAY line at all:\n%s", out) return "" } // TestACleanRunSaysNothingAboutWaste is the control. A section that printed on every run would be // noise, and a zero would be indistinguishable from a run nobody measured. func TestACleanRunSaysNothingAboutWaste(t *testing.T) { rows := []store.RequestLogView{ {TS: "t1", Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", FinishReason: "stop", OK: 1, CostUSD: 0.02}, {TS: "t2", Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", TMHit: 1, OK: 1, CostUSD: 0}, } var b strings.Builder if err := renderReport(&b, func() ([]store.RequestLogView, error) { return rows, nil }, func() ([]store.ChunkStatus, error) { return nil, nil }, func() ([]store.RetrievalState, error) { return nil, nil }, pricedByAnswerer, okLedger); err != nil { t.Fatal(err) } if out := b.String(); strings.Contains(out, "PAID AND THROWN AWAY") { t.Fatalf("a run where every call answered must not print the line at all:\n%s", out) } } // TestTheRunningReportSaysWhatTheBudgetWentTo covers the OTHER renderer. `tmctl translate` prints a // line per stage while a paid run is happening, and that is the reader who can still act on it — the // operator watching a book cost money. A thinking share visible only in the post-mortem arrives after // the budget is spent. // // The three cells are the same three answers the report table has to keep apart, and they are read by // their own marker (`think=`) rather than by a substring, because every other number on this line is // small and a bare "0" matches several of them. func TestTheRunningReportSaysWhatTheBudgetWentTo(t *testing.T) { ate, none := 8496, 0 res := &pipeline.BookResult{BookID: "b1", TotalUSD: 0.1, Chunks: []pipeline.ChunkOutcome{{ Chapter: 1, ChunkIdx: 0, Disposition: pipeline.DispOK, FinalText: "ТЕКСТ", Stages: []pipeline.StageResult{ {Stage: "draft", Model: "m", Disposition: pipeline.DispOK, Attempts: 2, FinishReason: "stop", Usage: llm.Usage{PromptTokens: 10, CompletionTokens: 8496, ReasoningInCompletion: &ate}}, {Stage: "edit", Model: "m", Disposition: pipeline.DispOK, Attempts: 1, FinishReason: "stop", Usage: llm.Usage{PromptTokens: 10, CompletionTokens: 2624, ReasoningInCompletion: &none}}, {Stage: "judge", Model: "m", Disposition: pipeline.DispOK, Attempts: 1, FinishReason: "stop", Usage: llm.Usage{PromptTokens: 10, CompletionTokens: 100}}, }, }}} var b strings.Builder if err := renderTranslate(&b, res, okLedger); err != nil { t.Fatal(err) } out := b.String() for stage, want := range map[string]string{"draft": "think=8496", "edit": "think=0", "judge": "think=?"} { line := stageLine(t, out, stage) if !strings.Contains(line, want) { t.Fatalf("the %s line must carry %q — a call that spent its budget thinking, one that did not, and one whose provider said nothing are three different answers: %q", stage, want, line) } } } // stageLine returns the running report's line for one stage, so an assertion about that stage cannot be // satisfied by a neighbouring one. func stageLine(t *testing.T, out, stage string) string { t.Helper() for _, l := range strings.Split(out, "\n") { if f := strings.Fields(l); len(f) > 0 && f[0] == stage { return l } } t.Fatalf("the running report has no line for stage %q:\n%s", stage, out) return "" } // TestAPaidFailureWithNoCauseIsCountedUnderAName is the branch that exists so a money line never prints // a bucket nobody can ask about — my own remedy for the nameless `×1` the first review found. // // ⚠ IT IS EXERCISED DIRECTLY, AND THE REASON IS STATED RATHER THAN GLOSSED. No writer in the engine // produces this row TODAY: every billed failure path fills `degraded` (the classifier's tag, the cut's // finish, `billed_2xx_decode_failed`), and the one shape that leaves both empty — a call that never // reached a billed response — costs $0 and is excluded a line earlier. So the branch is defensive over // the renderer's INPUT, and this renderer is a pure function of the rows it is handed: it must be right // for a row a future writer produces, not only for the ones today's writers do. Measured before this // test existed: a `panic` planted in that branch left the package green, while the same panic in a // reachable branch went red — the branch was decoration. func TestAPaidFailureWithNoCauseIsCountedUnderAName(t *testing.T) { rows := []store.RequestLogView{ {TS: "t1", Chapter: 1, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", OK: 0, CostUSD: 0.01, Degraded: "", FinishReason: ""}, {TS: "t2", Chapter: 1, Stage: "draft", Role: "translator", ModelRequested: "m", ModelActual: "m", OK: 1, CostUSD: 0.09, FinishReason: "stop"}, } var b strings.Builder if err := renderReport(&b, func() ([]store.RequestLogView, error) { return rows, nil }, func() ([]store.ChunkStatus, error) { return nil, nil }, func() ([]store.RetrievalState, error) { return nil, nil }, pricedByAnswerer, okLedger); err != nil { t.Fatal(err) } line := paidLine(t, b.String()) if !strings.Contains(line, "unnamed×1") { t.Fatalf("a BILLED failure that names no cause must still be counted under a name — an unlabelled "+ "bucket in a money line is a number the reader cannot ask about: %q", line) } if !strings.Contains(line, "1 call(s), $0.010000") { t.Fatalf("the billed failure must be counted and priced: %q", line) } }