textmachine/backend/internal/pipeline/moneystop_test.go

1042 lines
49 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"textmachine/backend/internal/chunk"
"textmachine/backend/internal/ledger"
"textmachine/backend/internal/obs"
"textmachine/backend/internal/runevents"
"textmachine/backend/internal/store"
)
// moneystop_test.go: the acceptance battery for backlog row 277/278 — a refused reservation means «do not
// start anything new», not «kill what is already running», and the run says how much was missing.
//
// Every test here is $0: the fake provider is an httptest server and no key of any real vendor is
// touched. The pack that ordered this work did NOT sanction a paid run.
// gatedProvider is a fake completion endpoint that HOLDS its first `hold` requests until the test
// releases them. It exists because the defect under test is a RACE with a very short window: the old
// behaviour cancelled sibling calls at the instant one reservation was refused, and a provider that
// answers immediately closes that window before a test can observe it. With the calls held open, the
// state «one call in flight, another worker refused» is not a timing accident but the only state the run
// can be in.
func gatedProvider(rec *reqRec, hold int, release <-chan struct{}) *httptest.Server {
held := make(chan struct{}, hold)
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
rec.record(string(body))
select {
case held <- struct{}{}:
// One of the first `hold` calls: wait until the test releases it, and ONLY that.
//
// ⛔ IT DELIBERATELY IGNORES THE REQUEST'S OWN CONTEXT, and the first version did not — which
// made two tests measure something else. With a `case <-r.Context().Done(): return` the held
// request answered the moment anything cancelled it, and the transport then RETRIED and got an
// instant answer from the `default` branch: a call the test believed frozen finished in 5.4 s
// and the run moved on. The subject here is what the CLIENT does when its call is killed, so
// the server must not quietly resurrect it.
<-release
default:
}
text, finish := draftEdit(string(body))
tb, _ := json.Marshal(text)
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":%q}],
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`,
tb, finish)
}))
}
// waitForRefusalInFlight blocks until the ceiling has REFUSED an admission while a call is STILL in
// flight — the exact window this whole pack is about — and reports whether it opened.
//
// ⛔ IT REPLACED A TRIGGER THAT OPENED A DIFFERENT WINDOW, and the difference cost the pack's named
// acceptance artifact. The first version released the held call as soon as a worker was QUEUED — but a
// worker queues only in the FIXED world, where a refusal makes it wait; in the broken world the refusal
// kills the wave outright and nobody ever queues. So the planting that restores the old behaviour did
// not red the test: it simply took a different, invisible path, and the assertion below was reached in
// both worlds with the same answer. Keyed on the REFUSAL, the window exists in both worlds, and what
// differs is what the run does with it — which is the thing under test.
func waitForRefusalInFlight(r *Runner) bool {
deadline := time.Now().Add(20 * time.Second)
for time.Now().Before(deadline) {
if inFlight, _ := r.gate.stateNow(); inFlight >= 1 && r.gate.refusalsSoFar() >= 1 {
return true
}
time.Sleep(2 * time.Millisecond)
}
return false
}
// waitForGate blocks until the run reaches the state the test is about and REPORTS whether it did. It
// polls the gate's own counters rather than sleeping for a guessed interval: what the test needs to
// observe is a STATE, and a sleep would assert a duration.
//
// ⚠ IT RETURNS A BOOL AND DOES NOT FAIL THE TEST, because every caller runs it on a SPAWNED goroutine.
// t.Fatalf outside the test goroutine calls runtime.Goexit on the wrong one: the assertion is lost, the
// helper's own goroutine dies silently, and the run it was supposed to unblock hangs until the package
// timeout — a green-looking suite turning into an unexplained timeout the moment a planting makes the
// state unreachable, which is exactly when the harness must be trustworthy.
func waitForGate(r *Runner, wantInFlight, wantWaiting int) bool {
deadline := time.Now().Add(20 * time.Second)
for time.Now().Before(deadline) {
if inFlight, waiting := r.gate.stateNow(); inFlight >= wantInFlight && waiting >= wantWaiting {
return true
}
time.Sleep(2 * time.Millisecond)
}
return false
}
// TestUnderACeilingTheWaveFinishesWhatItAdmitted is THE test of row 277, and its shape is the defect's
// shape. Four workers, a ceiling that admits one call, and the admitted call held open at the provider
// while a sibling is refused: the run must finish the held call and settle it.
//
// Before this pack the refusal cancelled the derived context, the held request died with
// `context.Canceled`, its reservation was released, and the book departed with committed spend of ZERO —
// the whole of the live 04.09 measurement, in one test.
func TestUnderACeilingTheWaveFinishesWhatItAdmitted(t *testing.T) {
rec := &reqRec{}
release := make(chan struct{})
srv := gatedProvider(rec, 1, release)
defer srv.Close()
// Five single-chunk chapters over four workers: several calls are attempted at once, and the ceiling
// admits one of them.
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100})
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
r := newRunner(t, bookPath)
defer r.Close()
r.CeilingUSD = oneCallCeilingUSD(t, r)
reached := make(chan bool, 1)
go func() {
// Release the held call only once the window this test is about is OPEN: the ceiling has refused
// somebody AND the admitted call is still running. Released either way — a run left holding a
// request forever would turn a failed assertion into a package timeout.
reached <- waitForRefusalInFlight(r)
close(release)
}()
res, err := r.TranslateBook(ctx)
if !<-reached {
t.Fatal("the ceiling never refused anything while a call was in flight — this fixture does not build the situation the pack is about")
}
if err == nil {
t.Fatal("a ceiling that admits one call must still stop the book")
}
var halt *CeilingHalt
if !errors.As(err, &halt) {
t.Fatalf("the stop must be a ceiling halt, got %v", err)
}
committed, reserved, serr := r.Store.SpentUSD("test-book")
if serr != nil {
t.Fatal(serr)
}
// ⛔ THE ASSERTION THE PACK EXISTS FOR. The admitted call was finished and paid for. Zero here is the
// old behaviour exactly: a run that spent nothing, produced nothing and took ten seconds to do it.
if committed <= 0 {
t.Fatalf("the admitted call was killed by a sibling's refusal: committed=%v, provider saw %d call(s)", committed, rec.count())
}
if reserved != 0 {
t.Fatalf("a reservation leaked through the stop: reserved=%v", reserved)
}
if rec.count() == 0 {
t.Fatal("nothing reached the provider at all — the fixture no longer admits a single call")
}
// The run departs with a RESULT, not with nothing: exit 4 and `paused` stay, and what was bought is
// accounted for rather than discarded.
if res == nil {
t.Fatal("a ceiling stop must carry the run's partial result, or the operator is told only that it stopped")
}
}
// TestASpendStopReportsNoVolumeLedger is the withholding of VolumeStop, asserted where it can FAIL.
//
// ⛔ THE ASSERTION LIVED IN THE TEST ABOVE AND WAS VACUOUS THERE: that fixture grants no volume at all,
// so `res.Volume` is nil whatever the code does, and the planting that removes the withholding SURVIVED.
// A volume ledger can only be wrongly attached by a run that HAS a volume grant, so that is what this
// fixture builds — and the money ceiling then stops it in the middle of the grant, which is the only
// state where the seven plan counters would describe units that were admitted and never done.
func TestASpendStopReportsNoVolumeLedger(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
// ONE worker: the admission order is then the item order, and the arithmetic below is a statement
// rather than a hope. What this test is about is which WAVE the stop lands in, not concurrency.
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(4, 1400), waveWorkers: 1, bookUSD: 100})
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
r := newRunner(t, bookPath)
defer r.Close()
// ⛔ BOTH NUMBERS ARE LOAD-BEARING, AND THE SECOND ONE TOOK TWO ATTEMPTS.
//
// `MaxUnits` gives the run a REAL volume grant, so the seven plan counters exist at all — without one
// `res.Volume` is nil whatever the code does, and the first version of this test asserted that nil.
//
// The ceiling must then stop the run IN THE EDIT WAVE, and that is the part the second attempt got
// wrong: `res.Volume` is attached BETWEEN the waves (waverun.go, right after the draft wave), so a
// ceiling tight enough to stop the DRAFT wave returns through a path that never attaches it — and the
// withholding under test is a no-op there too. So the ceiling is sized to pay for every draft of the
// grant and to run out on the first edit: three drafts settle at a fraction of their estimate, and the
// edit call's own estimate no longer fits.
r.MaxUnits = 3
r.CeilingUSD = callCeilingUSD(t, r, 1.64)
res, err := r.TranslateBook(ctx)
if err == nil {
t.Fatal("the tight ceiling must stop the run inside its volume grant")
}
var halt *CeilingHalt
if !errors.As(err, &halt) {
t.Fatalf("want a ceiling halt, got %v", err)
}
if res == nil {
t.Fatal("a ceiling stop must carry the run's partial result")
}
if res.Volume != nil {
t.Fatalf("a SPEND stop reported a VOLUME ledger %+v — those seven counters are a PLAN, and a run stopped by money leaves them counting units that were admitted and never done: Delivered over-counts, LeftFresh under-counts, and at Left()==0 the run would announce it reached the end of the book",
*res.Volume)
}
}
// oneCallCeilingUSD is a ceiling that admits exactly one of this book's calls: halfway between one
// reservation and two. It is DERIVED from the engine's own arithmetic rather than written as a literal,
// so a change to sizing or pricing moves the fixture with the code instead of silently making the test
// about a different situation.
func oneCallCeilingUSD(t *testing.T, r *Runner) float64 {
return callCeilingUSD(t, r, 1.5)
}
// callCeilingUSD is `mult` times ONE draft call's reservation for this fixture's chunk size. Derived from
// the engine's own arithmetic rather than written as a literal, so a change to sizing or pricing moves
// the fixture with the code instead of quietly making the test about a different situation. The sizing
// length is the fixture's own chapter length (cjkSource(_, chapterRunes)).
func callCeilingUSD(t *testing.T, r *Runner, mult float64) float64 {
t.Helper()
plan := r.pricePlanFor()
if plan == nil || len(plan.stages) == 0 {
t.Fatal("the fixture pipeline resolves no priceable stage")
}
sp := plan.stages[0]
base := r.baseMaxTokensFor(sp.st, fixtureChapterRunes)
// ⚠ NO INJECTION ALLOWANCE HERE, and the omission is the fixture's truth rather than a shortcut. These
// projects carry no glossary, so the bank injection is not rendered at all and the executor reserves
// without it. Charging the plan's 800-token allowance inflated every ceiling derived here by ~11 %, and
// that quietly turned a «one reservation at a time» fixture into one where two fit — which is how the
// delivering-ceiling test below came to pass without the waiting path ever running.
one := ledger.EstimateUSD(sp.price, sp.tplTokens+fixtureChapterRunes, base, sp.reasoning)
if one <= 0 {
t.Fatal("the fixture's first stage is free — a ceiling test needs a priced model")
}
return one * mult
}
// fixtureChapterRunes is the chapter length every fixture in this file uses. Named once so the ceilings
// derived from it cannot drift away from the books they are derived for.
const fixtureChapterRunes = 1400
// TestARefusalWithNothingInFlightIsFinal pins the other half of the rule: waiting is what makes a refusal
// non-final, and there is nothing to wait for when no call holds a reservation. A single-worker run whose
// ceiling admits nothing must stop at once rather than hang.
func TestARefusalWithNothingInFlightIsFinal(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{bookUSD: 0.0000001})
r := newRunner(t, bookPath)
defer r.Close()
done := make(chan error, 1)
go func() { _, err := r.TranslateBook(context.Background()); done <- err }()
select {
case err := <-done:
if err == nil {
t.Fatal("a ceiling that admits nothing must stop the run")
}
var halt *CeilingHalt
if !errors.As(err, &halt) {
t.Fatalf("want a ceiling halt, got %v", err)
}
case <-time.After(30 * time.Second):
t.Fatal("the run HUNG on a refusal with nothing in flight — waiting for a settle that can never come")
}
if rec.count() != 0 {
t.Fatalf("a denied reserve must not reach the provider, calls=%d", rec.count())
}
}
// TestTheGateWaitsWhileTheSkyIsFullAndStopsWhenItEmpties exercises the gate directly, because the two
// answers it gives are what the whole behaviour rests on and they must be pinned without a provider, a
// store or a clock.
func TestTheGateWaitsWhileTheSkyIsFullAndStopsWhenItEmpties(t *testing.T) {
var g reserveGate
if got := g.waitForSettle(context.Background(), true); got != waitNothingInFlight {
t.Fatalf("with nothing in flight the refusal is final, got outcome %d", got)
}
// Simulate an admitted call by hand — admit() needs a store, and what is under test here is the
// counter's contract, not the reservation's.
g.mu.Lock()
g.inFlight = 1
g.mu.Unlock()
woke := make(chan waitOutcome, 1)
go func() { woke <- g.waitForSettle(context.Background(), true) }()
// The waiter must be parked before the settle, or the test proves nothing about the broadcast.
deadline := time.Now().Add(5 * time.Second)
for {
if _, waiting := g.stateNow(); waiting == 1 {
break
}
if time.Now().After(deadline) {
t.Fatal("the waiter never parked")
}
time.Sleep(time.Millisecond)
}
g.done()
select {
case got := <-woke:
if got != waitSettled {
t.Fatalf("a settle must wake the waiter so it can re-ask for the headroom that just grew, got outcome %d", got)
}
case <-time.After(5 * time.Second):
t.Fatal("the settle did not wake the waiter — the broadcast channel was replaced without being closed")
}
if inFlight, waiting := g.stateNow(); inFlight != 0 || waiting != 0 {
t.Fatalf("after the settle the gate must be empty, got in_flight=%d waiting=%d", inFlight, waiting)
}
// And a context that dies while waiting must release the waiter rather than strand it.
g.mu.Lock()
g.inFlight = 1
g.mu.Unlock()
ctx, cancel := context.WithCancel(context.Background())
cancel()
// ⛔ AND IT MUST BE ITS OWN ANSWER. Reported as «nothing in flight», a cancelled wait becomes a
// CeilingHalt one level up, and a run somebody stopped with Ctrl-C publishes a `ceiling` event with
// a shortfall — telling the platform to add money for a stop that had nothing to do with money.
if got := g.waitForSettle(ctx, true); got != waitAborted {
t.Fatalf("a dead context must be distinguishable from an empty sky, got outcome %d", got)
}
}
// TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent pins the futility check. Waiting is bounded
// by the longest provider timeout, so waiting for a settle that provably cannot admit the call costs
// minutes to arrive at the answer already in hand.
func TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent(t *testing.T) {
// committed 0.9, estimate 0.2, ceiling 1.0 — every reservation could be released for free and 1.1
// would still be past 1.0.
if !settleCannotHelp(runevents.ScopeBook, 0.9, 0.2, 1.0) {
t.Error("committed alone already leaves no room: no settle can change that")
}
// committed 0.5, estimate 0.2 — the refusal came from RESERVED money, which a settle gives back.
if settleCannotHelp(runevents.ScopeBook, 0.5, 0.2, 1.0) {
t.Error("the refusal is about reserved money, and reserved money is exactly what a settle returns")
}
// The DAY ceiling sums other books, so this process's committed figure is not the day's: it must not
// be used to declare a refusal final.
if settleCannotHelp(runevents.ScopeDay, 0.9, 0.2, 1.0) {
t.Error("the day ceiling is not this book's arithmetic, so this book's committed spend cannot rule the wait out")
}
// No ceiling means no refusal to reason about.
if settleCannotHelp(runevents.ScopeBook, 9, 9, 0) {
t.Error("a zero ceiling is «no limit» and cannot refuse anything")
}
}
// TestTheShortfallIsTheDistanceToTheCeilingAndNotThePriceOfACall pins WHICH number leaves the engine.
// The denied estimate is the price of one call and the contract keeps our cost structure inside; the
// shortfall is the distance between a limit the platform set and a total it has already authorised.
func TestTheShortfallIsTheDistanceToTheCeilingAndNotThePriceOfACall(t *testing.T) {
// committed 0.30 + reserved 0.20 + estimate 0.07 = 0.57 against a ceiling of 0.55 → 0.02 missing.
got := shortfallMicroUSD(runevents.ScopeBook, 0.30, 0.20, 0.07, 0.55)
if got != 20_000 {
t.Errorf("shortfall = %d micro-USD, want 20000 (the amount the admission overshot by)", got)
}
// It is NOT the estimate: publishing that would publish the price of a call.
if got == 70_000 {
t.Error("the engine published the DENIED ESTIMATE — that is the price of one call and it may not leave")
}
// Rounded UP: a figure a person tops up against must never be short.
if up := shortfallMicroUSD(runevents.ScopeBook, 0, 0, 0.0000005, 0); up != 0 {
t.Errorf("a zero ceiling states nothing, got %d", up)
}
if up := shortfallMicroUSD(runevents.ScopeBook, 1, 0, 0.0000004, 1); up != 1 {
t.Errorf("a sub-micro-dollar shortfall must round UP to 1, got %d", up)
}
// The day scope cannot state it honestly and therefore does not.
if d := shortfallMicroUSD(runevents.ScopeDay, 0.9, 0, 0.2, 1.0); d != 0 {
t.Errorf("the day scope must state no shortfall (its figures are one book's, its ceiling is every book's), got %d", d)
}
}
// TestTheStoppedRunTellsAConsumerHowMuchWasMissing is the SEAM axis, read from the consumer's side: the
// events file is parsed as bytes, exactly as the platform tails it, rather than by asking the engine's
// own structs what they hold.
func TestTheStoppedRunTellsAConsumerHowMuchWasMissing(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(3, 1400), bookUSD: 100})
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
r := newRunner(t, bookPath)
r.CeilingUSD = callCeilingUSD(t, r, 1.05) // one reservation fits; the book needs six calls
if _, err := r.TranslateBook(ctx); err == nil {
t.Fatal("the tight ceiling must stop the run")
}
journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl")
r.Close()
raw, err := os.ReadFile(journal)
if err != nil {
t.Fatalf("no event journal to read from the consumer side: %v", err)
}
var sawCeiling, sawFinished bool
var version string
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
if line == "" {
continue
}
var env struct {
Type string `json:"type"`
Data json.RawMessage `json:"data"`
}
if err := json.Unmarshal([]byte(line), &env); err != nil {
t.Fatalf("the stream is not readable NDJSON: %v (%s)", err, line)
}
switch env.Type {
case "hello":
var h struct {
StreamVersion string `json:"stream_version"`
}
mustJSON(t, env.Data, &h)
version = h.StreamVersion
case "ceiling":
var c struct {
Halted bool `json:"halted"`
Scope string `json:"scope"`
ShortfallMicroUSD int64 `json:"shortfall_micro_usd"`
}
mustJSON(t, env.Data, &c)
sawCeiling = true
if !c.Halted || c.Scope != "book" {
t.Errorf("the ceiling frame lost its existing fields: %+v", c)
}
// ⛔ THE FIELD THIS PACK ADDS. Without it the number the engine already printed dies in the
// stderr of a transient unit and the buyer is told the run stopped and nothing else.
if c.ShortfallMicroUSD <= 0 {
t.Errorf("the ceiling frame does not say how much was missing: %+v", c)
}
case "finished":
var f struct {
Outcome string `json:"outcome"`
Money *struct {
UnitsResolved int `json:"units_resolved"`
UnitsDeferred int `json:"units_deferred"`
} `json:"money"`
}
mustJSON(t, env.Data, &f)
sawFinished = true
if f.Outcome != "ceiling" {
t.Errorf("outcome = %q, want ceiling — a resumable stop is never `failed` (PD-113)", f.Outcome)
}
if f.Money == nil {
t.Error("a run stopped by money must say what it bought and what is still owed")
} else if f.Money.UnitsDeferred == 0 {
t.Errorf("this fixture cannot afford the whole book, so something must be deferred: %+v", f.Money)
}
}
}
if !sawCeiling || !sawFinished {
t.Fatalf("the stream is incomplete: ceiling=%v finished=%v", sawCeiling, sawFinished)
}
// The seam rule: a frame gained a field, so the version says so. A reader comparing only the major is
// unaffected — and a stream whose bytes changed while its version did not is the one thing a version
// exists to prevent (runevents.go records this exact miss, made once already).
if version != runevents.StreamVersion {
t.Errorf("hello says stream_version=%q, build says %q", version, runevents.StreamVersion)
}
if version == "1.2" {
t.Error("the stream gained a field and kept its old version — the miss the constant's own comment records")
}
}
func mustJSON(t *testing.T, raw []byte, into any) {
t.Helper()
if err := json.Unmarshal(raw, into); err != nil {
t.Fatalf("frame payload unreadable: %v (%s)", err, raw)
}
}
// TestResumeAfterAMoneyStopRePaysNothing is the DETERMINISM axis. A run stopped by money, resumed under a
// raised ceiling, must finish the book without buying again what it already bought — the admitted calls
// were settled and checkpointed, so they replay for free.
func TestResumeAfterAMoneyStopRePaysNothing(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(3, 1400), waveWorkers: 2, bookUSD: 100})
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
r1 := newRunner(t, bookPath)
r1.CeilingUSD = callCeilingUSD(t, r1, 1.3) // room for a call or two, not for the book
if _, err := r1.TranslateBook(ctx); err == nil {
t.Fatal("the tight ceiling must stop the run")
}
firstPass := rec.count()
committedAfterStop, reservedAfterStop, err := r1.Store.SpentUSD("test-book")
if err != nil {
t.Fatal(err)
}
r1.Close()
if firstPass == 0 {
t.Fatal("fixture drifted: the stopped run must have paid for something")
}
if reservedAfterStop != 0 {
t.Fatalf("a stopped run must leave no reservation behind, got %v", reservedAfterStop)
}
r2 := newRunner(t, bookPath)
defer r2.Close()
if _, err := r2.TranslateBook(ctx); err != nil {
t.Fatalf("the resumed run must finish the book under the book's own ceiling: %v", err)
}
committedAfterFinish, _, err := r2.Store.SpentUSD("test-book")
if err != nil {
t.Fatal(err)
}
// ⛔ THE ASSERTION IS THE TOTAL, and its first version was two statements that cannot fail. «Committed
// spend went down» is impossible — committed only grows — and `secondPass >= firstPass + rec.count()`
// is arithmetically unreachable, since `secondPass` is defined as `rec.count() - firstPass`. Both read
// like checks and neither could ever red. What actually catches a re-purchase is the COUNT of provider
// calls across both runs: this book is 3 units × 2 stages, so a resume that re-bought its first pass
// would show more than six.
const callsInTheWholeBook = 3 * 2
secondPass := rec.count() - firstPass
if secondPass <= 0 {
t.Fatal("the resume did no work at all — the fixture is not testing a resume")
}
if total := rec.count(); total != callsInTheWholeBook {
t.Fatalf("the book cost %d provider calls across the stop and the resume, want exactly %d — anything more is work bought twice (first pass %d, resume %d)",
total, callsInTheWholeBook, firstPass, secondPass)
}
if committedAfterFinish <= committedAfterStop {
t.Fatalf("the resume settled nothing: committed %v → %v", committedAfterStop, committedAfterFinish)
}
}
// TestAMoneyStopInTheDraftWaveNeverStartsTheEditWave pins the rule that a partial draft is not editable.
//
// ⛔ THE FAILURE IT PREVENTS IS PAID. The edit wave looks a member's draft up in a map built over EVERY
// chunk, so an unstarted member is FOUND — holding the zero value: not flagged, empty text. It would be
// joined into the unit's draft as a clean member, and the editor would be paid to edit a unit with a hole
// in it and then ship it.
func TestAMoneyStopInTheDraftWaveNeverStartsTheEditWave(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(4, 1400), waveWorkers: 2, bookUSD: 100})
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
r := newRunner(t, bookPath)
defer r.Close()
r.CeilingUSD = callCeilingUSD(t, r, 1.05) // one draft call fits; the book needs eight
if _, err := r.TranslateBook(ctx); err == nil {
t.Fatal("the tight ceiling must stop the run in the draft wave")
}
for _, body := range rec.all() {
if isEditBody(body) {
t.Fatal("an EDIT call was issued after the draft wave was stopped by money — the editor was paid to edit a unit whose members were never drafted")
}
}
}
// TestNothingLeaksWhenTheGateSeesAnAdmissionError pins the counter's own failure mode: a reservation the
// store refused with an ERROR (not a denial) must not be counted as in flight, or every later refusal
// waits for a settle that can never arrive and the run hangs instead of stopping.
func TestNothingLeaksWhenTheGateSeesAnAdmissionError(t *testing.T) {
var g reserveGate
// A closed store makes Reserve fail as an error rather than deny.
dir := t.TempDir()
st, err := store.Open(filepath.Join(dir, "p.db"))
if err != nil {
t.Fatal(err)
}
st.Close()
if _, _, err := g.admit(st, "b", 0.01, store.Ceilings{BookUSD: 100}); err == nil {
t.Skip("this store does not error on a closed handle; the counter's contract is pinned by the gate test above")
}
if inFlight, _ := g.stateNow(); inFlight != 0 {
t.Fatalf("a failed admission must not count as in flight, got %d", inFlight)
}
}
// TestWaitingForHeadroomDELIVERSTheWholeBook is the positive half of row 277 and the one the rest of the
// battery does not reach. Every other ceiling test ends in a stop; this one ends in a FINISHED BOOK that
// a ceiling would have refused without the wait.
//
// The shape is the real one. A reservation is worst-case (whole prompt uncached, completion running to
// max_tokens) while a settle is the actual cost, so a ceiling can be too small to hold two reservations
// at once and perfectly able to pay for the whole book one call at a time. Before the wait existed, the
// second parallel worker's refusal killed the run; with it, the wave goes serial under the ceiling and
// the book is delivered.
func TestWaitingForHeadroomDeliversTheWholeBook(t *testing.T) {
rec := &reqRec{}
// ⚠ THE FIRST CALL IS HELD OPEN, and that is what makes the collision a STATE rather than a race the
// test hopes for. Without it the two draft workers may never hold a reservation at the same moment —
// under `-race` they reliably do not: the battery caught this test green-by-luck and then failing,
// with the log showing chunk 0 settled before chunk 1 was ever admitted. A ceiling cannot refuse what
// is not asked concurrently, so the waiting path went unexercised and the assertion below said so.
release := make(chan struct{})
srv := gatedProvider(rec, 1, release)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(2, 1400), waveWorkers: 4, bookUSD: 100})
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
r := newRunner(t, bookPath)
defer r.Close()
// Room for ONE reservation at a time, and — because a settle costs a fraction of its estimate — room
// in TOTAL for every call this two-chapter book makes. Both halves are needed and neither is
// incidental: the first is what makes the parallel wave collide with the ceiling at all, the second is
// what makes the collision survivable. The ceiling stays cumulative throughout, which is why the book
// is two chapters and not ten.
// The window is narrow and both edges are real: BELOW two reservations (so the parallel wave collides
// with the ceiling and a worker has to wait) and ABOVE the whole book's settled cost (so the waiting
// wave finishes it). A settle costs a fraction of its estimate, which is what makes such a window
// exist at all — and it is why the book is two chapters and not ten, the ceiling being cumulative.
r.CeilingUSD = callCeilingUSD(t, r, 1.85)
reached := make(chan bool, 1)
go func() {
// Release the held call only once a worker is genuinely queued behind the ceiling — the state the
// whole test is about. Released either way, so a missed state fails an assertion instead of
// hanging the package.
reached <- waitForGate(r, 1, 1)
close(release)
}()
res, err := r.TranslateBook(ctx)
if !<-reached {
t.Fatal("no worker was ever queued behind the ceiling — the two draft reservations never overlapped, so this fixture does not exercise the waiting path")
}
if err != nil {
t.Fatalf("a ceiling that admits one call AT A TIME must still deliver the book: %v", err)
}
// ⛔ THE MECHANISM, ASSERTED — not inferred from the outcome. This test's first version checked only
// that the book was delivered, and it PASSED on a fixture whose ceiling never refused anything: the
// waiting path did not run, the mutant that deletes the wait would have survived, and the assertion
// read as proof of a road the run never took. A delivered book is not evidence of waiting; a wait is.
if waits := r.gate.waitsSoFar(); waits == 0 {
t.Fatalf("the book was delivered WITHOUT the ceiling ever refusing a reservation — this fixture does not exercise the waiting path at all, so it proves nothing about it (shipped %d units)", len(res.Chunks))
}
if len(res.Chunks) != 2 {
t.Fatalf("want 2 shipped units, got %d — the wave stopped instead of queueing", len(res.Chunks))
}
for _, oc := range res.Chunks {
if oc.Disposition != DispOK {
t.Fatalf("a unit came back %s under the queueing ceiling", oc.Disposition)
}
}
committed, reserved, serr := r.Store.SpentUSD("test-book")
if serr != nil {
t.Fatal(serr)
}
if reserved != 0 {
t.Fatalf("no reservation may survive a completed run, reserved=%v", reserved)
}
// ⛔ AND THE CEILING STILL HELD. The whole point of keeping the reservation is that queueing never
// becomes overspending: committed spend is under the limit that was in force, every call of it.
if committed > r.CeilingUSD {
t.Fatalf("the queueing wave OVERSPENT its ceiling: committed=%v ceiling=%v", committed, r.CeilingUSD)
}
}
// TestACancelledWaitIsNotAMoneyStop pins the defect that a boolean answer created and a three-valued one
// closes. A worker waiting for headroom when the run is cancelled must surface the CANCELLATION.
//
// ⛔ WHAT IT PREVENTS. Reported as «nothing in flight», a cancelled wait became a CeilingHalt one level
// up: the run published a `ceiling` event carrying a shortfall, and the platform — which lets that event
// survive any exit code — recorded `paused` and told a person to add money for a run they had stopped
// themselves. The same path was reachable from a sibling's infra failure, which cancels.
func TestACancelledWaitIsNotAMoneyStop(t *testing.T) {
rec := &reqRec{}
release := make(chan struct{})
srv := gatedProvider(rec, 1, release)
defer srv.Close()
// ⚠ DECLARED AFTER srv's defer SO IT RUNS BEFORE IT — defers are LIFO, and httptest.Server.Close()
// WAITS for outstanding requests. Ordered the other way this deadlocks the package: the handler is
// parked on `release`, Close() is parked on the handler, and the close that would free both is queued
// behind Close(). Written the wrong way round it HUNG the package rather than failing it — the
// timeout-shaped failure this pack has already recorded as a class of its own, met once more.
defer close(release)
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100})
ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}))
r := newRunner(t, bookPath)
r.CeilingUSD = oneCallCeilingUSD(t, r)
reached := make(chan bool, 1)
go func() {
// Cancel only once a worker is genuinely parked on the gate — otherwise the test would be about
// cancelling a run, not about cancelling a WAIT. Cancelled either way, so a missed state fails an
// assertion instead of hanging the package.
reached <- waitForGate(r, 1, 1)
cancel()
}()
_, err := r.TranslateBook(ctx)
if !<-reached {
t.Fatal("no worker was ever parked on the gate — the cancellation did not interrupt a WAIT and this test is about nothing")
}
journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl")
r.Close()
if err == nil {
t.Fatal("a cancelled run must not report success")
}
var halt *CeilingHalt
if errors.As(err, &halt) {
t.Fatalf("a run cancelled while waiting for headroom reported a MONEY stop: %v", err)
}
if !errors.Is(err, context.Canceled) {
t.Fatalf("the cancellation must be what leaves the run, got %v", err)
}
raw, rerr := os.ReadFile(journal)
if rerr != nil {
t.Fatalf("no event journal: %v", rerr)
}
if strings.Contains(string(raw), `"ceiling"`) {
t.Fatalf("the stream published a ceiling stop for a run somebody cancelled:\n%s", raw)
}
}
// TestAnInfraFailureAfterAMoneyStopDoesNotDepartAsPaused is the two-slot error ranking, asserted.
//
// ⛔ WHY IT NEEDS ITS OWN TEST. A spend ceiling no longer cancels its siblings, so for the first time a
// run can latch on money and THEN break for an unrelated reason. Sharing one error slot, whichever
// happened first would speak for both — and a first-wins ceiling would have a broken run depart exit 4,
// which the platform records as `paused`: «add money» offered as the remedy for a machine that failed.
// So the failure ranks on the way out, and the money fact travels on the channel it was given for
// exactly this case: the `ceiling` event, written at the moment of the refusal.
func TestAnInfraFailureAfterAMoneyStopDoesNotDepartAsPaused(t *testing.T) {
r := &Runner{}
halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 1234,
err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)}
infra := errors.New("provider unreachable")
// ⚠ BOTH WORKERS MUST BE HOLDING AN ITEM BEFORE EITHER FAILS, and the first attempt at this test got
// it wrong in a way worth keeping: with the ceiling returned first, the latch closed and the feeder
// never handed out the second item at all — so the failure never happened and the test asserted
// nothing. That is the latch behaving exactly as designed (nothing new starts), and it is also why
// this co-occurrence is only reachable among calls that were ALREADY admitted.
holding := make(chan struct{})
err := r.runWave(context.Background(), 2, 2, func(ctx context.Context, i int) error {
if i == 1 {
close(holding)
return infra
}
<-holding
return halt
})
if err == nil {
t.Fatal("the wave must report something")
}
var got *CeilingHalt
if errors.As(err, &got) {
t.Fatalf("a run that BROKE departed as a money stop (exit 4 → the platform records `paused`): %v", err)
}
if !errors.Is(err, infra) {
t.Fatalf("the failure must be what leaves the wave, got %v", err)
}
}
// TestAnOptionalCallIsTurnedAwayWhileMandatoryOneWaits is the `mandatory` rule, ASSERTED.
//
// ⛔ ITS ENTIRE BEHAVIOUR USED TO BE AN ARGUMENT IN A COMMENT. Every fixture that refused an optional call
// was sequential, so `inFlight == 0` produced an immediate halt whichever way the flag was set — flipping
// all three optional call sites to `true` left the battery green. The flag decides something ONLY when a
// sibling is in flight, so that is the state this test builds, at the level where the rule now lives.
//
// What it protects: an escalation hop that queued for headroom would spend the money the primary wave
// needs to finish work it has already started, and it holds escMu while it waits, so it would block every
// other worker's escalation too. The repair sub-step runs INSIDE an edit-wave worker and is the same
// story. All three degrade on a ceiling; none of them may wait for one.
func TestAnOptionalCallIsTurnedAwayWhileMandatoryOneWaits(t *testing.T) {
var g reserveGate
g.mu.Lock()
g.inFlight = 1 // money IS still moving: this is the only state in which the rule decides anything
g.mu.Unlock()
// The optional caller is turned away AT ONCE — not because there is nothing to wait for, but because
// it has no claim on it. The two are opposite facts and must not share an answer.
//
// ⚠ THE CONTEXT IS BOUNDED, AND THAT IS WHAT GIVES THE FAILURE A SHAPE. With context.Background() a
// broken rule does not fail here, it HANGS: the optional caller parks on a settle that never comes in
// a unit test, the package times out, and the catalogue reports `NOTHING` — «the planting did not
// build, or the run died» — which is indistinguishable from machine load. That is the timeout-shaped
// failure this pack has already recorded as a class, met an eighth time and caught immediately: with
// a deadline the same break returns waitAborted in two seconds and reds an assertion.
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if got := g.waitForSettle(ctx, false); got != waitNotAllowed {
t.Fatalf("an optional call refused while money is still moving must be turned away, got outcome %d — an optional call that queues for the last dollars spends what the mandatory work needed, and the escalation hop blocks every other worker's escalation while it waits", got)
}
if n := g.optionalTurnedAwaySoFar(); n != 1 {
t.Fatalf("the turn-away was not recorded (%d) — without the fact there is nothing a test can see", n)
}
if w := g.waitsSoFar(); w != 0 {
t.Fatalf("an optional call WAITED (%d): «do not start anything new» became «the optional spends what the mandatory needed»", w)
}
// …and the MANDATORY caller in the very same state does wait, and is woken by the settle.
woke := make(chan waitOutcome, 1)
go func() { woke <- g.waitForSettle(context.Background(), true) }()
deadline := time.Now().Add(5 * time.Second)
for {
if _, waiting := g.stateNow(); waiting == 1 {
break
}
if time.Now().After(deadline) {
t.Fatal("the mandatory caller never parked — the rule turns everyone away, which stops the book it was written to keep alive")
}
time.Sleep(time.Millisecond)
}
g.done()
select {
case got := <-woke:
if got != waitSettled {
t.Fatalf("the mandatory caller must be woken by the settle, got outcome %d", got)
}
case <-time.After(5 * time.Second):
t.Fatal("the settle did not wake the mandatory caller")
}
if w := g.waitsSoFar(); w != 1 {
t.Fatalf("waits=%d, want exactly the mandatory caller's one", w)
}
if n := g.optionalTurnedAwaySoFar(); n != 1 {
t.Fatalf("the mandatory caller was counted as an optional turn-away (%d)", n)
}
}
// TestOnlyOneCeilingFrameReachesTheStreamHoweverManyWorkersAreRefused pins the idempotence the emitter
// claims. Under one worker a run is refused once and the claim is untestable; under four it is refused
// several times, and a reader that received a `ceiling` frame per refusal would count one stop as many.
func TestOnlyOneCeilingFrameReachesTheStreamHoweverManyWorkersAreRefused(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, draftEdit)
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100})
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
r := newRunner(t, bookPath)
r.CeilingUSD = callCeilingUSD(t, r, 1.05)
if _, err := r.TranslateBook(ctx); err == nil {
t.Fatal("the tight ceiling must stop the book")
}
journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl")
r.Close()
raw, err := os.ReadFile(journal)
if err != nil {
t.Fatal(err)
}
frames := 0
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
var env struct {
Type string `json:"type"`
}
if line == "" || json.Unmarshal([]byte(line), &env) != nil {
continue
}
if env.Type == "ceiling" {
frames++
}
}
if frames != 1 {
t.Fatalf("the stream carries %d ceiling frames for ONE stop — a reader counting stops would count refusals instead", frames)
}
}
// ⛔ THERE IS NO TEST HERE FOR settleCannotHelp's WIRING, and the absence is deliberate and argued.
//
// The predicate has its own test (TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent). What no
// test in this file exercises is the `if` that consults it — and the reason is that the state it fires in
// is nearly unreachable in a wave of similar calls. It needs a refusal that no settle could lift
// (`committed + estimate > ceiling`) WHILE another call is still in flight. But the in-flight call was
// itself admitted at the same committed level with a comparable estimate, so if there was room for it
// there is room for this one: the two conditions contradict each other unless the estimates differ
// materially — heterogeneous chunk sizes, or a draft and an edit meeting in one wave, which they never do.
// And once the sky empties, `waitNothingInFlight` stops the book anyway, with or without the predicate.
//
// ⚠ A TEST THAT CLAIMED TO COVER IT STOOD HERE AND WAS DELETED, because it passed for a reason that had
// nothing to do with its name. It held a call at the fake provider and asserted the run ended while the
// call was still held — but the held request hit the attempt timeout, the transport RETRIED, and the
// retry answered instantly (the trace shows `latency_ms=5389` on a call the test believed was frozen).
// So the run ended because the call finished, the three refused workers had WAITED and been woken, and
// the assertion was about a situation the fixture never built. Removing it is the honest move: the pack's
// own rule is that «checked» without an artifact does not count, and an artifact that checks something
// else is worse than none, because it reads as coverage. What remains is this paragraph and the entry in
// the report's obstacle section.
// TestTheTerminalVocabularyNeverContradictsTheExitCode pins the ORDER of terminal()'s branches — a
// regression made and caught inside this very pack, and until now held by nothing but a comment.
//
// ⛔ WHAT WENT WRONG. The money ledger is attached whenever a ceiling was reached, and the branch that
// does it was written ABOVE the signature stop and the cancellation. It swallowed both: a bank-stop on a
// book that had touched its ceiling reported `failed` while its exit code said 3, and a caught SIGTERM
// reported `failed` while its exit code said 5. Two channels contradicting each other about one run is
// exactly what this vocabulary exists to prevent — and re-ordering `case` arms is precisely the edit a
// tidy-up makes without noticing.
func TestTheTerminalVocabularyNeverContradictsTheExitCode(t *testing.T) {
halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 42,
err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)}
for _, tc := range []struct {
name string
err error
outcome string
}{
// Each of these has its OWN exit code (3, 5, 4, 1), and the stream must say the same thing.
{"a signature stop on a book that touched its ceiling", &WaveSignatureStop{Terms: 1, SignaturePath: "/x"}, runevents.OutcomeBankStop},
{"a caught SIGTERM after the money ran out", context.Canceled, runevents.OutcomeStopped},
{"the ceiling stop itself", halt, runevents.OutcomeCeiling},
{"an infra failure on top of a caught ceiling", errors.New("provider unreachable"), runevents.OutcomeFailed},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
st, err := store.Open(filepath.Join(dir, "p.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
e, err := openEmitter(st, dir, "trace-emitter-order", "test-book", slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatal(err)
}
e.ceilingReached(halt) // every case is a run whose money DID run out first
e.terminal(nil, tc.err)
e.flush()
raw, rerr := os.ReadFile(filepath.Join(dir, "events.jsonl"))
if rerr != nil {
t.Fatal(rerr)
}
var got string
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
var env struct {
Type string `json:"type"`
Data json.RawMessage `json:"data"`
}
if line == "" || json.Unmarshal([]byte(line), &env) != nil || env.Type != "finished" {
continue
}
var f struct {
Outcome string `json:"outcome"`
}
mustJSON(t, env.Data, &f)
got = f.Outcome
}
if got != tc.outcome {
t.Fatalf("the stream says %q for a run whose exit code says otherwise; want %q", got, tc.outcome)
}
})
}
}
// TestTheMoneyLedgerFollowsTheCEILINGAndNotTheOUTCOME pins BOTH directions of `Finished.money`'s presence
// rule, neither of which had a pin before acceptance found the contract file saying something else.
//
// ⛔ THE CONTRACT FILE SAID «present only on `outcome: ceiling`» AND BOTH VERIFIERS CAUGHT IT
// INDEPENDENTLY. It is the doc a CONSUMER IN ANOTHER ZONE reads, so the two readings it invited are two
// different bugs in somebody else's code: «money ⇒ the run stopped on a ceiling» is wrong on `stopped`
// and `failed`, and «ceiling ⇒ money is there» is wrong when the run never learned the book's cut. The
// real rule is «a ceiling was REACHED», and it is asserted here in both directions because a sentence
// nothing pins is a sentence that drifts.
func TestTheMoneyLedgerFollowsTheCeilingAndNotTheOutcome(t *testing.T) {
halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 7,
err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)}
// A cut the counters can be seeded from — one output unit, one draft stage, none of it resolved.
units := []editUnit{{Chapter: 1, FirstChunkIdx: 0, Members: []chunk.Chunk{{Chapter: 1, ChunkIdx: 0}}}}
shape := waveShape{draftNames: map[string]bool{"draft": true}, nDraft: 1}
for _, tc := range []struct {
name string
err error
ceiling bool
seedCut bool
wantOutcome string
wantMoney bool
}{
// DIRECTION 1 — money rides the ceiling, not the word. Each of these departs as something else.
{"a caught SIGTERM after the money ran out", context.Canceled, true, true, runevents.OutcomeStopped, true},
{"an infra failure on top of a caught ceiling", errors.New("provider unreachable"), true, true, runevents.OutcomeFailed, true},
{"a signing stop on a book that touched its ceiling", &WaveSignatureStop{Terms: 1, SignaturePath: "/x"}, true, true, runevents.OutcomeBankStop, true},
{"the tidy ceiling stop", halt, true, true, runevents.OutcomeCeiling, true},
// DIRECTION 2 — the word does not imply the field.
{"a ceiling stop before the run learned the book's cut", halt, true, false, runevents.OutcomeCeiling, false},
// …and no ceiling at all means no ledger, whatever else happened.
{"a plain infra failure, no ceiling anywhere", errors.New("provider unreachable"), false, true, runevents.OutcomeFailed, false},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
st, err := store.Open(filepath.Join(dir, "p.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
e, err := openEmitter(st, dir, "trace-money-rule", "test-book", slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatal(err)
}
if tc.seedCut {
e.beginWaves(units, shape, map[chunkKey][]store.ChunkStatus{})
}
if tc.ceiling {
e.ceilingReached(halt)
}
e.terminal(nil, tc.err)
e.flush()
raw, rerr := os.ReadFile(filepath.Join(dir, "events.jsonl"))
if rerr != nil {
t.Fatal(rerr)
}
var outcome string
var hasMoney bool
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
var env struct {
Type string `json:"type"`
Data json.RawMessage `json:"data"`
}
if line == "" || json.Unmarshal([]byte(line), &env) != nil || env.Type != "finished" {
continue
}
var f struct {
Outcome string `json:"outcome"`
Money json.RawMessage `json:"money"`
}
mustJSON(t, env.Data, &f)
outcome, hasMoney = f.Outcome, len(f.Money) > 0
}
if outcome != tc.wantOutcome {
t.Fatalf("outcome = %q, want %q", outcome, tc.wantOutcome)
}
if hasMoney != tc.wantMoney {
t.Fatalf("money present = %v, want %v — the field follows «a ceiling was REACHED», not the outcome word; a consumer building on either reading gets a different bug",
hasMoney, tc.wantMoney)
}
})
}
}