package pipeline import ( "context" "encoding/json" "errors" "fmt" "io" "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "testing" "time" "textmachine/backend/internal/chunk" "textmachine/backend/internal/ledger" "textmachine/backend/internal/obs" "textmachine/backend/internal/runevents" "textmachine/backend/internal/store" ) // moneystop_test.go: the acceptance battery for backlog row 277/278 — a refused reservation means «do not // start anything new», not «kill what is already running», and the run says how much was missing. // // Every test here is $0: the fake provider is an httptest server and no key of any real vendor is // touched. The pack that ordered this work did NOT sanction a paid run. // gatedProvider is a fake completion endpoint that HOLDS its first `hold` requests until the test // releases them. It exists because the defect under test is a RACE with a very short window: the old // behaviour cancelled sibling calls at the instant one reservation was refused, and a provider that // answers immediately closes that window before a test can observe it. With the calls held open, the // state «one call in flight, another worker refused» is not a timing accident but the only state the run // can be in. func gatedProvider(rec *reqRec, hold int, release <-chan struct{}) *httptest.Server { held := make(chan struct{}, hold) return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) rec.record(string(body)) select { case held <- struct{}{}: // One of the first `hold` calls: wait until the test releases it, and ONLY that. // // ⛔ IT DELIBERATELY IGNORES THE REQUEST'S OWN CONTEXT, and the first version did not — which // made two tests measure something else. With a `case <-r.Context().Done(): return` the held // request answered the moment anything cancelled it, and the transport then RETRIED and got an // instant answer from the `default` branch: a call the test believed frozen finished in 5.4 s // and the run moved on. The subject here is what the CLIENT does when its call is killed, so // the server must not quietly resurrect it. <-release default: } text, finish := draftEdit(string(body)) tb, _ := json.Marshal(text) fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":%q}], "usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`, tb, finish) })) } // waitForRefusalInFlight blocks until the ceiling has REFUSED an admission while a call is STILL in // flight — the exact window this whole pack is about — and reports whether it opened. // // ⛔ IT REPLACED A TRIGGER THAT OPENED A DIFFERENT WINDOW, and the difference cost the pack's named // acceptance artifact. The first version released the held call as soon as a worker was QUEUED — but a // worker queues only in the FIXED world, where a refusal makes it wait; in the broken world the refusal // kills the wave outright and nobody ever queues. So the planting that restores the old behaviour did // not red the test: it simply took a different, invisible path, and the assertion below was reached in // both worlds with the same answer. Keyed on the REFUSAL, the window exists in both worlds, and what // differs is what the run does with it — which is the thing under test. func waitForRefusalInFlight(r *Runner) bool { deadline := time.Now().Add(20 * time.Second) for time.Now().Before(deadline) { if inFlight, _ := r.gate.stateNow(); inFlight >= 1 && r.gate.refusalsSoFar() >= 1 { return true } time.Sleep(2 * time.Millisecond) } return false } // waitForGate blocks until the run reaches the state the test is about and REPORTS whether it did. It // polls the gate's own counters rather than sleeping for a guessed interval: what the test needs to // observe is a STATE, and a sleep would assert a duration. // // ⚠ IT RETURNS A BOOL AND DOES NOT FAIL THE TEST, because every caller runs it on a SPAWNED goroutine. // t.Fatalf outside the test goroutine calls runtime.Goexit on the wrong one: the assertion is lost, the // helper's own goroutine dies silently, and the run it was supposed to unblock hangs until the package // timeout — a green-looking suite turning into an unexplained timeout the moment a planting makes the // state unreachable, which is exactly when the harness must be trustworthy. func waitForGate(r *Runner, wantInFlight, wantWaiting int) bool { deadline := time.Now().Add(20 * time.Second) for time.Now().Before(deadline) { if inFlight, waiting := r.gate.stateNow(); inFlight >= wantInFlight && waiting >= wantWaiting { return true } time.Sleep(2 * time.Millisecond) } return false } // TestUnderACeilingTheWaveFinishesWhatItAdmitted is THE test of row 277, and its shape is the defect's // shape. Four workers, a ceiling that admits one call, and the admitted call held open at the provider // while a sibling is refused: the run must finish the held call and settle it. // // Before this pack the refusal cancelled the derived context, the held request died with // `context.Canceled`, its reservation was released, and the book departed with committed spend of ZERO — // the whole of the live 04.09 measurement, in one test. func TestUnderACeilingTheWaveFinishesWhatItAdmitted(t *testing.T) { rec := &reqRec{} release := make(chan struct{}) srv := gatedProvider(rec, 1, release) defer srv.Close() // Five single-chunk chapters over four workers: several calls are attempted at once, and the ceiling // admits one of them. bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) defer r.Close() r.CeilingUSD = oneCallCeilingUSD(t, r) reached := make(chan bool, 1) go func() { // Release the held call only once the window this test is about is OPEN: the ceiling has refused // somebody AND the admitted call is still running. Released either way — a run left holding a // request forever would turn a failed assertion into a package timeout. reached <- waitForRefusalInFlight(r) close(release) }() res, err := r.TranslateBook(ctx) if !<-reached { t.Fatal("the ceiling never refused anything while a call was in flight — this fixture does not build the situation the pack is about") } if err == nil { t.Fatal("a ceiling that admits one call must still stop the book") } var halt *CeilingHalt if !errors.As(err, &halt) { t.Fatalf("the stop must be a ceiling halt, got %v", err) } committed, reserved, serr := r.Store.SpentUSD("test-book") if serr != nil { t.Fatal(serr) } // ⛔ THE ASSERTION THE PACK EXISTS FOR. The admitted call was finished and paid for. Zero here is the // old behaviour exactly: a run that spent nothing, produced nothing and took ten seconds to do it. if committed <= 0 { t.Fatalf("the admitted call was killed by a sibling's refusal: committed=%v, provider saw %d call(s)", committed, rec.count()) } if reserved != 0 { t.Fatalf("a reservation leaked through the stop: reserved=%v", reserved) } if rec.count() == 0 { t.Fatal("nothing reached the provider at all — the fixture no longer admits a single call") } // The run departs with a RESULT, not with nothing: exit 4 and `paused` stay, and what was bought is // accounted for rather than discarded. if res == nil { t.Fatal("a ceiling stop must carry the run's partial result, or the operator is told only that it stopped") } } // TestASpendStopReportsNoVolumeLedger is the withholding of VolumeStop, asserted where it can FAIL. // // ⛔ THE ASSERTION LIVED IN THE TEST ABOVE AND WAS VACUOUS THERE: that fixture grants no volume at all, // so `res.Volume` is nil whatever the code does, and the planting that removes the withholding SURVIVED. // A volume ledger can only be wrongly attached by a run that HAS a volume grant, so that is what this // fixture builds — and the money ceiling then stops it in the middle of the grant, which is the only // state where the seven plan counters would describe units that were admitted and never done. func TestASpendStopReportsNoVolumeLedger(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() // ONE worker: the admission order is then the item order, and the arithmetic below is a statement // rather than a hope. What this test is about is which WAVE the stop lands in, not concurrency. bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(4, 1400), waveWorkers: 1, bookUSD: 100}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) defer r.Close() // ⛔ BOTH NUMBERS ARE LOAD-BEARING, AND THE SECOND ONE TOOK TWO ATTEMPTS. // // `MaxUnits` gives the run a REAL volume grant, so the seven plan counters exist at all — without one // `res.Volume` is nil whatever the code does, and the first version of this test asserted that nil. // // The ceiling must then stop the run IN THE EDIT WAVE, and that is the part the second attempt got // wrong: `res.Volume` is attached BETWEEN the waves (waverun.go, right after the draft wave), so a // ceiling tight enough to stop the DRAFT wave returns through a path that never attaches it — and the // withholding under test is a no-op there too. So the ceiling is sized to pay for every draft of the // grant and to run out on the first edit: three drafts settle at a fraction of their estimate, and the // edit call's own estimate no longer fits. r.MaxUnits = 3 r.CeilingUSD = callCeilingUSD(t, r, 1.64) res, err := r.TranslateBook(ctx) if err == nil { t.Fatal("the tight ceiling must stop the run inside its volume grant") } var halt *CeilingHalt if !errors.As(err, &halt) { t.Fatalf("want a ceiling halt, got %v", err) } if res == nil { t.Fatal("a ceiling stop must carry the run's partial result") } if res.Volume != nil { t.Fatalf("a SPEND stop reported a VOLUME ledger %+v — those seven counters are a PLAN, and a run stopped by money leaves them counting units that were admitted and never done: Delivered over-counts, LeftFresh under-counts, and at Left()==0 the run would announce it reached the end of the book", *res.Volume) } } // oneCallCeilingUSD is a ceiling that admits exactly one of this book's calls: halfway between one // reservation and two. It is DERIVED from the engine's own arithmetic rather than written as a literal, // so a change to sizing or pricing moves the fixture with the code instead of silently making the test // about a different situation. func oneCallCeilingUSD(t *testing.T, r *Runner) float64 { return callCeilingUSD(t, r, 1.5) } // callCeilingUSD is `mult` times ONE draft call's reservation for this fixture's chunk size. Derived from // the engine's own arithmetic rather than written as a literal, so a change to sizing or pricing moves // the fixture with the code instead of quietly making the test about a different situation. The sizing // length is the fixture's own chapter length (cjkSource(_, chapterRunes)). func callCeilingUSD(t *testing.T, r *Runner, mult float64) float64 { t.Helper() plan := r.pricePlanFor() if plan == nil || len(plan.stages) == 0 { t.Fatal("the fixture pipeline resolves no priceable stage") } sp := plan.stages[0] base := r.baseMaxTokensFor(sp.st, fixtureChapterRunes) // ⚠ NO INJECTION ALLOWANCE HERE, and the omission is the fixture's truth rather than a shortcut. These // projects carry no glossary, so the bank injection is not rendered at all and the executor reserves // without it. Charging the plan's 800-token allowance inflated every ceiling derived here by ~11 %, and // that quietly turned a «one reservation at a time» fixture into one where two fit — which is how the // delivering-ceiling test below came to pass without the waiting path ever running. one := ledger.EstimateUSD(sp.price, sp.tplTokens+fixtureChapterRunes, base, sp.reasoning) if one <= 0 { t.Fatal("the fixture's first stage is free — a ceiling test needs a priced model") } return one * mult } // fixtureChapterRunes is the chapter length every fixture in this file uses. Named once so the ceilings // derived from it cannot drift away from the books they are derived for. const fixtureChapterRunes = 1400 // TestARefusalWithNothingInFlightIsFinal pins the other half of the rule: waiting is what makes a refusal // non-final, and there is nothing to wait for when no call holds a reservation. A single-worker run whose // ceiling admits nothing must stop at once rather than hang. func TestARefusalWithNothingInFlightIsFinal(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{bookUSD: 0.0000001}) r := newRunner(t, bookPath) defer r.Close() done := make(chan error, 1) go func() { _, err := r.TranslateBook(context.Background()); done <- err }() select { case err := <-done: if err == nil { t.Fatal("a ceiling that admits nothing must stop the run") } var halt *CeilingHalt if !errors.As(err, &halt) { t.Fatalf("want a ceiling halt, got %v", err) } case <-time.After(30 * time.Second): t.Fatal("the run HUNG on a refusal with nothing in flight — waiting for a settle that can never come") } if rec.count() != 0 { t.Fatalf("a denied reserve must not reach the provider, calls=%d", rec.count()) } } // TestTheGateWaitsWhileTheSkyIsFullAndStopsWhenItEmpties exercises the gate directly, because the two // answers it gives are what the whole behaviour rests on and they must be pinned without a provider, a // store or a clock. func TestTheGateWaitsWhileTheSkyIsFullAndStopsWhenItEmpties(t *testing.T) { var g reserveGate if got := g.waitForSettle(context.Background(), true); got != waitNothingInFlight { t.Fatalf("with nothing in flight the refusal is final, got outcome %d", got) } // Simulate an admitted call by hand — admit() needs a store, and what is under test here is the // counter's contract, not the reservation's. g.mu.Lock() g.inFlight = 1 g.mu.Unlock() woke := make(chan waitOutcome, 1) go func() { woke <- g.waitForSettle(context.Background(), true) }() // The waiter must be parked before the settle, or the test proves nothing about the broadcast. deadline := time.Now().Add(5 * time.Second) for { if _, waiting := g.stateNow(); waiting == 1 { break } if time.Now().After(deadline) { t.Fatal("the waiter never parked") } time.Sleep(time.Millisecond) } g.done() select { case got := <-woke: if got != waitSettled { t.Fatalf("a settle must wake the waiter so it can re-ask for the headroom that just grew, got outcome %d", got) } case <-time.After(5 * time.Second): t.Fatal("the settle did not wake the waiter — the broadcast channel was replaced without being closed") } if inFlight, waiting := g.stateNow(); inFlight != 0 || waiting != 0 { t.Fatalf("after the settle the gate must be empty, got in_flight=%d waiting=%d", inFlight, waiting) } // And a context that dies while waiting must release the waiter rather than strand it. g.mu.Lock() g.inFlight = 1 g.mu.Unlock() ctx, cancel := context.WithCancel(context.Background()) cancel() // ⛔ AND IT MUST BE ITS OWN ANSWER. Reported as «nothing in flight», a cancelled wait becomes a // CeilingHalt one level up, and a run somebody stopped with Ctrl-C publishes a `ceiling` event with // a shortfall — telling the platform to add money for a stop that had nothing to do with money. if got := g.waitForSettle(ctx, true); got != waitAborted { t.Fatalf("a dead context must be distinguishable from an empty sky, got outcome %d", got) } } // TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent pins the futility check. Waiting is bounded // by the longest provider timeout, so waiting for a settle that provably cannot admit the call costs // minutes to arrive at the answer already in hand. func TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent(t *testing.T) { // committed 0.9, estimate 0.2, ceiling 1.0 — every reservation could be released for free and 1.1 // would still be past 1.0. if !settleCannotHelp(runevents.ScopeBook, 0.9, 0.2, 1.0) { t.Error("committed alone already leaves no room: no settle can change that") } // committed 0.5, estimate 0.2 — the refusal came from RESERVED money, which a settle gives back. if settleCannotHelp(runevents.ScopeBook, 0.5, 0.2, 1.0) { t.Error("the refusal is about reserved money, and reserved money is exactly what a settle returns") } // The DAY ceiling sums other books, so this process's committed figure is not the day's: it must not // be used to declare a refusal final. if settleCannotHelp(runevents.ScopeDay, 0.9, 0.2, 1.0) { t.Error("the day ceiling is not this book's arithmetic, so this book's committed spend cannot rule the wait out") } // No ceiling means no refusal to reason about. if settleCannotHelp(runevents.ScopeBook, 9, 9, 0) { t.Error("a zero ceiling is «no limit» and cannot refuse anything") } } // TestTheShortfallIsTheDistanceToTheCeilingAndNotThePriceOfACall pins WHICH number leaves the engine. // The denied estimate is the price of one call and the contract keeps our cost structure inside; the // shortfall is the distance between a limit the platform set and a total it has already authorised. func TestTheShortfallIsTheDistanceToTheCeilingAndNotThePriceOfACall(t *testing.T) { // committed 0.30 + reserved 0.20 + estimate 0.07 = 0.57 against a ceiling of 0.55 → 0.02 missing. got := shortfallMicroUSD(runevents.ScopeBook, 0.30, 0.20, 0.07, 0.55) if got != 20_000 { t.Errorf("shortfall = %d micro-USD, want 20000 (the amount the admission overshot by)", got) } // It is NOT the estimate: publishing that would publish the price of a call. if got == 70_000 { t.Error("the engine published the DENIED ESTIMATE — that is the price of one call and it may not leave") } // Rounded UP: a figure a person tops up against must never be short. if up := shortfallMicroUSD(runevents.ScopeBook, 0, 0, 0.0000005, 0); up != 0 { t.Errorf("a zero ceiling states nothing, got %d", up) } if up := shortfallMicroUSD(runevents.ScopeBook, 1, 0, 0.0000004, 1); up != 1 { t.Errorf("a sub-micro-dollar shortfall must round UP to 1, got %d", up) } // The day scope cannot state it honestly and therefore does not. if d := shortfallMicroUSD(runevents.ScopeDay, 0.9, 0, 0.2, 1.0); d != 0 { t.Errorf("the day scope must state no shortfall (its figures are one book's, its ceiling is every book's), got %d", d) } } // TestTheStoppedRunTellsAConsumerHowMuchWasMissing is the SEAM axis, read from the consumer's side: the // events file is parsed as bytes, exactly as the platform tails it, rather than by asking the engine's // own structs what they hold. func TestTheStoppedRunTellsAConsumerHowMuchWasMissing(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(3, 1400), bookUSD: 100}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) r.CeilingUSD = callCeilingUSD(t, r, 1.05) // one reservation fits; the book needs six calls if _, err := r.TranslateBook(ctx); err == nil { t.Fatal("the tight ceiling must stop the run") } journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl") r.Close() raw, err := os.ReadFile(journal) if err != nil { t.Fatalf("no event journal to read from the consumer side: %v", err) } var sawCeiling, sawFinished bool var version string for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { if line == "" { continue } var env struct { Type string `json:"type"` Data json.RawMessage `json:"data"` } if err := json.Unmarshal([]byte(line), &env); err != nil { t.Fatalf("the stream is not readable NDJSON: %v (%s)", err, line) } switch env.Type { case "hello": var h struct { StreamVersion string `json:"stream_version"` } mustJSON(t, env.Data, &h) version = h.StreamVersion case "ceiling": var c struct { Halted bool `json:"halted"` Scope string `json:"scope"` ShortfallMicroUSD int64 `json:"shortfall_micro_usd"` } mustJSON(t, env.Data, &c) sawCeiling = true if !c.Halted || c.Scope != "book" { t.Errorf("the ceiling frame lost its existing fields: %+v", c) } // ⛔ THE FIELD THIS PACK ADDS. Without it the number the engine already printed dies in the // stderr of a transient unit and the buyer is told the run stopped and nothing else. if c.ShortfallMicroUSD <= 0 { t.Errorf("the ceiling frame does not say how much was missing: %+v", c) } case "finished": var f struct { Outcome string `json:"outcome"` Money *struct { UnitsResolved int `json:"units_resolved"` UnitsDeferred int `json:"units_deferred"` } `json:"money"` } mustJSON(t, env.Data, &f) sawFinished = true if f.Outcome != "ceiling" { t.Errorf("outcome = %q, want ceiling — a resumable stop is never `failed` (PD-113)", f.Outcome) } if f.Money == nil { t.Error("a run stopped by money must say what it bought and what is still owed") } else if f.Money.UnitsDeferred == 0 { t.Errorf("this fixture cannot afford the whole book, so something must be deferred: %+v", f.Money) } } } if !sawCeiling || !sawFinished { t.Fatalf("the stream is incomplete: ceiling=%v finished=%v", sawCeiling, sawFinished) } // The seam rule: a frame gained a field, so the version says so. A reader comparing only the major is // unaffected — and a stream whose bytes changed while its version did not is the one thing a version // exists to prevent (runevents.go records this exact miss, made once already). if version != runevents.StreamVersion { t.Errorf("hello says stream_version=%q, build says %q", version, runevents.StreamVersion) } if version == "1.2" { t.Error("the stream gained a field and kept its old version — the miss the constant's own comment records") } } func mustJSON(t *testing.T, raw []byte, into any) { t.Helper() if err := json.Unmarshal(raw, into); err != nil { t.Fatalf("frame payload unreadable: %v (%s)", err, raw) } } // TestResumeAfterAMoneyStopRePaysNothing is the DETERMINISM axis. A run stopped by money, resumed under a // raised ceiling, must finish the book without buying again what it already bought — the admitted calls // were settled and checkpointed, so they replay for free. func TestResumeAfterAMoneyStopRePaysNothing(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(3, 1400), waveWorkers: 2, bookUSD: 100}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r1 := newRunner(t, bookPath) r1.CeilingUSD = callCeilingUSD(t, r1, 1.3) // room for a call or two, not for the book if _, err := r1.TranslateBook(ctx); err == nil { t.Fatal("the tight ceiling must stop the run") } firstPass := rec.count() committedAfterStop, reservedAfterStop, err := r1.Store.SpentUSD("test-book") if err != nil { t.Fatal(err) } r1.Close() if firstPass == 0 { t.Fatal("fixture drifted: the stopped run must have paid for something") } if reservedAfterStop != 0 { t.Fatalf("a stopped run must leave no reservation behind, got %v", reservedAfterStop) } r2 := newRunner(t, bookPath) defer r2.Close() if _, err := r2.TranslateBook(ctx); err != nil { t.Fatalf("the resumed run must finish the book under the book's own ceiling: %v", err) } committedAfterFinish, _, err := r2.Store.SpentUSD("test-book") if err != nil { t.Fatal(err) } // ⛔ THE ASSERTION IS THE TOTAL, and its first version was two statements that cannot fail. «Committed // spend went down» is impossible — committed only grows — and `secondPass >= firstPass + rec.count()` // is arithmetically unreachable, since `secondPass` is defined as `rec.count() - firstPass`. Both read // like checks and neither could ever red. What actually catches a re-purchase is the COUNT of provider // calls across both runs: this book is 3 units × 2 stages, so a resume that re-bought its first pass // would show more than six. const callsInTheWholeBook = 3 * 2 secondPass := rec.count() - firstPass if secondPass <= 0 { t.Fatal("the resume did no work at all — the fixture is not testing a resume") } if total := rec.count(); total != callsInTheWholeBook { t.Fatalf("the book cost %d provider calls across the stop and the resume, want exactly %d — anything more is work bought twice (first pass %d, resume %d)", total, callsInTheWholeBook, firstPass, secondPass) } if committedAfterFinish <= committedAfterStop { t.Fatalf("the resume settled nothing: committed %v → %v", committedAfterStop, committedAfterFinish) } } // TestAMoneyStopInTheDraftWaveNeverStartsTheEditWave pins the rule that a partial draft is not editable. // // ⛔ THE FAILURE IT PREVENTS IS PAID. The edit wave looks a member's draft up in a map built over EVERY // chunk, so an unstarted member is FOUND — holding the zero value: not flagged, empty text. It would be // joined into the unit's draft as a clean member, and the editor would be paid to edit a unit with a hole // in it and then ship it. func TestAMoneyStopInTheDraftWaveNeverStartsTheEditWave(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(4, 1400), waveWorkers: 2, bookUSD: 100}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) defer r.Close() r.CeilingUSD = callCeilingUSD(t, r, 1.05) // one draft call fits; the book needs eight if _, err := r.TranslateBook(ctx); err == nil { t.Fatal("the tight ceiling must stop the run in the draft wave") } for _, body := range rec.all() { if isEditBody(body) { t.Fatal("an EDIT call was issued after the draft wave was stopped by money — the editor was paid to edit a unit whose members were never drafted") } } } // TestNothingLeaksWhenTheGateSeesAnAdmissionError pins the counter's own failure mode: a reservation the // store refused with an ERROR (not a denial) must not be counted as in flight, or every later refusal // waits for a settle that can never arrive and the run hangs instead of stopping. func TestNothingLeaksWhenTheGateSeesAnAdmissionError(t *testing.T) { var g reserveGate // A closed store makes Reserve fail as an error rather than deny. dir := t.TempDir() st, err := store.Open(filepath.Join(dir, "p.db")) if err != nil { t.Fatal(err) } st.Close() if _, _, err := g.admit(st, "b", 0.01, store.Ceilings{BookUSD: 100}); err == nil { t.Skip("this store does not error on a closed handle; the counter's contract is pinned by the gate test above") } if inFlight, _ := g.stateNow(); inFlight != 0 { t.Fatalf("a failed admission must not count as in flight, got %d", inFlight) } } // TestWaitingForHeadroomDELIVERSTheWholeBook is the positive half of row 277 and the one the rest of the // battery does not reach. Every other ceiling test ends in a stop; this one ends in a FINISHED BOOK that // a ceiling would have refused without the wait. // // The shape is the real one. A reservation is worst-case (whole prompt uncached, completion running to // max_tokens) while a settle is the actual cost, so a ceiling can be too small to hold two reservations // at once and perfectly able to pay for the whole book one call at a time. Before the wait existed, the // second parallel worker's refusal killed the run; with it, the wave goes serial under the ceiling and // the book is delivered. func TestWaitingForHeadroomDeliversTheWholeBook(t *testing.T) { rec := &reqRec{} // ⚠ THE FIRST CALL IS HELD OPEN, and that is what makes the collision a STATE rather than a race the // test hopes for. Without it the two draft workers may never hold a reservation at the same moment — // under `-race` they reliably do not: the battery caught this test green-by-luck and then failing, // with the log showing chunk 0 settled before chunk 1 was ever admitted. A ceiling cannot refuse what // is not asked concurrently, so the waiting path went unexercised and the assertion below said so. release := make(chan struct{}) srv := gatedProvider(rec, 1, release) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(2, 1400), waveWorkers: 4, bookUSD: 100}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) defer r.Close() // Room for ONE reservation at a time, and — because a settle costs a fraction of its estimate — room // in TOTAL for every call this two-chapter book makes. Both halves are needed and neither is // incidental: the first is what makes the parallel wave collide with the ceiling at all, the second is // what makes the collision survivable. The ceiling stays cumulative throughout, which is why the book // is two chapters and not ten. // The window is narrow and both edges are real: BELOW two reservations (so the parallel wave collides // with the ceiling and a worker has to wait) and ABOVE the whole book's settled cost (so the waiting // wave finishes it). A settle costs a fraction of its estimate, which is what makes such a window // exist at all — and it is why the book is two chapters and not ten, the ceiling being cumulative. r.CeilingUSD = callCeilingUSD(t, r, 1.85) reached := make(chan bool, 1) go func() { // Release the held call only once a worker is genuinely queued behind the ceiling — the state the // whole test is about. Released either way, so a missed state fails an assertion instead of // hanging the package. reached <- waitForGate(r, 1, 1) close(release) }() res, err := r.TranslateBook(ctx) if !<-reached { t.Fatal("no worker was ever queued behind the ceiling — the two draft reservations never overlapped, so this fixture does not exercise the waiting path") } if err != nil { t.Fatalf("a ceiling that admits one call AT A TIME must still deliver the book: %v", err) } // ⛔ THE MECHANISM, ASSERTED — not inferred from the outcome. This test's first version checked only // that the book was delivered, and it PASSED on a fixture whose ceiling never refused anything: the // waiting path did not run, the mutant that deletes the wait would have survived, and the assertion // read as proof of a road the run never took. A delivered book is not evidence of waiting; a wait is. if waits := r.gate.waitsSoFar(); waits == 0 { t.Fatalf("the book was delivered WITHOUT the ceiling ever refusing a reservation — this fixture does not exercise the waiting path at all, so it proves nothing about it (shipped %d units)", len(res.Chunks)) } if len(res.Chunks) != 2 { t.Fatalf("want 2 shipped units, got %d — the wave stopped instead of queueing", len(res.Chunks)) } for _, oc := range res.Chunks { if oc.Disposition != DispOK { t.Fatalf("a unit came back %s under the queueing ceiling", oc.Disposition) } } committed, reserved, serr := r.Store.SpentUSD("test-book") if serr != nil { t.Fatal(serr) } if reserved != 0 { t.Fatalf("no reservation may survive a completed run, reserved=%v", reserved) } // ⛔ AND THE CEILING STILL HELD. The whole point of keeping the reservation is that queueing never // becomes overspending: committed spend is under the limit that was in force, every call of it. if committed > r.CeilingUSD { t.Fatalf("the queueing wave OVERSPENT its ceiling: committed=%v ceiling=%v", committed, r.CeilingUSD) } } // TestACancelledWaitIsNotAMoneyStop pins the defect that a boolean answer created and a three-valued one // closes. A worker waiting for headroom when the run is cancelled must surface the CANCELLATION. // // ⛔ WHAT IT PREVENTS. Reported as «nothing in flight», a cancelled wait became a CeilingHalt one level // up: the run published a `ceiling` event carrying a shortfall, and the platform — which lets that event // survive any exit code — recorded `paused` and told a person to add money for a run they had stopped // themselves. The same path was reachable from a sibling's infra failure, which cancels. func TestACancelledWaitIsNotAMoneyStop(t *testing.T) { rec := &reqRec{} release := make(chan struct{}) srv := gatedProvider(rec, 1, release) defer srv.Close() // ⚠ DECLARED AFTER srv's defer SO IT RUNS BEFORE IT — defers are LIFO, and httptest.Server.Close() // WAITS for outstanding requests. Ordered the other way this deadlocks the package: the handler is // parked on `release`, Close() is parked on the handler, and the close that would free both is queued // behind Close(). Written the wrong way round it HUNG the package rather than failing it — the // timeout-shaped failure this pack has already recorded as a class of its own, met once more. defer close(release) bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100}) ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})) r := newRunner(t, bookPath) r.CeilingUSD = oneCallCeilingUSD(t, r) reached := make(chan bool, 1) go func() { // Cancel only once a worker is genuinely parked on the gate — otherwise the test would be about // cancelling a run, not about cancelling a WAIT. Cancelled either way, so a missed state fails an // assertion instead of hanging the package. reached <- waitForGate(r, 1, 1) cancel() }() _, err := r.TranslateBook(ctx) if !<-reached { t.Fatal("no worker was ever parked on the gate — the cancellation did not interrupt a WAIT and this test is about nothing") } journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl") r.Close() if err == nil { t.Fatal("a cancelled run must not report success") } var halt *CeilingHalt if errors.As(err, &halt) { t.Fatalf("a run cancelled while waiting for headroom reported a MONEY stop: %v", err) } if !errors.Is(err, context.Canceled) { t.Fatalf("the cancellation must be what leaves the run, got %v", err) } raw, rerr := os.ReadFile(journal) if rerr != nil { t.Fatalf("no event journal: %v", rerr) } if strings.Contains(string(raw), `"ceiling"`) { t.Fatalf("the stream published a ceiling stop for a run somebody cancelled:\n%s", raw) } } // TestAnInfraFailureAfterAMoneyStopDoesNotDepartAsPaused is the two-slot error ranking, asserted. // // ⛔ WHY IT NEEDS ITS OWN TEST. A spend ceiling no longer cancels its siblings, so for the first time a // run can latch on money and THEN break for an unrelated reason. Sharing one error slot, whichever // happened first would speak for both — and a first-wins ceiling would have a broken run depart exit 4, // which the platform records as `paused`: «add money» offered as the remedy for a machine that failed. // So the failure ranks on the way out, and the money fact travels on the channel it was given for // exactly this case: the `ceiling` event, written at the moment of the refusal. func TestAnInfraFailureAfterAMoneyStopDoesNotDepartAsPaused(t *testing.T) { r := &Runner{} halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 1234, err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)} infra := errors.New("provider unreachable") // ⚠ BOTH WORKERS MUST BE HOLDING AN ITEM BEFORE EITHER FAILS, and the first attempt at this test got // it wrong in a way worth keeping: with the ceiling returned first, the latch closed and the feeder // never handed out the second item at all — so the failure never happened and the test asserted // nothing. That is the latch behaving exactly as designed (nothing new starts), and it is also why // this co-occurrence is only reachable among calls that were ALREADY admitted. holding := make(chan struct{}) err := r.runWave(context.Background(), 2, 2, func(ctx context.Context, i int) error { if i == 1 { close(holding) return infra } <-holding return halt }) if err == nil { t.Fatal("the wave must report something") } var got *CeilingHalt if errors.As(err, &got) { t.Fatalf("a run that BROKE departed as a money stop (exit 4 → the platform records `paused`): %v", err) } if !errors.Is(err, infra) { t.Fatalf("the failure must be what leaves the wave, got %v", err) } } // TestAnOptionalCallIsTurnedAwayWhileMandatoryOneWaits is the `mandatory` rule, ASSERTED. // // ⛔ ITS ENTIRE BEHAVIOUR USED TO BE AN ARGUMENT IN A COMMENT. Every fixture that refused an optional call // was sequential, so `inFlight == 0` produced an immediate halt whichever way the flag was set — flipping // all three optional call sites to `true` left the battery green. The flag decides something ONLY when a // sibling is in flight, so that is the state this test builds, at the level where the rule now lives. // // What it protects: an escalation hop that queued for headroom would spend the money the primary wave // needs to finish work it has already started, and it holds escMu while it waits, so it would block every // other worker's escalation too. The repair sub-step runs INSIDE an edit-wave worker and is the same // story. All three degrade on a ceiling; none of them may wait for one. func TestAnOptionalCallIsTurnedAwayWhileMandatoryOneWaits(t *testing.T) { var g reserveGate g.mu.Lock() g.inFlight = 1 // money IS still moving: this is the only state in which the rule decides anything g.mu.Unlock() // The optional caller is turned away AT ONCE — not because there is nothing to wait for, but because // it has no claim on it. The two are opposite facts and must not share an answer. // // ⚠ THE CONTEXT IS BOUNDED, AND THAT IS WHAT GIVES THE FAILURE A SHAPE. With context.Background() a // broken rule does not fail here, it HANGS: the optional caller parks on a settle that never comes in // a unit test, the package times out, and the catalogue reports `NOTHING` — «the planting did not // build, or the run died» — which is indistinguishable from machine load. That is the timeout-shaped // failure this pack has already recorded as a class, met an eighth time and caught immediately: with // a deadline the same break returns waitAborted in two seconds and reds an assertion. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if got := g.waitForSettle(ctx, false); got != waitNotAllowed { t.Fatalf("an optional call refused while money is still moving must be turned away, got outcome %d — an optional call that queues for the last dollars spends what the mandatory work needed, and the escalation hop blocks every other worker's escalation while it waits", got) } if n := g.optionalTurnedAwaySoFar(); n != 1 { t.Fatalf("the turn-away was not recorded (%d) — without the fact there is nothing a test can see", n) } if w := g.waitsSoFar(); w != 0 { t.Fatalf("an optional call WAITED (%d): «do not start anything new» became «the optional spends what the mandatory needed»", w) } // …and the MANDATORY caller in the very same state does wait, and is woken by the settle. woke := make(chan waitOutcome, 1) go func() { woke <- g.waitForSettle(context.Background(), true) }() deadline := time.Now().Add(5 * time.Second) for { if _, waiting := g.stateNow(); waiting == 1 { break } if time.Now().After(deadline) { t.Fatal("the mandatory caller never parked — the rule turns everyone away, which stops the book it was written to keep alive") } time.Sleep(time.Millisecond) } g.done() select { case got := <-woke: if got != waitSettled { t.Fatalf("the mandatory caller must be woken by the settle, got outcome %d", got) } case <-time.After(5 * time.Second): t.Fatal("the settle did not wake the mandatory caller") } if w := g.waitsSoFar(); w != 1 { t.Fatalf("waits=%d, want exactly the mandatory caller's one", w) } if n := g.optionalTurnedAwaySoFar(); n != 1 { t.Fatalf("the mandatory caller was counted as an optional turn-away (%d)", n) } } // TestOnlyOneCeilingFrameReachesTheStreamHoweverManyWorkersAreRefused pins the idempotence the emitter // claims. Under one worker a run is refused once and the claim is untestable; under four it is refused // several times, and a reader that received a `ceiling` frame per refusal would count one stop as many. func TestOnlyOneCeilingFrameReachesTheStreamHoweverManyWorkersAreRefused(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) r.CeilingUSD = callCeilingUSD(t, r, 1.05) if _, err := r.TranslateBook(ctx); err == nil { t.Fatal("the tight ceiling must stop the book") } journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl") r.Close() raw, err := os.ReadFile(journal) if err != nil { t.Fatal(err) } frames := 0 for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { var env struct { Type string `json:"type"` } if line == "" || json.Unmarshal([]byte(line), &env) != nil { continue } if env.Type == "ceiling" { frames++ } } if frames != 1 { t.Fatalf("the stream carries %d ceiling frames for ONE stop — a reader counting stops would count refusals instead", frames) } } // ⛔ THERE IS NO TEST HERE FOR settleCannotHelp's WIRING, and the absence is deliberate and argued. // // The predicate has its own test (TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent). What no // test in this file exercises is the `if` that consults it — and the reason is that the state it fires in // is nearly unreachable in a wave of similar calls. It needs a refusal that no settle could lift // (`committed + estimate > ceiling`) WHILE another call is still in flight. But the in-flight call was // itself admitted at the same committed level with a comparable estimate, so if there was room for it // there is room for this one: the two conditions contradict each other unless the estimates differ // materially — heterogeneous chunk sizes, or a draft and an edit meeting in one wave, which they never do. // And once the sky empties, `waitNothingInFlight` stops the book anyway, with or without the predicate. // // ⚠ A TEST THAT CLAIMED TO COVER IT STOOD HERE AND WAS DELETED, because it passed for a reason that had // nothing to do with its name. It held a call at the fake provider and asserted the run ended while the // call was still held — but the held request hit the attempt timeout, the transport RETRIED, and the // retry answered instantly (the trace shows `latency_ms=5389` on a call the test believed was frozen). // So the run ended because the call finished, the three refused workers had WAITED and been woken, and // the assertion was about a situation the fixture never built. Removing it is the honest move: the pack's // own rule is that «checked» without an artifact does not count, and an artifact that checks something // else is worse than none, because it reads as coverage. What remains is this paragraph and the entry in // the report's obstacle section. // TestTheTerminalVocabularyNeverContradictsTheExitCode pins the ORDER of terminal()'s branches — a // regression made and caught inside this very pack, and until now held by nothing but a comment. // // ⛔ WHAT WENT WRONG. The money ledger is attached whenever a ceiling was reached, and the branch that // does it was written ABOVE the signature stop and the cancellation. It swallowed both: a bank-stop on a // book that had touched its ceiling reported `failed` while its exit code said 3, and a caught SIGTERM // reported `failed` while its exit code said 5. Two channels contradicting each other about one run is // exactly what this vocabulary exists to prevent — and re-ordering `case` arms is precisely the edit a // tidy-up makes without noticing. func TestTheTerminalVocabularyNeverContradictsTheExitCode(t *testing.T) { halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 42, err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)} for _, tc := range []struct { name string err error outcome string }{ // Each of these has its OWN exit code (3, 5, 4, 1), and the stream must say the same thing. {"a signature stop on a book that touched its ceiling", &WaveSignatureStop{Terms: 1, SignaturePath: "/x"}, runevents.OutcomeBankStop}, {"a caught SIGTERM after the money ran out", context.Canceled, runevents.OutcomeStopped}, {"the ceiling stop itself", halt, runevents.OutcomeCeiling}, {"an infra failure on top of a caught ceiling", errors.New("provider unreachable"), runevents.OutcomeFailed}, } { t.Run(tc.name, func(t *testing.T) { dir := t.TempDir() st, err := store.Open(filepath.Join(dir, "p.db")) if err != nil { t.Fatal(err) } defer st.Close() e, err := openEmitter(st, dir, "trace-emitter-order", "test-book", slog.New(slog.NewTextHandler(io.Discard, nil))) if err != nil { t.Fatal(err) } e.ceilingReached(halt) // every case is a run whose money DID run out first e.terminal(nil, tc.err) e.flush() raw, rerr := os.ReadFile(filepath.Join(dir, "events.jsonl")) if rerr != nil { t.Fatal(rerr) } var got string for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { var env struct { Type string `json:"type"` Data json.RawMessage `json:"data"` } if line == "" || json.Unmarshal([]byte(line), &env) != nil || env.Type != "finished" { continue } var f struct { Outcome string `json:"outcome"` } mustJSON(t, env.Data, &f) got = f.Outcome } if got != tc.outcome { t.Fatalf("the stream says %q for a run whose exit code says otherwise; want %q", got, tc.outcome) } }) } } // TestTheMoneyLedgerFollowsTheCEILINGAndNotTheOUTCOME pins BOTH directions of `Finished.money`'s presence // rule, neither of which had a pin before acceptance found the contract file saying something else. // // ⛔ THE CONTRACT FILE SAID «present only on `outcome: ceiling`» AND BOTH VERIFIERS CAUGHT IT // INDEPENDENTLY. It is the doc a CONSUMER IN ANOTHER ZONE reads, so the two readings it invited are two // different bugs in somebody else's code: «money ⇒ the run stopped on a ceiling» is wrong on `stopped` // and `failed`, and «ceiling ⇒ money is there» is wrong when the run never learned the book's cut. The // real rule is «a ceiling was REACHED», and it is asserted here in both directions because a sentence // nothing pins is a sentence that drifts. func TestTheMoneyLedgerFollowsTheCeilingAndNotTheOutcome(t *testing.T) { halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 7, err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)} // A cut the counters can be seeded from — one output unit, one draft stage, none of it resolved. units := []editUnit{{Chapter: 1, FirstChunkIdx: 0, Members: []chunk.Chunk{{Chapter: 1, ChunkIdx: 0}}}} shape := waveShape{draftNames: map[string]bool{"draft": true}, nDraft: 1} for _, tc := range []struct { name string err error ceiling bool seedCut bool wantOutcome string wantMoney bool }{ // DIRECTION 1 — money rides the ceiling, not the word. Each of these departs as something else. {"a caught SIGTERM after the money ran out", context.Canceled, true, true, runevents.OutcomeStopped, true}, {"an infra failure on top of a caught ceiling", errors.New("provider unreachable"), true, true, runevents.OutcomeFailed, true}, {"a signing stop on a book that touched its ceiling", &WaveSignatureStop{Terms: 1, SignaturePath: "/x"}, true, true, runevents.OutcomeBankStop, true}, {"the tidy ceiling stop", halt, true, true, runevents.OutcomeCeiling, true}, // DIRECTION 2 — the word does not imply the field. {"a ceiling stop before the run learned the book's cut", halt, true, false, runevents.OutcomeCeiling, false}, // …and no ceiling at all means no ledger, whatever else happened. {"a plain infra failure, no ceiling anywhere", errors.New("provider unreachable"), false, true, runevents.OutcomeFailed, false}, } { t.Run(tc.name, func(t *testing.T) { dir := t.TempDir() st, err := store.Open(filepath.Join(dir, "p.db")) if err != nil { t.Fatal(err) } defer st.Close() e, err := openEmitter(st, dir, "trace-money-rule", "test-book", slog.New(slog.NewTextHandler(io.Discard, nil))) if err != nil { t.Fatal(err) } if tc.seedCut { e.beginWaves(units, shape, map[chunkKey][]store.ChunkStatus{}) } if tc.ceiling { e.ceilingReached(halt) } e.terminal(nil, tc.err) e.flush() raw, rerr := os.ReadFile(filepath.Join(dir, "events.jsonl")) if rerr != nil { t.Fatal(rerr) } var outcome string var hasMoney bool for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { var env struct { Type string `json:"type"` Data json.RawMessage `json:"data"` } if line == "" || json.Unmarshal([]byte(line), &env) != nil || env.Type != "finished" { continue } var f struct { Outcome string `json:"outcome"` Money json.RawMessage `json:"money"` } mustJSON(t, env.Data, &f) outcome, hasMoney = f.Outcome, len(f.Money) > 0 } if outcome != tc.wantOutcome { t.Fatalf("outcome = %q, want %q", outcome, tc.wantOutcome) } if hasMoney != tc.wantMoney { t.Fatalf("money present = %v, want %v — the field follows «a ceiling was REACHED», not the outcome word; a consumer building on either reading gets a different bug", hasMoney, tc.wantMoney) } }) } }