1042 lines
49 KiB
Go
1042 lines
49 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"log/slog"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/ledger"
|
||
"textmachine/backend/internal/obs"
|
||
"textmachine/backend/internal/runevents"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// moneystop_test.go: the acceptance battery for backlog row 277/278 — a refused reservation means «do not
|
||
// start anything new», not «kill what is already running», and the run says how much was missing.
|
||
//
|
||
// Every test here is $0: the fake provider is an httptest server and no key of any real vendor is
|
||
// touched. The pack that ordered this work did NOT sanction a paid run.
|
||
|
||
// gatedProvider is a fake completion endpoint that HOLDS its first `hold` requests until the test
|
||
// releases them. It exists because the defect under test is a RACE with a very short window: the old
|
||
// behaviour cancelled sibling calls at the instant one reservation was refused, and a provider that
|
||
// answers immediately closes that window before a test can observe it. With the calls held open, the
|
||
// state «one call in flight, another worker refused» is not a timing accident but the only state the run
|
||
// can be in.
|
||
func gatedProvider(rec *reqRec, hold int, release <-chan struct{}) *httptest.Server {
|
||
held := make(chan struct{}, hold)
|
||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
rec.record(string(body))
|
||
select {
|
||
case held <- struct{}{}:
|
||
// One of the first `hold` calls: wait until the test releases it, and ONLY that.
|
||
//
|
||
// ⛔ IT DELIBERATELY IGNORES THE REQUEST'S OWN CONTEXT, and the first version did not — which
|
||
// made two tests measure something else. With a `case <-r.Context().Done(): return` the held
|
||
// request answered the moment anything cancelled it, and the transport then RETRIED and got an
|
||
// instant answer from the `default` branch: a call the test believed frozen finished in 5.4 s
|
||
// and the run moved on. The subject here is what the CLIENT does when its call is killed, so
|
||
// the server must not quietly resurrect it.
|
||
<-release
|
||
default:
|
||
}
|
||
text, finish := draftEdit(string(body))
|
||
tb, _ := json.Marshal(text)
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":%q}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`,
|
||
tb, finish)
|
||
}))
|
||
}
|
||
|
||
// waitForRefusalInFlight blocks until the ceiling has REFUSED an admission while a call is STILL in
|
||
// flight — the exact window this whole pack is about — and reports whether it opened.
|
||
//
|
||
// ⛔ IT REPLACED A TRIGGER THAT OPENED A DIFFERENT WINDOW, and the difference cost the pack's named
|
||
// acceptance artifact. The first version released the held call as soon as a worker was QUEUED — but a
|
||
// worker queues only in the FIXED world, where a refusal makes it wait; in the broken world the refusal
|
||
// kills the wave outright and nobody ever queues. So the planting that restores the old behaviour did
|
||
// not red the test: it simply took a different, invisible path, and the assertion below was reached in
|
||
// both worlds with the same answer. Keyed on the REFUSAL, the window exists in both worlds, and what
|
||
// differs is what the run does with it — which is the thing under test.
|
||
func waitForRefusalInFlight(r *Runner) bool {
|
||
deadline := time.Now().Add(20 * time.Second)
|
||
for time.Now().Before(deadline) {
|
||
if inFlight, _ := r.gate.stateNow(); inFlight >= 1 && r.gate.refusalsSoFar() >= 1 {
|
||
return true
|
||
}
|
||
time.Sleep(2 * time.Millisecond)
|
||
}
|
||
return false
|
||
}
|
||
|
||
// waitForGate blocks until the run reaches the state the test is about and REPORTS whether it did. It
|
||
// polls the gate's own counters rather than sleeping for a guessed interval: what the test needs to
|
||
// observe is a STATE, and a sleep would assert a duration.
|
||
//
|
||
// ⚠ IT RETURNS A BOOL AND DOES NOT FAIL THE TEST, because every caller runs it on a SPAWNED goroutine.
|
||
// t.Fatalf outside the test goroutine calls runtime.Goexit on the wrong one: the assertion is lost, the
|
||
// helper's own goroutine dies silently, and the run it was supposed to unblock hangs until the package
|
||
// timeout — a green-looking suite turning into an unexplained timeout the moment a planting makes the
|
||
// state unreachable, which is exactly when the harness must be trustworthy.
|
||
func waitForGate(r *Runner, wantInFlight, wantWaiting int) bool {
|
||
deadline := time.Now().Add(20 * time.Second)
|
||
for time.Now().Before(deadline) {
|
||
if inFlight, waiting := r.gate.stateNow(); inFlight >= wantInFlight && waiting >= wantWaiting {
|
||
return true
|
||
}
|
||
time.Sleep(2 * time.Millisecond)
|
||
}
|
||
return false
|
||
}
|
||
|
||
// TestUnderACeilingTheWaveFinishesWhatItAdmitted is THE test of row 277, and its shape is the defect's
|
||
// shape. Four workers, a ceiling that admits one call, and the admitted call held open at the provider
|
||
// while a sibling is refused: the run must finish the held call and settle it.
|
||
//
|
||
// Before this pack the refusal cancelled the derived context, the held request died with
|
||
// `context.Canceled`, its reservation was released, and the book departed with committed spend of ZERO —
|
||
// the whole of the live 04.09 measurement, in one test.
|
||
func TestUnderACeilingTheWaveFinishesWhatItAdmitted(t *testing.T) {
|
||
rec := &reqRec{}
|
||
release := make(chan struct{})
|
||
srv := gatedProvider(rec, 1, release)
|
||
defer srv.Close()
|
||
// Five single-chunk chapters over four workers: several calls are attempted at once, and the ceiling
|
||
// admits one of them.
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
r.CeilingUSD = oneCallCeilingUSD(t, r)
|
||
|
||
reached := make(chan bool, 1)
|
||
go func() {
|
||
// Release the held call only once the window this test is about is OPEN: the ceiling has refused
|
||
// somebody AND the admitted call is still running. Released either way — a run left holding a
|
||
// request forever would turn a failed assertion into a package timeout.
|
||
reached <- waitForRefusalInFlight(r)
|
||
close(release)
|
||
}()
|
||
|
||
res, err := r.TranslateBook(ctx)
|
||
if !<-reached {
|
||
t.Fatal("the ceiling never refused anything while a call was in flight — this fixture does not build the situation the pack is about")
|
||
}
|
||
if err == nil {
|
||
t.Fatal("a ceiling that admits one call must still stop the book")
|
||
}
|
||
var halt *CeilingHalt
|
||
if !errors.As(err, &halt) {
|
||
t.Fatalf("the stop must be a ceiling halt, got %v", err)
|
||
}
|
||
committed, reserved, serr := r.Store.SpentUSD("test-book")
|
||
if serr != nil {
|
||
t.Fatal(serr)
|
||
}
|
||
// ⛔ THE ASSERTION THE PACK EXISTS FOR. The admitted call was finished and paid for. Zero here is the
|
||
// old behaviour exactly: a run that spent nothing, produced nothing and took ten seconds to do it.
|
||
if committed <= 0 {
|
||
t.Fatalf("the admitted call was killed by a sibling's refusal: committed=%v, provider saw %d call(s)", committed, rec.count())
|
||
}
|
||
if reserved != 0 {
|
||
t.Fatalf("a reservation leaked through the stop: reserved=%v", reserved)
|
||
}
|
||
if rec.count() == 0 {
|
||
t.Fatal("nothing reached the provider at all — the fixture no longer admits a single call")
|
||
}
|
||
// The run departs with a RESULT, not with nothing: exit 4 and `paused` stay, and what was bought is
|
||
// accounted for rather than discarded.
|
||
if res == nil {
|
||
t.Fatal("a ceiling stop must carry the run's partial result, or the operator is told only that it stopped")
|
||
}
|
||
}
|
||
|
||
// TestASpendStopReportsNoVolumeLedger is the withholding of VolumeStop, asserted where it can FAIL.
|
||
//
|
||
// ⛔ THE ASSERTION LIVED IN THE TEST ABOVE AND WAS VACUOUS THERE: that fixture grants no volume at all,
|
||
// so `res.Volume` is nil whatever the code does, and the planting that removes the withholding SURVIVED.
|
||
// A volume ledger can only be wrongly attached by a run that HAS a volume grant, so that is what this
|
||
// fixture builds — and the money ceiling then stops it in the middle of the grant, which is the only
|
||
// state where the seven plan counters would describe units that were admitted and never done.
|
||
func TestASpendStopReportsNoVolumeLedger(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
// ONE worker: the admission order is then the item order, and the arithmetic below is a statement
|
||
// rather than a hope. What this test is about is which WAVE the stop lands in, not concurrency.
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(4, 1400), waveWorkers: 1, bookUSD: 100})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
// ⛔ BOTH NUMBERS ARE LOAD-BEARING, AND THE SECOND ONE TOOK TWO ATTEMPTS.
|
||
//
|
||
// `MaxUnits` gives the run a REAL volume grant, so the seven plan counters exist at all — without one
|
||
// `res.Volume` is nil whatever the code does, and the first version of this test asserted that nil.
|
||
//
|
||
// The ceiling must then stop the run IN THE EDIT WAVE, and that is the part the second attempt got
|
||
// wrong: `res.Volume` is attached BETWEEN the waves (waverun.go, right after the draft wave), so a
|
||
// ceiling tight enough to stop the DRAFT wave returns through a path that never attaches it — and the
|
||
// withholding under test is a no-op there too. So the ceiling is sized to pay for every draft of the
|
||
// grant and to run out on the first edit: three drafts settle at a fraction of their estimate, and the
|
||
// edit call's own estimate no longer fits.
|
||
r.MaxUnits = 3
|
||
r.CeilingUSD = callCeilingUSD(t, r, 1.64)
|
||
|
||
res, err := r.TranslateBook(ctx)
|
||
if err == nil {
|
||
t.Fatal("the tight ceiling must stop the run inside its volume grant")
|
||
}
|
||
var halt *CeilingHalt
|
||
if !errors.As(err, &halt) {
|
||
t.Fatalf("want a ceiling halt, got %v", err)
|
||
}
|
||
if res == nil {
|
||
t.Fatal("a ceiling stop must carry the run's partial result")
|
||
}
|
||
if res.Volume != nil {
|
||
t.Fatalf("a SPEND stop reported a VOLUME ledger %+v — those seven counters are a PLAN, and a run stopped by money leaves them counting units that were admitted and never done: Delivered over-counts, LeftFresh under-counts, and at Left()==0 the run would announce it reached the end of the book",
|
||
*res.Volume)
|
||
}
|
||
}
|
||
|
||
// oneCallCeilingUSD is a ceiling that admits exactly one of this book's calls: halfway between one
|
||
// reservation and two. It is DERIVED from the engine's own arithmetic rather than written as a literal,
|
||
// so a change to sizing or pricing moves the fixture with the code instead of silently making the test
|
||
// about a different situation.
|
||
func oneCallCeilingUSD(t *testing.T, r *Runner) float64 {
|
||
return callCeilingUSD(t, r, 1.5)
|
||
}
|
||
|
||
// callCeilingUSD is `mult` times ONE draft call's reservation for this fixture's chunk size. Derived from
|
||
// the engine's own arithmetic rather than written as a literal, so a change to sizing or pricing moves
|
||
// the fixture with the code instead of quietly making the test about a different situation. The sizing
|
||
// length is the fixture's own chapter length (cjkSource(_, chapterRunes)).
|
||
func callCeilingUSD(t *testing.T, r *Runner, mult float64) float64 {
|
||
t.Helper()
|
||
plan := r.pricePlanFor()
|
||
if plan == nil || len(plan.stages) == 0 {
|
||
t.Fatal("the fixture pipeline resolves no priceable stage")
|
||
}
|
||
sp := plan.stages[0]
|
||
base := r.baseMaxTokensFor(sp.st, fixtureChapterRunes)
|
||
// ⚠ NO INJECTION ALLOWANCE HERE, and the omission is the fixture's truth rather than a shortcut. These
|
||
// projects carry no glossary, so the bank injection is not rendered at all and the executor reserves
|
||
// without it. Charging the plan's 800-token allowance inflated every ceiling derived here by ~11 %, and
|
||
// that quietly turned a «one reservation at a time» fixture into one where two fit — which is how the
|
||
// delivering-ceiling test below came to pass without the waiting path ever running.
|
||
one := ledger.EstimateUSD(sp.price, sp.tplTokens+fixtureChapterRunes, base, sp.reasoning)
|
||
if one <= 0 {
|
||
t.Fatal("the fixture's first stage is free — a ceiling test needs a priced model")
|
||
}
|
||
return one * mult
|
||
}
|
||
|
||
// fixtureChapterRunes is the chapter length every fixture in this file uses. Named once so the ceilings
|
||
// derived from it cannot drift away from the books they are derived for.
|
||
const fixtureChapterRunes = 1400
|
||
|
||
// TestARefusalWithNothingInFlightIsFinal pins the other half of the rule: waiting is what makes a refusal
|
||
// non-final, and there is nothing to wait for when no call holds a reservation. A single-worker run whose
|
||
// ceiling admits nothing must stop at once rather than hang.
|
||
func TestARefusalWithNothingInFlightIsFinal(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{bookUSD: 0.0000001})
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
|
||
done := make(chan error, 1)
|
||
go func() { _, err := r.TranslateBook(context.Background()); done <- err }()
|
||
select {
|
||
case err := <-done:
|
||
if err == nil {
|
||
t.Fatal("a ceiling that admits nothing must stop the run")
|
||
}
|
||
var halt *CeilingHalt
|
||
if !errors.As(err, &halt) {
|
||
t.Fatalf("want a ceiling halt, got %v", err)
|
||
}
|
||
case <-time.After(30 * time.Second):
|
||
t.Fatal("the run HUNG on a refusal with nothing in flight — waiting for a settle that can never come")
|
||
}
|
||
if rec.count() != 0 {
|
||
t.Fatalf("a denied reserve must not reach the provider, calls=%d", rec.count())
|
||
}
|
||
}
|
||
|
||
// TestTheGateWaitsWhileTheSkyIsFullAndStopsWhenItEmpties exercises the gate directly, because the two
|
||
// answers it gives are what the whole behaviour rests on and they must be pinned without a provider, a
|
||
// store or a clock.
|
||
func TestTheGateWaitsWhileTheSkyIsFullAndStopsWhenItEmpties(t *testing.T) {
|
||
var g reserveGate
|
||
if got := g.waitForSettle(context.Background(), true); got != waitNothingInFlight {
|
||
t.Fatalf("with nothing in flight the refusal is final, got outcome %d", got)
|
||
}
|
||
// Simulate an admitted call by hand — admit() needs a store, and what is under test here is the
|
||
// counter's contract, not the reservation's.
|
||
g.mu.Lock()
|
||
g.inFlight = 1
|
||
g.mu.Unlock()
|
||
|
||
woke := make(chan waitOutcome, 1)
|
||
go func() { woke <- g.waitForSettle(context.Background(), true) }()
|
||
// The waiter must be parked before the settle, or the test proves nothing about the broadcast.
|
||
deadline := time.Now().Add(5 * time.Second)
|
||
for {
|
||
if _, waiting := g.stateNow(); waiting == 1 {
|
||
break
|
||
}
|
||
if time.Now().After(deadline) {
|
||
t.Fatal("the waiter never parked")
|
||
}
|
||
time.Sleep(time.Millisecond)
|
||
}
|
||
g.done()
|
||
select {
|
||
case got := <-woke:
|
||
if got != waitSettled {
|
||
t.Fatalf("a settle must wake the waiter so it can re-ask for the headroom that just grew, got outcome %d", got)
|
||
}
|
||
case <-time.After(5 * time.Second):
|
||
t.Fatal("the settle did not wake the waiter — the broadcast channel was replaced without being closed")
|
||
}
|
||
if inFlight, waiting := g.stateNow(); inFlight != 0 || waiting != 0 {
|
||
t.Fatalf("after the settle the gate must be empty, got in_flight=%d waiting=%d", inFlight, waiting)
|
||
}
|
||
// And a context that dies while waiting must release the waiter rather than strand it.
|
||
g.mu.Lock()
|
||
g.inFlight = 1
|
||
g.mu.Unlock()
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
cancel()
|
||
// ⛔ AND IT MUST BE ITS OWN ANSWER. Reported as «nothing in flight», a cancelled wait becomes a
|
||
// CeilingHalt one level up, and a run somebody stopped with Ctrl-C publishes a `ceiling` event with
|
||
// a shortfall — telling the platform to add money for a stop that had nothing to do with money.
|
||
if got := g.waitForSettle(ctx, true); got != waitAborted {
|
||
t.Fatalf("a dead context must be distinguishable from an empty sky, got outcome %d", got)
|
||
}
|
||
}
|
||
|
||
// TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent pins the futility check. Waiting is bounded
|
||
// by the longest provider timeout, so waiting for a settle that provably cannot admit the call costs
|
||
// minutes to arrive at the answer already in hand.
|
||
func TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent(t *testing.T) {
|
||
// committed 0.9, estimate 0.2, ceiling 1.0 — every reservation could be released for free and 1.1
|
||
// would still be past 1.0.
|
||
if !settleCannotHelp(runevents.ScopeBook, 0.9, 0.2, 1.0) {
|
||
t.Error("committed alone already leaves no room: no settle can change that")
|
||
}
|
||
// committed 0.5, estimate 0.2 — the refusal came from RESERVED money, which a settle gives back.
|
||
if settleCannotHelp(runevents.ScopeBook, 0.5, 0.2, 1.0) {
|
||
t.Error("the refusal is about reserved money, and reserved money is exactly what a settle returns")
|
||
}
|
||
// The DAY ceiling sums other books, so this process's committed figure is not the day's: it must not
|
||
// be used to declare a refusal final.
|
||
if settleCannotHelp(runevents.ScopeDay, 0.9, 0.2, 1.0) {
|
||
t.Error("the day ceiling is not this book's arithmetic, so this book's committed spend cannot rule the wait out")
|
||
}
|
||
// No ceiling means no refusal to reason about.
|
||
if settleCannotHelp(runevents.ScopeBook, 9, 9, 0) {
|
||
t.Error("a zero ceiling is «no limit» and cannot refuse anything")
|
||
}
|
||
}
|
||
|
||
// TestTheShortfallIsTheDistanceToTheCeilingAndNotThePriceOfACall pins WHICH number leaves the engine.
|
||
// The denied estimate is the price of one call and the contract keeps our cost structure inside; the
|
||
// shortfall is the distance between a limit the platform set and a total it has already authorised.
|
||
func TestTheShortfallIsTheDistanceToTheCeilingAndNotThePriceOfACall(t *testing.T) {
|
||
// committed 0.30 + reserved 0.20 + estimate 0.07 = 0.57 against a ceiling of 0.55 → 0.02 missing.
|
||
got := shortfallMicroUSD(runevents.ScopeBook, 0.30, 0.20, 0.07, 0.55)
|
||
if got != 20_000 {
|
||
t.Errorf("shortfall = %d micro-USD, want 20000 (the amount the admission overshot by)", got)
|
||
}
|
||
// It is NOT the estimate: publishing that would publish the price of a call.
|
||
if got == 70_000 {
|
||
t.Error("the engine published the DENIED ESTIMATE — that is the price of one call and it may not leave")
|
||
}
|
||
// Rounded UP: a figure a person tops up against must never be short.
|
||
if up := shortfallMicroUSD(runevents.ScopeBook, 0, 0, 0.0000005, 0); up != 0 {
|
||
t.Errorf("a zero ceiling states nothing, got %d", up)
|
||
}
|
||
if up := shortfallMicroUSD(runevents.ScopeBook, 1, 0, 0.0000004, 1); up != 1 {
|
||
t.Errorf("a sub-micro-dollar shortfall must round UP to 1, got %d", up)
|
||
}
|
||
// The day scope cannot state it honestly and therefore does not.
|
||
if d := shortfallMicroUSD(runevents.ScopeDay, 0.9, 0, 0.2, 1.0); d != 0 {
|
||
t.Errorf("the day scope must state no shortfall (its figures are one book's, its ceiling is every book's), got %d", d)
|
||
}
|
||
}
|
||
|
||
// TestTheStoppedRunTellsAConsumerHowMuchWasMissing is the SEAM axis, read from the consumer's side: the
|
||
// events file is parsed as bytes, exactly as the platform tails it, rather than by asking the engine's
|
||
// own structs what they hold.
|
||
func TestTheStoppedRunTellsAConsumerHowMuchWasMissing(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(3, 1400), bookUSD: 100})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r := newRunner(t, bookPath)
|
||
r.CeilingUSD = callCeilingUSD(t, r, 1.05) // one reservation fits; the book needs six calls
|
||
if _, err := r.TranslateBook(ctx); err == nil {
|
||
t.Fatal("the tight ceiling must stop the run")
|
||
}
|
||
journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl")
|
||
r.Close()
|
||
|
||
raw, err := os.ReadFile(journal)
|
||
if err != nil {
|
||
t.Fatalf("no event journal to read from the consumer side: %v", err)
|
||
}
|
||
var sawCeiling, sawFinished bool
|
||
var version string
|
||
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
||
if line == "" {
|
||
continue
|
||
}
|
||
var env struct {
|
||
Type string `json:"type"`
|
||
Data json.RawMessage `json:"data"`
|
||
}
|
||
if err := json.Unmarshal([]byte(line), &env); err != nil {
|
||
t.Fatalf("the stream is not readable NDJSON: %v (%s)", err, line)
|
||
}
|
||
switch env.Type {
|
||
case "hello":
|
||
var h struct {
|
||
StreamVersion string `json:"stream_version"`
|
||
}
|
||
mustJSON(t, env.Data, &h)
|
||
version = h.StreamVersion
|
||
case "ceiling":
|
||
var c struct {
|
||
Halted bool `json:"halted"`
|
||
Scope string `json:"scope"`
|
||
ShortfallMicroUSD int64 `json:"shortfall_micro_usd"`
|
||
}
|
||
mustJSON(t, env.Data, &c)
|
||
sawCeiling = true
|
||
if !c.Halted || c.Scope != "book" {
|
||
t.Errorf("the ceiling frame lost its existing fields: %+v", c)
|
||
}
|
||
// ⛔ THE FIELD THIS PACK ADDS. Without it the number the engine already printed dies in the
|
||
// stderr of a transient unit and the buyer is told the run stopped and nothing else.
|
||
if c.ShortfallMicroUSD <= 0 {
|
||
t.Errorf("the ceiling frame does not say how much was missing: %+v", c)
|
||
}
|
||
case "finished":
|
||
var f struct {
|
||
Outcome string `json:"outcome"`
|
||
Money *struct {
|
||
UnitsResolved int `json:"units_resolved"`
|
||
UnitsDeferred int `json:"units_deferred"`
|
||
} `json:"money"`
|
||
}
|
||
mustJSON(t, env.Data, &f)
|
||
sawFinished = true
|
||
if f.Outcome != "ceiling" {
|
||
t.Errorf("outcome = %q, want ceiling — a resumable stop is never `failed` (PD-113)", f.Outcome)
|
||
}
|
||
if f.Money == nil {
|
||
t.Error("a run stopped by money must say what it bought and what is still owed")
|
||
} else if f.Money.UnitsDeferred == 0 {
|
||
t.Errorf("this fixture cannot afford the whole book, so something must be deferred: %+v", f.Money)
|
||
}
|
||
}
|
||
}
|
||
if !sawCeiling || !sawFinished {
|
||
t.Fatalf("the stream is incomplete: ceiling=%v finished=%v", sawCeiling, sawFinished)
|
||
}
|
||
// The seam rule: a frame gained a field, so the version says so. A reader comparing only the major is
|
||
// unaffected — and a stream whose bytes changed while its version did not is the one thing a version
|
||
// exists to prevent (runevents.go records this exact miss, made once already).
|
||
if version != runevents.StreamVersion {
|
||
t.Errorf("hello says stream_version=%q, build says %q", version, runevents.StreamVersion)
|
||
}
|
||
if version == "1.2" {
|
||
t.Error("the stream gained a field and kept its old version — the miss the constant's own comment records")
|
||
}
|
||
}
|
||
|
||
func mustJSON(t *testing.T, raw []byte, into any) {
|
||
t.Helper()
|
||
if err := json.Unmarshal(raw, into); err != nil {
|
||
t.Fatalf("frame payload unreadable: %v (%s)", err, raw)
|
||
}
|
||
}
|
||
|
||
// TestResumeAfterAMoneyStopRePaysNothing is the DETERMINISM axis. A run stopped by money, resumed under a
|
||
// raised ceiling, must finish the book without buying again what it already bought — the admitted calls
|
||
// were settled and checkpointed, so they replay for free.
|
||
func TestResumeAfterAMoneyStopRePaysNothing(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(3, 1400), waveWorkers: 2, bookUSD: 100})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
r1.CeilingUSD = callCeilingUSD(t, r1, 1.3) // room for a call or two, not for the book
|
||
if _, err := r1.TranslateBook(ctx); err == nil {
|
||
t.Fatal("the tight ceiling must stop the run")
|
||
}
|
||
firstPass := rec.count()
|
||
committedAfterStop, reservedAfterStop, err := r1.Store.SpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
if firstPass == 0 {
|
||
t.Fatal("fixture drifted: the stopped run must have paid for something")
|
||
}
|
||
if reservedAfterStop != 0 {
|
||
t.Fatalf("a stopped run must leave no reservation behind, got %v", reservedAfterStop)
|
||
}
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
if _, err := r2.TranslateBook(ctx); err != nil {
|
||
t.Fatalf("the resumed run must finish the book under the book's own ceiling: %v", err)
|
||
}
|
||
committedAfterFinish, _, err := r2.Store.SpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// ⛔ THE ASSERTION IS THE TOTAL, and its first version was two statements that cannot fail. «Committed
|
||
// spend went down» is impossible — committed only grows — and `secondPass >= firstPass + rec.count()`
|
||
// is arithmetically unreachable, since `secondPass` is defined as `rec.count() - firstPass`. Both read
|
||
// like checks and neither could ever red. What actually catches a re-purchase is the COUNT of provider
|
||
// calls across both runs: this book is 3 units × 2 stages, so a resume that re-bought its first pass
|
||
// would show more than six.
|
||
const callsInTheWholeBook = 3 * 2
|
||
secondPass := rec.count() - firstPass
|
||
if secondPass <= 0 {
|
||
t.Fatal("the resume did no work at all — the fixture is not testing a resume")
|
||
}
|
||
if total := rec.count(); total != callsInTheWholeBook {
|
||
t.Fatalf("the book cost %d provider calls across the stop and the resume, want exactly %d — anything more is work bought twice (first pass %d, resume %d)",
|
||
total, callsInTheWholeBook, firstPass, secondPass)
|
||
}
|
||
if committedAfterFinish <= committedAfterStop {
|
||
t.Fatalf("the resume settled nothing: committed %v → %v", committedAfterStop, committedAfterFinish)
|
||
}
|
||
}
|
||
|
||
// TestAMoneyStopInTheDraftWaveNeverStartsTheEditWave pins the rule that a partial draft is not editable.
|
||
//
|
||
// ⛔ THE FAILURE IT PREVENTS IS PAID. The edit wave looks a member's draft up in a map built over EVERY
|
||
// chunk, so an unstarted member is FOUND — holding the zero value: not flagged, empty text. It would be
|
||
// joined into the unit's draft as a clean member, and the editor would be paid to edit a unit with a hole
|
||
// in it and then ship it.
|
||
func TestAMoneyStopInTheDraftWaveNeverStartsTheEditWave(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(4, 1400), waveWorkers: 2, bookUSD: 100})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
r.CeilingUSD = callCeilingUSD(t, r, 1.05) // one draft call fits; the book needs eight
|
||
if _, err := r.TranslateBook(ctx); err == nil {
|
||
t.Fatal("the tight ceiling must stop the run in the draft wave")
|
||
}
|
||
for _, body := range rec.all() {
|
||
if isEditBody(body) {
|
||
t.Fatal("an EDIT call was issued after the draft wave was stopped by money — the editor was paid to edit a unit whose members were never drafted")
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestNothingLeaksWhenTheGateSeesAnAdmissionError pins the counter's own failure mode: a reservation the
|
||
// store refused with an ERROR (not a denial) must not be counted as in flight, or every later refusal
|
||
// waits for a settle that can never arrive and the run hangs instead of stopping.
|
||
func TestNothingLeaksWhenTheGateSeesAnAdmissionError(t *testing.T) {
|
||
var g reserveGate
|
||
// A closed store makes Reserve fail as an error rather than deny.
|
||
dir := t.TempDir()
|
||
st, err := store.Open(filepath.Join(dir, "p.db"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
st.Close()
|
||
if _, _, err := g.admit(st, "b", 0.01, store.Ceilings{BookUSD: 100}); err == nil {
|
||
t.Skip("this store does not error on a closed handle; the counter's contract is pinned by the gate test above")
|
||
}
|
||
if inFlight, _ := g.stateNow(); inFlight != 0 {
|
||
t.Fatalf("a failed admission must not count as in flight, got %d", inFlight)
|
||
}
|
||
}
|
||
|
||
// TestWaitingForHeadroomDELIVERSTheWholeBook is the positive half of row 277 and the one the rest of the
|
||
// battery does not reach. Every other ceiling test ends in a stop; this one ends in a FINISHED BOOK that
|
||
// a ceiling would have refused without the wait.
|
||
//
|
||
// The shape is the real one. A reservation is worst-case (whole prompt uncached, completion running to
|
||
// max_tokens) while a settle is the actual cost, so a ceiling can be too small to hold two reservations
|
||
// at once and perfectly able to pay for the whole book one call at a time. Before the wait existed, the
|
||
// second parallel worker's refusal killed the run; with it, the wave goes serial under the ceiling and
|
||
// the book is delivered.
|
||
func TestWaitingForHeadroomDeliversTheWholeBook(t *testing.T) {
|
||
rec := &reqRec{}
|
||
// ⚠ THE FIRST CALL IS HELD OPEN, and that is what makes the collision a STATE rather than a race the
|
||
// test hopes for. Without it the two draft workers may never hold a reservation at the same moment —
|
||
// under `-race` they reliably do not: the battery caught this test green-by-luck and then failing,
|
||
// with the log showing chunk 0 settled before chunk 1 was ever admitted. A ceiling cannot refuse what
|
||
// is not asked concurrently, so the waiting path went unexercised and the assertion below said so.
|
||
release := make(chan struct{})
|
||
srv := gatedProvider(rec, 1, release)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(2, 1400), waveWorkers: 4, bookUSD: 100})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
// Room for ONE reservation at a time, and — because a settle costs a fraction of its estimate — room
|
||
// in TOTAL for every call this two-chapter book makes. Both halves are needed and neither is
|
||
// incidental: the first is what makes the parallel wave collide with the ceiling at all, the second is
|
||
// what makes the collision survivable. The ceiling stays cumulative throughout, which is why the book
|
||
// is two chapters and not ten.
|
||
// The window is narrow and both edges are real: BELOW two reservations (so the parallel wave collides
|
||
// with the ceiling and a worker has to wait) and ABOVE the whole book's settled cost (so the waiting
|
||
// wave finishes it). A settle costs a fraction of its estimate, which is what makes such a window
|
||
// exist at all — and it is why the book is two chapters and not ten, the ceiling being cumulative.
|
||
r.CeilingUSD = callCeilingUSD(t, r, 1.85)
|
||
|
||
reached := make(chan bool, 1)
|
||
go func() {
|
||
// Release the held call only once a worker is genuinely queued behind the ceiling — the state the
|
||
// whole test is about. Released either way, so a missed state fails an assertion instead of
|
||
// hanging the package.
|
||
reached <- waitForGate(r, 1, 1)
|
||
close(release)
|
||
}()
|
||
|
||
res, err := r.TranslateBook(ctx)
|
||
if !<-reached {
|
||
t.Fatal("no worker was ever queued behind the ceiling — the two draft reservations never overlapped, so this fixture does not exercise the waiting path")
|
||
}
|
||
if err != nil {
|
||
t.Fatalf("a ceiling that admits one call AT A TIME must still deliver the book: %v", err)
|
||
}
|
||
// ⛔ THE MECHANISM, ASSERTED — not inferred from the outcome. This test's first version checked only
|
||
// that the book was delivered, and it PASSED on a fixture whose ceiling never refused anything: the
|
||
// waiting path did not run, the mutant that deletes the wait would have survived, and the assertion
|
||
// read as proof of a road the run never took. A delivered book is not evidence of waiting; a wait is.
|
||
if waits := r.gate.waitsSoFar(); waits == 0 {
|
||
t.Fatalf("the book was delivered WITHOUT the ceiling ever refusing a reservation — this fixture does not exercise the waiting path at all, so it proves nothing about it (shipped %d units)", len(res.Chunks))
|
||
}
|
||
if len(res.Chunks) != 2 {
|
||
t.Fatalf("want 2 shipped units, got %d — the wave stopped instead of queueing", len(res.Chunks))
|
||
}
|
||
for _, oc := range res.Chunks {
|
||
if oc.Disposition != DispOK {
|
||
t.Fatalf("a unit came back %s under the queueing ceiling", oc.Disposition)
|
||
}
|
||
}
|
||
committed, reserved, serr := r.Store.SpentUSD("test-book")
|
||
if serr != nil {
|
||
t.Fatal(serr)
|
||
}
|
||
if reserved != 0 {
|
||
t.Fatalf("no reservation may survive a completed run, reserved=%v", reserved)
|
||
}
|
||
// ⛔ AND THE CEILING STILL HELD. The whole point of keeping the reservation is that queueing never
|
||
// becomes overspending: committed spend is under the limit that was in force, every call of it.
|
||
if committed > r.CeilingUSD {
|
||
t.Fatalf("the queueing wave OVERSPENT its ceiling: committed=%v ceiling=%v", committed, r.CeilingUSD)
|
||
}
|
||
}
|
||
|
||
// TestACancelledWaitIsNotAMoneyStop pins the defect that a boolean answer created and a three-valued one
|
||
// closes. A worker waiting for headroom when the run is cancelled must surface the CANCELLATION.
|
||
//
|
||
// ⛔ WHAT IT PREVENTS. Reported as «nothing in flight», a cancelled wait became a CeilingHalt one level
|
||
// up: the run published a `ceiling` event carrying a shortfall, and the platform — which lets that event
|
||
// survive any exit code — recorded `paused` and told a person to add money for a run they had stopped
|
||
// themselves. The same path was reachable from a sibling's infra failure, which cancels.
|
||
func TestACancelledWaitIsNotAMoneyStop(t *testing.T) {
|
||
rec := &reqRec{}
|
||
release := make(chan struct{})
|
||
srv := gatedProvider(rec, 1, release)
|
||
defer srv.Close()
|
||
// ⚠ DECLARED AFTER srv's defer SO IT RUNS BEFORE IT — defers are LIFO, and httptest.Server.Close()
|
||
// WAITS for outstanding requests. Ordered the other way this deadlocks the package: the handler is
|
||
// parked on `release`, Close() is parked on the handler, and the close that would free both is queued
|
||
// behind Close(). Written the wrong way round it HUNG the package rather than failing it — the
|
||
// timeout-shaped failure this pack has already recorded as a class of its own, met once more.
|
||
defer close(release)
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100})
|
||
ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}))
|
||
|
||
r := newRunner(t, bookPath)
|
||
r.CeilingUSD = oneCallCeilingUSD(t, r)
|
||
|
||
reached := make(chan bool, 1)
|
||
go func() {
|
||
// Cancel only once a worker is genuinely parked on the gate — otherwise the test would be about
|
||
// cancelling a run, not about cancelling a WAIT. Cancelled either way, so a missed state fails an
|
||
// assertion instead of hanging the package.
|
||
reached <- waitForGate(r, 1, 1)
|
||
cancel()
|
||
}()
|
||
_, err := r.TranslateBook(ctx)
|
||
if !<-reached {
|
||
t.Fatal("no worker was ever parked on the gate — the cancellation did not interrupt a WAIT and this test is about nothing")
|
||
}
|
||
journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl")
|
||
r.Close()
|
||
|
||
if err == nil {
|
||
t.Fatal("a cancelled run must not report success")
|
||
}
|
||
var halt *CeilingHalt
|
||
if errors.As(err, &halt) {
|
||
t.Fatalf("a run cancelled while waiting for headroom reported a MONEY stop: %v", err)
|
||
}
|
||
if !errors.Is(err, context.Canceled) {
|
||
t.Fatalf("the cancellation must be what leaves the run, got %v", err)
|
||
}
|
||
raw, rerr := os.ReadFile(journal)
|
||
if rerr != nil {
|
||
t.Fatalf("no event journal: %v", rerr)
|
||
}
|
||
if strings.Contains(string(raw), `"ceiling"`) {
|
||
t.Fatalf("the stream published a ceiling stop for a run somebody cancelled:\n%s", raw)
|
||
}
|
||
}
|
||
|
||
// TestAnInfraFailureAfterAMoneyStopDoesNotDepartAsPaused is the two-slot error ranking, asserted.
|
||
//
|
||
// ⛔ WHY IT NEEDS ITS OWN TEST. A spend ceiling no longer cancels its siblings, so for the first time a
|
||
// run can latch on money and THEN break for an unrelated reason. Sharing one error slot, whichever
|
||
// happened first would speak for both — and a first-wins ceiling would have a broken run depart exit 4,
|
||
// which the platform records as `paused`: «add money» offered as the remedy for a machine that failed.
|
||
// So the failure ranks on the way out, and the money fact travels on the channel it was given for
|
||
// exactly this case: the `ceiling` event, written at the moment of the refusal.
|
||
func TestAnInfraFailureAfterAMoneyStopDoesNotDepartAsPaused(t *testing.T) {
|
||
r := &Runner{}
|
||
halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 1234,
|
||
err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)}
|
||
infra := errors.New("provider unreachable")
|
||
|
||
// ⚠ BOTH WORKERS MUST BE HOLDING AN ITEM BEFORE EITHER FAILS, and the first attempt at this test got
|
||
// it wrong in a way worth keeping: with the ceiling returned first, the latch closed and the feeder
|
||
// never handed out the second item at all — so the failure never happened and the test asserted
|
||
// nothing. That is the latch behaving exactly as designed (nothing new starts), and it is also why
|
||
// this co-occurrence is only reachable among calls that were ALREADY admitted.
|
||
holding := make(chan struct{})
|
||
err := r.runWave(context.Background(), 2, 2, func(ctx context.Context, i int) error {
|
||
if i == 1 {
|
||
close(holding)
|
||
return infra
|
||
}
|
||
<-holding
|
||
return halt
|
||
})
|
||
if err == nil {
|
||
t.Fatal("the wave must report something")
|
||
}
|
||
var got *CeilingHalt
|
||
if errors.As(err, &got) {
|
||
t.Fatalf("a run that BROKE departed as a money stop (exit 4 → the platform records `paused`): %v", err)
|
||
}
|
||
if !errors.Is(err, infra) {
|
||
t.Fatalf("the failure must be what leaves the wave, got %v", err)
|
||
}
|
||
}
|
||
|
||
// TestAnOptionalCallIsTurnedAwayWhileMandatoryOneWaits is the `mandatory` rule, ASSERTED.
|
||
//
|
||
// ⛔ ITS ENTIRE BEHAVIOUR USED TO BE AN ARGUMENT IN A COMMENT. Every fixture that refused an optional call
|
||
// was sequential, so `inFlight == 0` produced an immediate halt whichever way the flag was set — flipping
|
||
// all three optional call sites to `true` left the battery green. The flag decides something ONLY when a
|
||
// sibling is in flight, so that is the state this test builds, at the level where the rule now lives.
|
||
//
|
||
// What it protects: an escalation hop that queued for headroom would spend the money the primary wave
|
||
// needs to finish work it has already started, and it holds escMu while it waits, so it would block every
|
||
// other worker's escalation too. The repair sub-step runs INSIDE an edit-wave worker and is the same
|
||
// story. All three degrade on a ceiling; none of them may wait for one.
|
||
func TestAnOptionalCallIsTurnedAwayWhileMandatoryOneWaits(t *testing.T) {
|
||
var g reserveGate
|
||
g.mu.Lock()
|
||
g.inFlight = 1 // money IS still moving: this is the only state in which the rule decides anything
|
||
g.mu.Unlock()
|
||
|
||
// The optional caller is turned away AT ONCE — not because there is nothing to wait for, but because
|
||
// it has no claim on it. The two are opposite facts and must not share an answer.
|
||
//
|
||
// ⚠ THE CONTEXT IS BOUNDED, AND THAT IS WHAT GIVES THE FAILURE A SHAPE. With context.Background() a
|
||
// broken rule does not fail here, it HANGS: the optional caller parks on a settle that never comes in
|
||
// a unit test, the package times out, and the catalogue reports `NOTHING` — «the planting did not
|
||
// build, or the run died» — which is indistinguishable from machine load. That is the timeout-shaped
|
||
// failure this pack has already recorded as a class, met an eighth time and caught immediately: with
|
||
// a deadline the same break returns waitAborted in two seconds and reds an assertion.
|
||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||
defer cancel()
|
||
if got := g.waitForSettle(ctx, false); got != waitNotAllowed {
|
||
t.Fatalf("an optional call refused while money is still moving must be turned away, got outcome %d — an optional call that queues for the last dollars spends what the mandatory work needed, and the escalation hop blocks every other worker's escalation while it waits", got)
|
||
}
|
||
if n := g.optionalTurnedAwaySoFar(); n != 1 {
|
||
t.Fatalf("the turn-away was not recorded (%d) — without the fact there is nothing a test can see", n)
|
||
}
|
||
if w := g.waitsSoFar(); w != 0 {
|
||
t.Fatalf("an optional call WAITED (%d): «do not start anything new» became «the optional spends what the mandatory needed»", w)
|
||
}
|
||
|
||
// …and the MANDATORY caller in the very same state does wait, and is woken by the settle.
|
||
woke := make(chan waitOutcome, 1)
|
||
go func() { woke <- g.waitForSettle(context.Background(), true) }()
|
||
deadline := time.Now().Add(5 * time.Second)
|
||
for {
|
||
if _, waiting := g.stateNow(); waiting == 1 {
|
||
break
|
||
}
|
||
if time.Now().After(deadline) {
|
||
t.Fatal("the mandatory caller never parked — the rule turns everyone away, which stops the book it was written to keep alive")
|
||
}
|
||
time.Sleep(time.Millisecond)
|
||
}
|
||
g.done()
|
||
select {
|
||
case got := <-woke:
|
||
if got != waitSettled {
|
||
t.Fatalf("the mandatory caller must be woken by the settle, got outcome %d", got)
|
||
}
|
||
case <-time.After(5 * time.Second):
|
||
t.Fatal("the settle did not wake the mandatory caller")
|
||
}
|
||
if w := g.waitsSoFar(); w != 1 {
|
||
t.Fatalf("waits=%d, want exactly the mandatory caller's one", w)
|
||
}
|
||
if n := g.optionalTurnedAwaySoFar(); n != 1 {
|
||
t.Fatalf("the mandatory caller was counted as an optional turn-away (%d)", n)
|
||
}
|
||
}
|
||
|
||
// TestOnlyOneCeilingFrameReachesTheStreamHoweverManyWorkersAreRefused pins the idempotence the emitter
|
||
// claims. Under one worker a run is refused once and the claim is untestable; under four it is refused
|
||
// several times, and a reader that received a `ceiling` frame per refusal would count one stop as many.
|
||
func TestOnlyOneCeilingFrameReachesTheStreamHoweverManyWorkersAreRefused(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: cjkSource(5, 1400), waveWorkers: 4, bookUSD: 100})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r := newRunner(t, bookPath)
|
||
r.CeilingUSD = callCeilingUSD(t, r, 1.05)
|
||
if _, err := r.TranslateBook(ctx); err == nil {
|
||
t.Fatal("the tight ceiling must stop the book")
|
||
}
|
||
journal := filepath.Join(filepath.Dir(r.Book.ProjectDB), "events.jsonl")
|
||
r.Close()
|
||
|
||
raw, err := os.ReadFile(journal)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
frames := 0
|
||
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
||
var env struct {
|
||
Type string `json:"type"`
|
||
}
|
||
if line == "" || json.Unmarshal([]byte(line), &env) != nil {
|
||
continue
|
||
}
|
||
if env.Type == "ceiling" {
|
||
frames++
|
||
}
|
||
}
|
||
if frames != 1 {
|
||
t.Fatalf("the stream carries %d ceiling frames for ONE stop — a reader counting stops would count refusals instead", frames)
|
||
}
|
||
}
|
||
|
||
// ⛔ THERE IS NO TEST HERE FOR settleCannotHelp's WIRING, and the absence is deliberate and argued.
|
||
//
|
||
// The predicate has its own test (TestSettleCannotHelpRefusesToWaitForMoneyThatIsAlreadySpent). What no
|
||
// test in this file exercises is the `if` that consults it — and the reason is that the state it fires in
|
||
// is nearly unreachable in a wave of similar calls. It needs a refusal that no settle could lift
|
||
// (`committed + estimate > ceiling`) WHILE another call is still in flight. But the in-flight call was
|
||
// itself admitted at the same committed level with a comparable estimate, so if there was room for it
|
||
// there is room for this one: the two conditions contradict each other unless the estimates differ
|
||
// materially — heterogeneous chunk sizes, or a draft and an edit meeting in one wave, which they never do.
|
||
// And once the sky empties, `waitNothingInFlight` stops the book anyway, with or without the predicate.
|
||
//
|
||
// ⚠ A TEST THAT CLAIMED TO COVER IT STOOD HERE AND WAS DELETED, because it passed for a reason that had
|
||
// nothing to do with its name. It held a call at the fake provider and asserted the run ended while the
|
||
// call was still held — but the held request hit the attempt timeout, the transport RETRIED, and the
|
||
// retry answered instantly (the trace shows `latency_ms=5389` on a call the test believed was frozen).
|
||
// So the run ended because the call finished, the three refused workers had WAITED and been woken, and
|
||
// the assertion was about a situation the fixture never built. Removing it is the honest move: the pack's
|
||
// own rule is that «checked» without an artifact does not count, and an artifact that checks something
|
||
// else is worse than none, because it reads as coverage. What remains is this paragraph and the entry in
|
||
// the report's obstacle section.
|
||
|
||
// TestTheTerminalVocabularyNeverContradictsTheExitCode pins the ORDER of terminal()'s branches — a
|
||
// regression made and caught inside this very pack, and until now held by nothing but a comment.
|
||
//
|
||
// ⛔ WHAT WENT WRONG. The money ledger is attached whenever a ceiling was reached, and the branch that
|
||
// does it was written ABOVE the signature stop and the cancellation. It swallowed both: a bank-stop on a
|
||
// book that had touched its ceiling reported `failed` while its exit code said 3, and a caught SIGTERM
|
||
// reported `failed` while its exit code said 5. Two channels contradicting each other about one run is
|
||
// exactly what this vocabulary exists to prevent — and re-ordering `case` arms is precisely the edit a
|
||
// tidy-up makes without noticing.
|
||
func TestTheTerminalVocabularyNeverContradictsTheExitCode(t *testing.T) {
|
||
halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 42,
|
||
err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)}
|
||
for _, tc := range []struct {
|
||
name string
|
||
err error
|
||
outcome string
|
||
}{
|
||
// Each of these has its OWN exit code (3, 5, 4, 1), and the stream must say the same thing.
|
||
{"a signature stop on a book that touched its ceiling", &WaveSignatureStop{Terms: 1, SignaturePath: "/x"}, runevents.OutcomeBankStop},
|
||
{"a caught SIGTERM after the money ran out", context.Canceled, runevents.OutcomeStopped},
|
||
{"the ceiling stop itself", halt, runevents.OutcomeCeiling},
|
||
{"an infra failure on top of a caught ceiling", errors.New("provider unreachable"), runevents.OutcomeFailed},
|
||
} {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
dir := t.TempDir()
|
||
st, err := store.Open(filepath.Join(dir, "p.db"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer st.Close()
|
||
e, err := openEmitter(st, dir, "trace-emitter-order", "test-book", slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
e.ceilingReached(halt) // every case is a run whose money DID run out first
|
||
e.terminal(nil, tc.err)
|
||
e.flush()
|
||
|
||
raw, rerr := os.ReadFile(filepath.Join(dir, "events.jsonl"))
|
||
if rerr != nil {
|
||
t.Fatal(rerr)
|
||
}
|
||
var got string
|
||
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
||
var env struct {
|
||
Type string `json:"type"`
|
||
Data json.RawMessage `json:"data"`
|
||
}
|
||
if line == "" || json.Unmarshal([]byte(line), &env) != nil || env.Type != "finished" {
|
||
continue
|
||
}
|
||
var f struct {
|
||
Outcome string `json:"outcome"`
|
||
}
|
||
mustJSON(t, env.Data, &f)
|
||
got = f.Outcome
|
||
}
|
||
if got != tc.outcome {
|
||
t.Fatalf("the stream says %q for a run whose exit code says otherwise; want %q", got, tc.outcome)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestTheMoneyLedgerFollowsTheCEILINGAndNotTheOUTCOME pins BOTH directions of `Finished.money`'s presence
|
||
// rule, neither of which had a pin before acceptance found the contract file saying something else.
|
||
//
|
||
// ⛔ THE CONTRACT FILE SAID «present only on `outcome: ceiling`» AND BOTH VERIFIERS CAUGHT IT
|
||
// INDEPENDENTLY. It is the doc a CONSUMER IN ANOTHER ZONE reads, so the two readings it invited are two
|
||
// different bugs in somebody else's code: «money ⇒ the run stopped on a ceiling» is wrong on `stopped`
|
||
// and `failed`, and «ceiling ⇒ money is there» is wrong when the run never learned the book's cut. The
|
||
// real rule is «a ceiling was REACHED», and it is asserted here in both directions because a sentence
|
||
// nothing pins is a sentence that drifts.
|
||
func TestTheMoneyLedgerFollowsTheCeilingAndNotTheOutcome(t *testing.T) {
|
||
halt := &CeilingHalt{Scope: runevents.ScopeBook, ShortfallMicroUSD: 7,
|
||
err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)}
|
||
|
||
// A cut the counters can be seeded from — one output unit, one draft stage, none of it resolved.
|
||
units := []editUnit{{Chapter: 1, FirstChunkIdx: 0, Members: []chunk.Chunk{{Chapter: 1, ChunkIdx: 0}}}}
|
||
shape := waveShape{draftNames: map[string]bool{"draft": true}, nDraft: 1}
|
||
|
||
for _, tc := range []struct {
|
||
name string
|
||
err error
|
||
ceiling bool
|
||
seedCut bool
|
||
wantOutcome string
|
||
wantMoney bool
|
||
}{
|
||
// DIRECTION 1 — money rides the ceiling, not the word. Each of these departs as something else.
|
||
{"a caught SIGTERM after the money ran out", context.Canceled, true, true, runevents.OutcomeStopped, true},
|
||
{"an infra failure on top of a caught ceiling", errors.New("provider unreachable"), true, true, runevents.OutcomeFailed, true},
|
||
{"a signing stop on a book that touched its ceiling", &WaveSignatureStop{Terms: 1, SignaturePath: "/x"}, true, true, runevents.OutcomeBankStop, true},
|
||
{"the tidy ceiling stop", halt, true, true, runevents.OutcomeCeiling, true},
|
||
// DIRECTION 2 — the word does not imply the field.
|
||
{"a ceiling stop before the run learned the book's cut", halt, true, false, runevents.OutcomeCeiling, false},
|
||
// …and no ceiling at all means no ledger, whatever else happened.
|
||
{"a plain infra failure, no ceiling anywhere", errors.New("provider unreachable"), false, true, runevents.OutcomeFailed, false},
|
||
} {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
dir := t.TempDir()
|
||
st, err := store.Open(filepath.Join(dir, "p.db"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer st.Close()
|
||
e, err := openEmitter(st, dir, "trace-money-rule", "test-book", slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if tc.seedCut {
|
||
e.beginWaves(units, shape, map[chunkKey][]store.ChunkStatus{})
|
||
}
|
||
if tc.ceiling {
|
||
e.ceilingReached(halt)
|
||
}
|
||
e.terminal(nil, tc.err)
|
||
e.flush()
|
||
|
||
raw, rerr := os.ReadFile(filepath.Join(dir, "events.jsonl"))
|
||
if rerr != nil {
|
||
t.Fatal(rerr)
|
||
}
|
||
var outcome string
|
||
var hasMoney bool
|
||
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
||
var env struct {
|
||
Type string `json:"type"`
|
||
Data json.RawMessage `json:"data"`
|
||
}
|
||
if line == "" || json.Unmarshal([]byte(line), &env) != nil || env.Type != "finished" {
|
||
continue
|
||
}
|
||
var f struct {
|
||
Outcome string `json:"outcome"`
|
||
Money json.RawMessage `json:"money"`
|
||
}
|
||
mustJSON(t, env.Data, &f)
|
||
outcome, hasMoney = f.Outcome, len(f.Money) > 0
|
||
}
|
||
if outcome != tc.wantOutcome {
|
||
t.Fatalf("outcome = %q, want %q", outcome, tc.wantOutcome)
|
||
}
|
||
if hasMoney != tc.wantMoney {
|
||
t.Fatalf("money present = %v, want %v — the field follows «a ceiling was REACHED», not the outcome word; a consumer building on either reading gets a different bug",
|
||
hasMoney, tc.wantMoney)
|
||
}
|
||
})
|
||
}
|
||
}
|