textmachine/platform/internal/runs/seam_test.go

648 lines
29 KiB
Go

package runs
import (
"errors"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"textmachine/platform/internal/ingest"
"textmachine/platform/internal/money"
"textmachine/platform/internal/pgstore"
"textmachine/platform/internal/runner"
)
// The consumer half of the run-event seam (П-15), driven through the WHOLE sweep and not through the
// pure function next door. The decision tables in reconcile_test.go say what `outcome` returns; these
// say that a run really does end up in that state, with its money settled and its next action taken
// — which is the gap a reviewer found once already, by deleting a branch a table test could not miss.
// spawned admits a run and starts its unit, returning the live attempt.
func spawned(t *testing.T, f *fixture, chapters int) pgstore.LiveRun {
t.Helper()
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), CeilingChapters: chapters})
if err != nil {
t.Fatal(err)
}
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
t.Fatal(err)
}
f.runner.alive = false // the unit is gone by the time the sweep looks, which is what a marker means
return f.live(t)
}
// ends writes the exit marker of a live attempt, as ExecStopPost would.
func ends(t *testing.T, f *fixture, l pgstore.LiveRun, m runner.Marker) {
t.Helper()
m.Unit = l.UnitName
if m.At.IsZero() {
m.At = f.now.Add(time.Minute)
}
if err := runner.WriteMarker(f.svc.markerPath(l.RunID, l.AttemptNo), m); err != nil {
t.Fatal(err)
}
}
func exitedWith(code string) runner.Marker {
return runner.Marker{Result: "exit-code", Code: "exited", Status: code}
}
// card reads the run the book's card carries.
func card(t *testing.T, f *fixture) pgstore.Run {
t.Helper()
_, run, err := f.store.GetBook(f.ctx, "u1", f.bookID(t))
if err != nil {
t.Fatal(err)
}
if run == nil {
t.Fatal("the book has no run")
}
return *run
}
// PD-113, end to end and with NO event on the stream at all — which is the half only the exit code
// can carry, and the half that did not exist before D39.131. A journal the engine could not write
// still leaves exit 4 behind.
//
// Three things have to be true together, and the third is why this is a sweep test: the run is
// `paused`, it carries a reason (a paused run with a null one gives the screen nothing to say), and
// it is NOT restarted — a ceiling halt is a run that has spent what was bought for it, and starting
// a second engine would walk it into the same stop and charge for the privilege.
//
// Mutation caught: dropping ingest.OutcomeCeiling from outcome(), and dropping PausedReason from
// RunEnding.
func TestACeilingHaltWithNoEventIsPausedWithItsReasonAndNotRestarted(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
ends(t, f, l, exitedWith("4"))
spent := f.svc.Pricing.Ceiling(100)
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 40, Spend: usd(spent), Reserved: usd(0)}, nil)
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
got := card(t, f)
if got.Status != "paused" {
t.Fatalf("a ceiling halt was recorded as %q; the contract forbids `failed` for a resumable stop", got.Status)
}
// The reason the platform can stand behind: a ceiling stopped it, and with no event to say WHOSE
// ceiling, that is exactly what is recorded. `credit_exhausted` would be a guess that lights the
// account's own halted flag.
if got.PausedReason != pgstore.PausedCeilingUnknown {
t.Errorf("paused_reason %q, want %q", got.PausedReason, pgstore.PausedCeilingUnknown)
}
if n := len(f.runner.starts()); n != 1 {
t.Errorf("%d units started: a ceiling halt must not be respawned", n)
}
if acct := f.account(t); acct.Reserved != 0 {
t.Errorf("the hold is still open after a ceiling halt: %s", acct.Reserved.USD())
}
}
// The other channel, and the reason the re-read exists. The ceiling event arrives in the drain of
// THIS very sweep, so the snapshot the pass started from still says nothing — and judging the ending
// from that snapshot answered `failed` for a run whose own stream had just said `ceiling`
// (acceptance of D39.131, п.3). It also carries the SCOPE, which the exit code cannot.
//
// Mutation caught: reading paused_reason from the sweep's snapshot instead of re-reading it after
// the drain.
func TestACeilingEventArrivingInThisVerySweepStillDecidesTheEnding(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
journal := filepath.Join(f.workdir, ingest.JournalFile)
body := hello(t, f) +
`{"seq":2,"type":"ceiling","data":{"halted":true,"scope":"day"}}` + "\n" +
`{"seq":3,"type":"finished","data":{"outcome":"ceiling"}}` + "\n"
if err := os.WriteFile(journal, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
// The marker says only that the process ended; the class it ended with is on the stream. Written
// as exit 1 on purpose — that is the shape this platform saw for a ceiling halt before the engine
// had a code for it, and the branch must not depend on the code being there.
ends(t, f, l, exitedWith("1"))
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 40, Spend: usd(1_000_000), Reserved: usd(0)}, nil)
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
got := card(t, f)
if got.Status != "paused" || got.PausedReason != pgstore.PausedDailyCeiling {
t.Fatalf("the run ended as %q/%q, want paused/%s", got.Status, got.PausedReason, pgstore.PausedDailyCeiling)
}
}
// A run the ENGINE'S OWN daily ceiling stopped cannot be continued by anything this platform can do:
// `ceilings.day_usd` lives in a book.yaml the operator owns. Resuming would open an attempt that
// meets the same limit inside the same day and comes straight back, paying for a status call and a
// transient unit each time — so the answer is a refusal with a reason, not a loop.
// ⚠ 0.3.0 WIDENED this: EVERY paused run is refused with `ceiling_reached`, not only the one the
// engine's daily ceiling stopped. The limit travels with the start of a run, so this call cannot
// move any of them, and the remedy the client offers is a NEW run with a larger ceiling — legal from
// any paused book, whatever the reason says, including `null` (canon §resumeRun).
//
// Mutation caught: re-opening a `paused` run instead of refusing it; refusing only the daily
// ceiling and letting the platform's own reason fall through into a re-open.
func TestAResumeOfARunPausedAtALimitIsRefusedWhoseverLimitItWas(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
// Only a little of the budget is gone, so `exhausted` is NOT what refuses this: the run has money
// left and would happily re-open without the guard.
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 5, Spend: usd(100_000), Reserved: usd(0)}, nil)
if _, err := f.store.PauseRun(f.ctx, l.RunID, l.AttemptID, pgstore.PausedDailyCeiling, f.now); err != nil {
t.Fatal(err)
}
if err := f.svc.Sweep(f.ctx); err != nil { // settles the attempt, so `deferred` is not what refuses either
t.Fatal(err)
}
if _, err := f.svc.Resume(f.ctx, "u1", l.RunID); !errors.Is(err, ErrCeilingReached) {
t.Fatalf("resume of a run the engine's daily ceiling stopped: %v, want ErrCeilingReached", err)
}
if got := card(t, f); got.Status != "paused" || got.PausedReason != pgstore.PausedDailyCeiling {
t.Errorf("the refused resume changed the run to %q/%q", got.Status, got.PausedReason)
}
if n := len(f.runner.starts()); n != 1 {
t.Errorf("%d units started: the refusal must not have spawned anything", n)
}
// The ceiling the PLATFORM set answers the same way and for the same reason. It used to be the
// opposite case — re-opened silently — and the client could not tell that 202 from a run that had
// actually continued.
if _, err := f.store.Pool().Exec(f.ctx,
`update runs set paused_reason = $2 where id = $1`, l.RunID, pgstore.PausedCreditExhausted); err != nil {
t.Fatal(err)
}
if _, err := f.svc.Resume(f.ctx, "u1", l.RunID); !errors.Is(err, ErrCeilingReached) {
t.Fatalf("resume of a run the platform's own ceiling stopped: %v, want ErrCeilingReached", err)
}
if n := len(f.runner.starts()); n != 1 {
t.Errorf("%d units started after the second refusal", n)
}
}
// Exit 5 is a caught SIGTERM and says nothing about who sent it. With nothing recorded on our side
// it is an INTERRUPTION — a host reboot, or an operator's `systemctl stop` — and the run comes back
// with what is left of its budget (unified backlog row 138). Burying it as `stopped` would leave
// every run on the host dead after a routine restart, which is the remaining half of PD-152.
//
// Mutation caught: removing the interruptedBySomeoneElse branch from reconcile().
func TestAGracefulSignalNobodyAskedForBringsTheRunBack(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
spent := money.MicroUSD(900_000)
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 30, Spend: usd(spent), Reserved: usd(0)}, nil)
ends(t, f, l, exitedWith("5"))
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
next := f.live(t)
if next.AttemptNo != 2 {
t.Fatalf("a run wound down by a signal nobody recorded was not restarted: attempt %d", next.AttemptNo)
}
if want := f.svc.Pricing.Ceiling(100) - spent; next.Ceiling != want {
t.Errorf("the new attempt reserved %s, want the budget minus what was spent (%s)",
next.Ceiling.USD(), want.USD())
}
if n := len(f.runner.starts()); n != 2 {
t.Errorf("%d units started in all, want the original and the restart", n)
}
if acct := f.account(t); acct.Balance != acct.LedgerSum {
t.Errorf("balance %s and ledger %s disagree after the restart", acct.Balance.USD(), acct.LedgerSum.USD())
}
}
// …and the same exit code with an intent recorded is the user's stop, which is never restarted: the
// money would go on work its owner had just cancelled.
func TestAGracefulSignalWeAskedForEndsTheRun(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 30, Spend: usd(900_000), Reserved: usd(0)}, nil)
if _, err := f.svc.Stop(f.ctx, "u1", l.RunID); err != nil {
t.Fatal(err)
}
ends(t, f, l, exitedWith("5"))
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
if got := card(t, f); got.Status != "stopped" {
t.Fatalf("a stop the user asked for ended as %q", got.Status)
}
if n := len(f.runner.starts()); n != 1 {
t.Errorf("%d units started: a run its owner stopped must not come back", n)
}
}
// A marker that EXISTS and does not parse used to fail the reconciliation of its run on every sweep,
// for good: the write is atomic, so nothing this platform does produces one, and nothing it does
// repairs one either (register row PD-164). The run is ended with the fact recorded instead.
//
// Mutation caught: mapping a decode failure back onto a plain error return.
func TestAnUnreadableExitMarkerEndsTheRunInsteadOfWedgingIt(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
path := f.svc.markerPath(l.RunID, l.AttemptNo)
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil {
t.Fatal(err)
}
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 10, Spend: usd(300_000), Reserved: usd(0)}, nil)
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
got := card(t, f)
if got.Status != "failed" || got.FinishedAt == nil {
t.Fatalf("a run whose marker cannot be read is %q (finished: %v)", got.Status, got.FinishedAt)
}
// The machine's word and this platform's conclusion have to be tellable apart by an operator.
var result string
if err := f.store.Pool().QueryRow(f.ctx,
`select exit_result from run_attempts where id = $1`, l.AttemptID).Scan(&result); err != nil {
t.Fatal(err)
}
if result != runner.UnreadableMarkerResult {
t.Errorf("exit_result %q, want %q", result, runner.UnreadableMarkerResult)
}
// And the money is resolved rather than left holding: the whole cost of the old behaviour was a
// hold that no sweep could ever close.
if acct := f.account(t); acct.Reserved != 0 {
t.Errorf("the hold is still open: %s", acct.Reserved.USD())
}
}
// The same corrupted marker over a run whose STREAM said a ceiling stopped it — and the stream is
// then the only witness there is, since a marker that does not parse carries no exit code either.
// Closing such a run `failed` with no reason contradicts the contract twice over: the stop is
// resumable, and the screen is left with a run that ended for no stated reason (acceptance dofix
// ФП-4).
//
// Mutation caught: re-reading the paused reason only on the branch where the marker parses.
func TestACeilingSurvivesAMarkerThatCannotBeRead(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
journal := filepath.Join(f.workdir, ingest.JournalFile)
body := hello(t, f) +
`{"seq":2,"type":"ceiling","data":{"halted":true,"scope":"day"}}` + "\n" +
`{"seq":3,"type":"finished","data":{"outcome":"ceiling"}}` + "\n"
if err := os.WriteFile(journal, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
path := f.svc.markerPath(l.RunID, l.AttemptNo)
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil {
t.Fatal(err)
}
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 10, Spend: usd(300_000), Reserved: usd(0)}, nil)
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
got := card(t, f)
if got.Status != "paused" || got.PausedReason != pgstore.PausedDailyCeiling {
t.Fatalf("the run ended as %q/%q, want paused/%s", got.Status, got.PausedReason, pgstore.PausedDailyCeiling)
}
// The unreadable marker is still recorded as what it was: the ceiling decides the STATUS, not
// what an operator gets to know about the marker.
var result string
if err := f.store.Pool().QueryRow(f.ctx,
`select exit_result from run_attempts where id = $1`, l.AttemptID).Scan(&result); err != nil {
t.Fatal(err)
}
if result != runner.UnreadableMarkerResult {
t.Errorf("exit_result %q, want %q", result, runner.UnreadableMarkerResult)
}
}
// WHICH build is asked about a book, after its run is over, is a money question and not a detail.
//
// A run started before an engine deploy and finished after it leaves its project file in the schema
// its own build wrote. The currently deployed binary refuses a file older than itself — that refusal
// is the whole of the v15 deadlock — so a settlement that asked the CONFIGURED path would never get
// the committed figure, and the hold would stay reserved with no sweep able to close it. The attempt
// records the path it ran with (row 139) and every question about that attempt goes back to it.
//
// Mutation caught: `s.Engine.Status(ctx, s.Cfg.EngineBinary, …)` in settle — and in bookMeter and
// maybeResync, which are the same question asked earlier.
func TestTheMoneyOfAFinishedRunIsReadWithTheBuildItRanWith(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
const pinned = "/opt/engine/2026.08.01/tmctl"
if l.EngineBinary != pinned {
t.Fatalf("the attempt was pinned to %q", l.EngineBinary)
}
// The deployment moves on while the run is still going, exactly as an engine upgrade does.
f.svc.Cfg.EngineBinary = "/opt/engine/2026.08.14/tmctl"
ends(t, f, l, exitedWith("0"))
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 100, Spend: usd(1_000_000), Reserved: usd(0)}, nil)
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
asked := f.engine.askedAs()
if len(asked) == 0 {
t.Fatal("the engine was never asked about the finished run")
}
for _, bin := range asked {
if bin != pinned {
t.Errorf("the engine was asked as %q; every question about this attempt goes to %q", bin, pinned)
}
}
// And the settlement actually happened, so the assertion above is about a path that ran.
if acct := f.account(t); acct.Reserved != 0 {
t.Errorf("the hold is still open: %s", acct.Reserved.USD())
}
}
// A ceiling the stream announced during THIS pass, a marker that does not parse — and a projection
// the reconciler cannot re-read. Nothing may be decided from that.
//
// The corrupt-marker branch is where it costs the most: it carries no exit code to say `ceiling`
// independently, and it CLOSES the run by design (PD-164), so there is no next pass to correct it.
// A re-read that answered "" on failure therefore turned an ordinary Postgres blip into a resumable
// ceiling halt recorded as `failed` with no reason at all. Found by the state-machine lens of the
// dofix review, on live Postgres.
//
// What the mutation moves is WHICH decision the reconciler refuses to make: with the database gone
// nothing can be written either way, so the assertion is on the refusal itself.
//
// Mutation caught: freshPausedReason falling back to the snapshot instead of reporting the failure.
func TestAnEndingIsNeverDecidedFromAReadThatFailed(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
journal := filepath.Join(f.workdir, ingest.JournalFile)
body := hello(t, f) +
`{"seq":2,"type":"ceiling","data":{"halted":true,"scope":"day"}}` + "\n"
if err := os.WriteFile(journal, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
// The drain of this very pass materializes the ceiling; the marker cannot say anything about it.
if _, err := f.svc.reconcile(f.ctx, l); err != nil {
t.Fatal(err)
}
path := f.svc.markerPath(l.RunID, l.AttemptNo)
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil {
t.Fatal(err)
}
// …and now the database goes away before the pass can establish what the projection says.
f.store.Pool().Close()
_, err := f.svc.reconcile(f.ctx, l)
if err == nil {
t.Fatal("a pass that could not read the run's projection reported success")
}
if !strings.Contains(err.Error(), "paused reason") {
t.Fatalf("the ending was decided past the read that failed: %v", err)
}
// The run is still there to be reconciled by the next pass — read with a connection this test did
// not break.
store, err := pgstore.Open(f.ctx, f.dsn)
if err != nil {
t.Fatal(err)
}
defer store.Close()
var finished *time.Time
if err := store.Pool().QueryRow(f.ctx, `select finished_at from runs where id = $1`, l.RunID).Scan(&finished); err != nil {
t.Fatal(err)
}
if finished != nil {
t.Errorf("the run was closed on a fact the pass could not establish (finished %v)", finished)
}
}
// engineScript writes a `tmctl` whose whole behaviour is two files the test rewrites between calls:
// what it prints and the code it exits with. Used where the property under test is a disposition of
// the SHELL contract, which the stub engine cannot have — it answers in Go values, on the far side
// of the exit code.
func engineScript(t *testing.T) (bin string, answers func(report string, code int)) {
t.Helper()
dir := t.TempDir()
report, code := filepath.Join(dir, "report.json"), filepath.Join(dir, "code")
bin = filepath.Join(dir, "tmctl")
script := "#!/bin/sh\ncat '" + report + "'\nexit \"$(cat '" + code + "')\"\n"
if err := os.WriteFile(bin, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
answers = func(r string, c int) {
t.Helper()
if err := os.WriteFile(report, []byte(r), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(code, []byte(strconv.Itoa(c)+"\n"), 0o600); err != nil {
t.Fatal(err)
}
}
answers(`{"committed_usd":0,"reserved_usd":0}`, 0)
return bin, answers
}
// The money of a book that flagged a unit, through the REAL engine channel and the real exit
// contract (acceptance dofix ФП-1).
//
// This is the ordinary path, not a corner: the engine exits 2 from `status --json` for any book with
// a flagged unit, and the platform asks that same command for the committed figure of every run that
// ends. Reading the code as a refusal deferred the settlement forever — the hold stayed reserved,
// the account could not start another run of that book, and nothing in the read model said why. The
// stub engine cannot catch that class at all, so this test runs a process.
//
// Mutation caught: any non-zero exit read as an error in runner.Status.
func TestTheMoneyOfAFlaggedBookIsSettledThroughTheRealExitContract(t *testing.T) {
f := newFixture(t, "10", 500)
bin, answers := engineScript(t)
f.svc.Engine = runner.New(nil)
f.svc.Cfg.EngineBinary = bin
l := spawned(t, f, 100) // the meter reads zero on a book that has never run
hold := f.svc.Pricing.Ceiling(100)
if got := f.account(t); got.Reserved != hold {
t.Fatalf("the hold was not taken: reserved %s", got.Reserved.USD())
}
// The run finished and one unit needs a human: exit 2 from the unit AND from every read of the
// book afterwards.
const spent = 1_250_000
answers(`{"book_id":"gu","total_units":10,"done":9,"flagged":1,"committed_usd":1.25,"reserved_usd":0}`, 2)
ends(t, f, l, exitedWith("2"))
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
if got := card(t, f); got.Status != "ready" {
t.Fatalf("a completed run with flagged units ended as %q", got.Status)
}
acct := f.account(t)
if acct.Reserved != 0 {
t.Fatalf("the hold of a flagged run is still open: %s", acct.Reserved.USD())
}
// Twice, from two independent facts: the balance row the account screen reads, and the ledger it
// is derived from. A settlement that wrote one without the other would leave the account able to
// spend money it has already spent.
granted, err := money.ParseUSD("10")
if err != nil {
t.Fatal(err)
}
if want := granted - spent; acct.Balance != want {
t.Errorf("balance %s, want %s", acct.Balance.USD(), want.USD())
}
var ledger money.MicroUSD
if err := f.store.Pool().QueryRow(f.ctx,
`select coalesce(sum(amount_micro_usd), 0) from credit_ledger where user_id = 'u1'`).Scan(&ledger); err != nil {
t.Fatal(err)
}
if ledger != acct.Balance {
t.Errorf("the ledger sums to %s and the balance says %s", ledger.USD(), acct.Balance.USD())
}
}
// Two generations of the sweep on one run, which is what an overlapping deploy produces: one pass
// closes it while the other is still holding a snapshot that says it is live.
//
// The stale pass must find NOTHING to do. Re-opening would clear `finished_at`, take a second hold
// and start an engine for a run whose owner has already been told it ended — the class of PD-181,
// reached through the branch this pack added (exit 5 with no recorded intent restarts a run).
//
// Mutation caught: dropping OnlyIfLive from the reconciler's RestartInput.
func TestAStalePassDoesNotResurrectARunAnotherPassHasFinished(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 30, Spend: usd(900_000), Reserved: usd(0)}, nil)
// The snapshot this pass will decide from — taken while the run was live and BEFORE its owner
// pressed stop. That is the whole window: everything below happens inside it.
stale := f.live(t)
// The user stops the run, and another generation of the sweep closes it as stopped.
if _, err := f.svc.Stop(f.ctx, "u1", l.RunID); err != nil {
t.Fatal(err)
}
ends(t, f, l, exitedWith("5"))
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
before := card(t, f)
if before.Status != "stopped" {
t.Fatalf("the first pass ended the run as %q", before.Status)
}
spawns := len(f.runner.starts())
// …and now the stale pass acts on its own snapshot. Driven through reconcile directly, because
// that is exactly what the second generation would call with the list it read before.
if _, err := f.svc.reconcile(f.ctx, stale); err != nil {
t.Fatalf("the stale pass failed instead of doing nothing: %v", err)
}
after := card(t, f)
if after.Status != before.Status || after.FinishedAt == nil {
t.Fatalf("a stale pass re-opened a run another pass had finished: %q → %q (finished: %v)",
before.Status, after.Status, after.FinishedAt)
}
if n := len(f.runner.starts()); n != spawns {
t.Errorf("the stale pass started %d more engines", n-spawns)
}
acct := f.account(t)
if acct.Reserved != 0 {
t.Errorf("the stale pass took a second hold: %s reserved", acct.Reserved.USD())
}
if acct.Balance != acct.LedgerSum {
t.Errorf("balance %s and ledger %s disagree", acct.Balance.USD(), acct.LedgerSum.USD())
}
}
// The refusal band: the engine was turned down before it did any work, so nothing was spent and the
// hold comes back. `failed` for the run — it cannot proceed — and never a restart, which would meet
// the same answer on every sweep for as long as the deployment stays broken.
func TestARefusedInvocationEndsTheRunAndGivesTheWholeHoldBack(t *testing.T) {
f := newFixture(t, "10", 500)
l := spawned(t, f, 100)
ends(t, f, l, exitedWith("10")) // the configuration will not load
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil)
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
if got := card(t, f); got.Status != "failed" {
t.Fatalf("a refused invocation ended as %q", got.Status)
}
if n := len(f.runner.starts()); n != 1 {
t.Errorf("%d units started: a refusal is reproducible, so a restart is a loop", n)
}
acct := f.account(t)
if acct.Reserved != 0 {
t.Errorf("the hold is still open after a refusal: %s", acct.Reserved.USD())
}
if acct.Balance != money.MicroUSD(10_000_000) {
t.Errorf("balance %s: a refusal spent nothing and must cost nothing", acct.Balance.USD())
}
}
// A run whose STREAM already reported a ceiling, whose unit then vanishes without a marker — a
// reboot in the seconds between the engine's last line and its exit.
//
// It must be closed as the pause it already is, and NOT restarted: the engine stopped on purpose and
// the next process would meet the same limit. The second half is the one the adversarial review of
// this pack found: the restart path's own pause hard-codes `credit_exhausted`, so a run stopped by
// the ENGINE's daily ceiling used to come back wearing the ACCOUNT's reason — which lights the
// account-level halted flag and re-opens the resume the day-ceiling guard exists to refuse.
//
// Mutation caught: removing the PausedReason branch from reconcile's no-marker path, and
// hard-coding PausedCreditExhausted in restart's exhausted branch.
func TestACeilingThatTheStreamReportedSurvivesAUnitThatVanished(t *testing.T) {
f := newFixture(t, "10", 500)
spawned(t, f, 100)
journal := filepath.Join(f.workdir, ingest.JournalFile)
body := hello(t, f) + `{"seq":2,"type":"ceiling","data":{"halted":true,"scope":"day"}}` + "\n"
if err := os.WriteFile(journal, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
// No marker at all, and long enough ago that the spawn grace has expired: the shape of a reboot.
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 5, Spend: usd(100_000), Reserved: usd(0)}, nil)
f.svc.Now = func() time.Time { return f.now.Add(2 * time.Hour) }
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
got := card(t, f)
if got.Status != "paused" || got.PausedReason != pgstore.PausedDailyCeiling {
t.Fatalf("the run ended as %q/%q, want paused/%s", got.Status, got.PausedReason, pgstore.PausedDailyCeiling)
}
if n := len(f.runner.starts()); n != 1 {
t.Errorf("%d units started: a run the engine stopped on purpose must not be restarted", n)
}
if acct := f.account(t); acct.Reserved != 0 {
t.Errorf("the hold is still open: %s", acct.Reserved.USD())
}
}
// The stream is named when the ATTEMPT ROW is written, not when its unit is — so there is no window
// in which a drain has no name of its own and adopts whatever handshake it meets. Between admission
// and spawn the reconciler really does drain (an admitted attempt is in ListLiveRuns), and that
// window is seconds of a `tmctl status` call or a whole sweep interval.
//
// Mutation caught: moving EngineStreamID back out of the attempt INSERTs into RecordSpawn.
func TestAnAttemptIsNamedBeforeAnythingCanBeSpawnedForIt(t *testing.T) {
f := newFixture(t, "10", 500)
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), CeilingChapters: 100})
if err != nil {
t.Fatal(err)
}
// Nothing has been spawned yet — this is exactly the window.
if n := len(f.runner.starts()); n != 0 {
t.Fatalf("%d units started before the assertion", n)
}
live := f.live(t)
if live.EngineRunID != pgstore.EngineStreamID(live.RunID, live.AttemptNo) {
t.Fatalf("an admitted attempt carries engine_run_id %q; a drain in this window would adopt a stranger's stream", live.EngineRunID)
}
// …and a restarted attempt is named the same way, in the same statement that creates it.
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
t.Fatal(err)
}
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 1, Spend: usd(10_000), Reserved: usd(0)}, nil)
ends(t, f, f.live(t), exitedWith("5"))
if err := f.svc.Sweep(f.ctx); err != nil {
t.Fatal(err)
}
next := f.live(t)
if next.AttemptNo != 2 {
t.Fatalf("the run was not restarted: attempt %d", next.AttemptNo)
}
if next.EngineRunID != pgstore.EngineStreamID(next.RunID, 2) {
t.Errorf("attempt 2 carries engine_run_id %q", next.EngineRunID)
}
}