package runs import ( "testing" "time" ) // settledAt is what the run row says about its own money. func settledAt(t *testing.T, f *fixture, runID string) *time.Time { t.Helper() var at *time.Time if err := f.store.Pool().QueryRow(f.ctx, `select settled_at from runs where id = $1`, runID).Scan(&at); err != nil { t.Fatal(err) } return at } func openReservations(t *testing.T, f *fixture, runID string) int { t.Helper() var n int if err := f.store.Pool().QueryRow(f.ctx, ` select count(*) from reservations where split_part(engine_run_id, '#', 1) = $1 and state = 'open'`, runID).Scan(&n); err != nil { t.Fatal(err) } return n } // ⛔ THE HALF-APPLIED STATE IN ITS PUREST FORM: two writes, one survivor, and no reader left to // notice. Settling closes the reservation in one transaction and stamps the run in a second // statement, so a process that dies between them leaves the ledger right and the column empty — and // nothing ever comes back to it, because the settlement worklist keys on an OPEN reservation and // this run has none. The run is out of every list that would look at it. // // The crash is simulated the only honest way available: by clearing the column AFTER a real // settlement, which is exactly the state such a crash leaves — the money closed, the mark absent. // What is asserted is that the sweep re-derives it. func TestASettledRunWhoseMarkNeverLandedIsRepairedByTheNextPass(t *testing.T) { f := newFixture(t, "10", 500) l := spawned(t, f, 100) ends(t, f, l, exitedWith("0")) if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } if settledAt(t, f, l.RunID) == nil { t.Fatal("the ordinary path did not stamp the run at all, so this test cannot be about repairing it") } if n := openReservations(t, f, l.RunID); n != 0 { t.Fatalf("%d reservations are still open after the settlement: the fixture is not in the state this "+ "test is about", n) } // The crash: the money is closed and the mark is gone. if _, err := f.store.Pool().Exec(f.ctx, `update runs set settled_at = null where id = $1`, l.RunID); err != nil { t.Fatal(err) } // The control that makes the repair mean something: this run is invisible to the worklist, so // nothing but the repair could possibly stamp it again. unsettled, err := f.store.UnsettledRuns(f.ctx, f.now) if err != nil { t.Fatal(err) } for _, u := range unsettled { if u.RunID == l.RunID { t.Fatalf("the run is still on the settlement worklist, so a later pass would settle it the ordinary "+ "way and this test would pass without any repair existing: %d entries", len(unsettled)) } } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } if at := settledAt(t, f, l.RunID); at == nil { t.Error("a run whose money had closed is still carrying no settled mark after a full sweep: the column " + "disagrees with the ledger, and nothing will ever come back to it") } } // The mirror, and it is what stops the repair from becoming a lie: a LIVE run must not be stamped. // The mark is a statement about the WHOLE run's money, and a run stamped here would claim a // resolution its own life contradicts — the next attempt has not spent anything yet. // // ⛔ TWO FIXTURES, AND THE SECOND IS THE ONE THAT MATTERS. The repair excludes a live run twice over: // because it holds an open reservation, and because its own life is not over. The first exclusion // hides the second — and this test asserted only the first until a mutation showed it: dropping the // «the run has finished» term left every assertion green, because the fixture's run also had a hold. // // The second fixture is the state where the terms come apart, and it is an ORDINARY one rather than a // corner: BETWEEN ATTEMPTS. The sweep settles the attempt that ended — closing its reservation — and // then reopens the run with a new one. In that window the run is live and holds nothing, and only // «has this run finished» keeps the repair off it. func TestALiveRunIsNeverStampedSettled(t *testing.T) { t.Run("while its hold is still open", func(t *testing.T) { f := newFixture(t, "10", 500) l := spawned(t, f, 100) f.runner.alive = true if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } if n := openReservations(t, f, l.RunID); n != 1 { t.Fatalf("the live run holds %d open reservations, want 1: the fixture is not in the state this "+ "test is about, so the assertion below would prove nothing", n) } if at := settledAt(t, f, l.RunID); at != nil { t.Errorf("a live run was stamped settled at %v while its hold is still open", at) } }) t.Run("between attempts, holding nothing at all", func(t *testing.T) { f := newFixture(t, "10", 500) l := spawned(t, f, 100) f.runner.alive = true // The window the sweep passes through on every restart: the ended attempt's reservation is // closed and the next one has not been opened. Written straight to the row because what is // under test is the repair's PREDICATE, not the road by which the state arises. if _, err := f.store.Pool().Exec(f.ctx, ` update reservations set state = 'settled', closed_at = now() where split_part(engine_run_id, '#', 1) = $1 and state = 'open'`, l.RunID); err != nil { t.Fatal(err) } if n := openReservations(t, f, l.RunID); n != 0 { t.Fatalf("%d reservations are still open: this fixture exists to have NONE, and with one the "+ "assertion below passes on the wrong term", n) } var finished *time.Time if err := f.store.Pool().QueryRow(f.ctx, `select finished_at from runs where id = $1`, l.RunID). Scan(&finished); err != nil { t.Fatal(err) } if finished != nil { t.Fatalf("the run has already finished at %v: the fixture is not «live and holding nothing»", finished) } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } if at := settledAt(t, f, l.RunID); at != nil { t.Errorf("a run that is still LIVE was stamped settled at %v because it happened to hold nothing: "+ "between two attempts every run looks like this, and the mark would claim its money was "+ "resolved while the next attempt has not spent anything yet", at) } }) }