305 lines
13 KiB
Go
305 lines
13 KiB
Go
package runs
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
|
|
"textmachine/platform/internal/money"
|
|
"textmachine/platform/internal/pgstore"
|
|
)
|
|
|
|
// abandon_orphan_test.go: the half of `PD-424` that stood open through two packs, and the durable
|
|
// half of `PD-418` that rode in with it.
|
|
//
|
|
// The population is a run that is LIVE by its own row, whose attempt still names an engine, and
|
|
// whose settlement the reconciler cannot compute. Pack P11 made it COUNTED, P12 made it VISIBLE on
|
|
// all three surfaces, and until now there was no handle: `run abandon` refused it — correctly,
|
|
// because a named unit is not a dead one — and the only exit was the user pressing Stop.
|
|
|
|
// The proof is the WHOLE of what opens the branch, and without it nothing changed. Written first
|
|
// because it is the property that must survive every later edit: an abandon that stopped requiring
|
|
// the proof would close runs over live engines, which is the one outcome worse than the stall.
|
|
//
|
|
// Mutation caught: making the orphan branch unconditional (dropping the ProcessGone check), or
|
|
// defaulting the field to true.
|
|
func TestALiveAttemptIsStillRefusedWithoutTheProofThatItsProcessIsGone(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
if _, err := f.store.AbandonRun(f.ctx, pgstore.AbandonOrder{
|
|
RunID: run.ID, Reason: "the host is gone", Now: f.now,
|
|
}); !errors.Is(err, pgstore.ErrRunMayHaveAProcess) {
|
|
t.Fatalf("a stalled live run with no proof answered %v, want the process refusal", err)
|
|
}
|
|
}
|
|
|
|
// The floor, and it is the same one the settlement branch waits for: seeing a run go wrong and being
|
|
// allowed to end it are different permissions. A run that has failed once is one the reconciler is
|
|
// still retrying, and writing it off a tick early destroys a run the sweep would have closed
|
|
// correctly — measured on the settlement branch at $1.234567 spent and $0.000000 charged.
|
|
//
|
|
// Mutation caught: dropping the failure floor from the orphan branch.
|
|
func TestAProvenAbandonIsStillRefusedBeforeTheReconcilerHasGivenUp(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set reconcile_failures = $2 where run_id = $1`, run.ID, StalledAfter-1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.AbandonRun(f.ctx, pgstore.AbandonOrder{
|
|
RunID: run.ID, Reason: "systemd says it is gone", ProcessGone: true, Now: f.now,
|
|
}); !errors.Is(err, pgstore.ErrRunNotStalledYet) {
|
|
t.Fatalf("a proven abandon one failure below the floor answered %v, want ErrRunNotStalledYet", err)
|
|
}
|
|
}
|
|
|
|
// The handle itself, and the money it decides. The attempt reported a figure and carries a baseline,
|
|
// so what it demonstrably spent is CHARGED and only the rest comes back — strictly more honest than
|
|
// the whole-hold refund the settlement branch is forced into, because here there IS a figure.
|
|
//
|
|
// Mutation caught: releasing the hold whole in the orphan branch (the account is paid for work it
|
|
// received); charging the whole hold (it is billed for work nobody can show); leaving the run live.
|
|
func TestAProvenOrphanIsEndedAndChargedWhatTheEngineLastReported(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
// The stream's last word about this attempt: a baseline and a figure above it. Both columns are
|
|
// what the reconciler would have settled from, and the point of the handle is that it cannot.
|
|
var baseline int64
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select coalesce(spend_baseline_micro_usd, 0) from run_attempts where run_id = $1`, run.ID).
|
|
Scan(&baseline); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const attemptSpent = 40_000 // $0.04 of this attempt's own work
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set spend_micro_usd = $2 where run_id = $1`, run.ID, baseline+attemptSpent); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := f.account(t)
|
|
verdict, err := f.store.AbandonRun(f.ctx, pgstore.AbandonOrder{
|
|
RunID: run.ID, Reason: "systemd says the unit is gone", ProcessGone: true, Now: f.now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("a proven orphan was not ended: %v", err)
|
|
}
|
|
if verdict != pgstore.AbandonedOrphan {
|
|
t.Errorf("verdict %q, want %q — the three verdicts mean different things to whoever typed the command",
|
|
verdict, pgstore.AbandonedOrphan)
|
|
}
|
|
after := f.account(t)
|
|
if after.Reserved != 0 {
|
|
t.Errorf("the hold is still open at %s: the orphan branch has no sweep to hand it to",
|
|
after.Reserved.USD())
|
|
}
|
|
// The hold was DEBITED when it was taken (credits.go: a hold moves the cached balance with the
|
|
// ledger row), so closing it credits the balance back and the charge comes off that.
|
|
if want := before.Balance + before.Reserved - money.MicroUSD(attemptSpent); after.Balance != want {
|
|
t.Errorf("balance %s, want %s — the attempt's own reported spend is charged and the rest comes back",
|
|
after.Balance.USD(), want.USD())
|
|
}
|
|
// The run is OVER on every surface the operator and the client read, and the book owes a reading
|
|
// surface like it does after every other ending: whatever was translated is bought and paid for.
|
|
var runStatus, bookStatus string
|
|
var finished, settled, ended *string
|
|
if err := f.store.Pool().QueryRow(f.ctx, `
|
|
select r.status, b.status, r.finished_at::text, r.settled_at::text,
|
|
(select a.ended_at::text from run_attempts a where a.run_id = r.id order by a.attempt_no desc limit 1)
|
|
from runs r join books b on b.id = r.book_id where r.id = $1`, run.ID).
|
|
Scan(&runStatus, &bookStatus, &finished, &settled, &ended); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if runStatus != "failed" || bookStatus != "failed" {
|
|
t.Errorf("run %q, book %q — an abandoned run and its book must not give a client two answers",
|
|
runStatus, bookStatus)
|
|
}
|
|
if finished == nil || ended == nil {
|
|
t.Error("the run or its attempt is still open after the verdict")
|
|
}
|
|
if settled == nil {
|
|
t.Error("the run's money was closed and `settled_at` was not stamped: the settlement list would " +
|
|
"never see it again and the run would read unsettled for good")
|
|
}
|
|
}
|
|
|
|
// An attempt with NO baseline is the one nothing can price, and the handle says so with money rather
|
|
// than with a guess: the hold comes back whole. It is the settlement branch's own reasoning arriving
|
|
// at the live half — there is no figure this platform could justify charging.
|
|
func TestAProvenOrphanWithNothingToPriceGivesTheHoldBackWhole(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
// A spawned attempt from before the baseline column: the unit's name is the only evidence, and
|
|
// it is the shape `settle` refuses to price at all.
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set spend_baseline_micro_usd = null where run_id = $1`, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := f.account(t)
|
|
verdict, err := f.store.AbandonRun(f.ctx, pgstore.AbandonOrder{
|
|
RunID: run.ID, Reason: "systemd says the unit is gone", ProcessGone: true, Now: f.now,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if verdict != pgstore.AbandonedOrphan {
|
|
t.Errorf("verdict %q, want %q", verdict, pgstore.AbandonedOrphan)
|
|
}
|
|
after := f.account(t)
|
|
if after.Reserved != 0 {
|
|
t.Errorf("the hold is still open at %s", after.Reserved.USD())
|
|
}
|
|
if after.Balance != before.Balance+before.Reserved {
|
|
t.Errorf("balance %s, want %s — an attempt nothing can price gets its hold back WHOLE",
|
|
after.Balance.USD(), (before.Balance + before.Reserved).USD())
|
|
}
|
|
}
|
|
|
|
// `PD-418`, and it is the same query answering a different question. A run that is LIVE can carry an
|
|
// ENDED attempt whose reservation never closed, and every terminal path used to ask
|
|
// `runs.finished_at` — which says nothing about that. Branching on the PRESENCE of the orphan is the
|
|
// durable cure both rows name.
|
|
//
|
|
// ⚠ The live attempt is NOT touched and the run is NOT ended: what was stuck is one attempt's money,
|
|
// and ending a translation over it would take work away from a user who did not ask for that.
|
|
//
|
|
// Mutation caught: restoring `r.finished_at is not null` to the orphan query (the row becomes
|
|
// unreachable again); stamping `settled_at` on a live run (the settlement list stops seeing the
|
|
// attempt whose money is still open).
|
|
func TestALiveRunsOrphanedAttemptIsSettledWithoutEndingTheRun(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The shape `PD-418` describes and today's code cannot produce on its own: an attempt that ENDED
|
|
// with its reservation open, under a run that is still live. Written directly, because the whole
|
|
// row exists to say what happens if the invariant that keeps it unreachable ever loosens.
|
|
if _, err := f.store.Pool().Exec(f.ctx, `
|
|
update run_attempts set ended_at = $2, reconcile_failures = $3 where run_id = $1`,
|
|
run.ID, f.now, StalledAfter); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := f.account(t)
|
|
if before.Reserved == 0 {
|
|
t.Fatal("the fixture holds nothing: it cannot observe a settlement")
|
|
}
|
|
verdict, err := f.store.AbandonRun(f.ctx, pgstore.AbandonOrder{
|
|
RunID: run.ID, Reason: "its settlement will never close", Now: f.now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("a live run's orphaned attempt could not be given up on: %v", err)
|
|
}
|
|
if verdict != pgstore.AbandonedSettlement {
|
|
t.Errorf("verdict %q, want %q: what was ended is an attempt's money, not the run",
|
|
verdict, pgstore.AbandonedSettlement)
|
|
}
|
|
after := f.account(t)
|
|
if after.Reserved != 0 {
|
|
t.Errorf("the orphaned hold is still open at %s", after.Reserved.USD())
|
|
}
|
|
var finished, settled *string
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select finished_at::text, settled_at::text from runs where id = $1`, run.ID).
|
|
Scan(&finished, &settled); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if finished != nil {
|
|
t.Error("the run was ended over an orphaned attempt's money; the user did not ask for that")
|
|
}
|
|
if settled != nil {
|
|
t.Error("a LIVE run was stamped settled: the settlement list would stop seeing the attempt " +
|
|
"whose money is still the reconciler's to close")
|
|
}
|
|
}
|
|
|
|
// stallTheReconciler puts a run in the population the operator is told about: reconciliation has
|
|
// failed the threshold number of times running. It writes the column directly because producing the
|
|
// failures for real needs an engine that cannot be asked, which is the state under test rather than
|
|
// a fixture this test could build.
|
|
func stallTheReconciler(t *testing.T, f *fixture, runID string) {
|
|
t.Helper()
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set reconcile_failures = $2 where run_id = $1`, runID, StalledAfter); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// The write-off is CAPPED at the hold, and the cap is a money guard rather than arithmetic hygiene.
|
|
//
|
|
// `spend_micro_usd` is what the engine's stream last reported for the attempt, and a run whose
|
|
// journal ran ahead of its settlement can report more than the account ever agreed to hold. Charging
|
|
// that would bill an account past its own consent — the one thing the hold exists to make
|
|
// impossible — so the surplus is the DEPLOYMENT's, exactly as the ordinary settlement decides it
|
|
// (`pgstore.Settle`, «capped at the hold»).
|
|
//
|
|
// Mutation caught: dropping the `spent > held` branch in the orphan write-off.
|
|
func TestTheWriteOffNeverChargesMoreThanTheAccountAgreedToHold(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
before := f.account(t)
|
|
if before.Reserved == 0 {
|
|
t.Fatal("the fixture holds nothing: it cannot observe a cap")
|
|
}
|
|
// The stream reports FAR more than the hold — the shape a journal that ran ahead of its
|
|
// settlement produces.
|
|
var baseline int64
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select coalesce(spend_baseline_micro_usd, 0) from run_attempts where run_id = $1`, run.ID).
|
|
Scan(&baseline); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set spend_micro_usd = $2 where run_id = $1`,
|
|
run.ID, baseline+int64(before.Reserved)*7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.AbandonRun(f.ctx, pgstore.AbandonOrder{
|
|
RunID: run.ID, Reason: "systemd says the unit is gone", ProcessGone: true, Now: f.now,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after := f.account(t)
|
|
if after.Reserved != 0 {
|
|
t.Errorf("the hold is still open at %s", after.Reserved.USD())
|
|
}
|
|
// The account paid its hold and not a micro-dollar more; the surplus is the deployment's.
|
|
if after.Balance != before.Balance {
|
|
t.Errorf("balance %s, want %s — charging past the hold bills an account beyond its own consent",
|
|
after.Balance.USD(), before.Balance.USD())
|
|
}
|
|
}
|