411 lines
19 KiB
Go
411 lines
19 KiB
Go
package runs
|
|
|
|
import (
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"textmachine/platform/internal/money"
|
|
"textmachine/platform/internal/pgstore"
|
|
)
|
|
|
|
// abandon_orphan_test.go: the half of `PD-424` that stood open through two packs, and the durable
|
|
// half of `PD-418` that rode in with it.
|
|
//
|
|
// The population is a run that is LIVE by its own row, whose attempt still names an engine, and
|
|
// whose settlement the reconciler cannot compute. Pack P11 made it COUNTED, P12 made it VISIBLE on
|
|
// all three surfaces, and until now there was no handle: `run abandon` refused it — correctly,
|
|
// because a named unit is not a dead one — and the only exit was the user pressing Stop.
|
|
|
|
// The proof is the WHOLE of what opens the branch, and without it nothing changed. Written first
|
|
// because it is the property that must survive every later edit: an abandon that stopped requiring
|
|
// the proof would close runs over live engines, which is the one outcome worse than the stall.
|
|
//
|
|
// Mutation caught: making the orphan branch unconditional (dropping the ProcessGone check), or
|
|
// defaulting the field to true.
|
|
func TestALiveAttemptIsStillRefusedWithoutTheProofThatItsProcessIsGone(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
if _, err := f.store.AbandonRun(f.ctx, pgstore.AbandonOrder{
|
|
RunID: run.ID, Reason: "the host is gone", Now: f.now,
|
|
}); !errors.Is(err, pgstore.ErrRunMayHaveAProcess) {
|
|
t.Fatalf("a stalled live run with no proof answered %v, want the process refusal", err)
|
|
}
|
|
}
|
|
|
|
// The floor, and it is the same one the settlement branch waits for: seeing a run go wrong and being
|
|
// allowed to end it are different permissions. A run that has failed once is one the reconciler is
|
|
// still retrying, and writing it off a tick early destroys a run the sweep would have closed
|
|
// correctly — measured on the settlement branch at $1.234567 spent and $0.000000 charged.
|
|
//
|
|
// Mutation caught: dropping the failure floor from the orphan branch.
|
|
func TestAProvenAbandonIsStillRefusedBeforeTheReconcilerHasGivenUp(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set reconcile_failures = $2 where run_id = $1`, run.ID, StalledAfter-1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.AbandonRun(f.ctx, proven(t, f, run.ID, "systemd says it is gone")); !errors.Is(err, pgstore.ErrRunNotStalledYet) {
|
|
t.Fatalf("a proven abandon one failure below the floor answered %v, want ErrRunNotStalledYet", err)
|
|
}
|
|
}
|
|
|
|
// The handle itself, and the money it decides. The attempt reported a figure and carries a baseline,
|
|
// so what it demonstrably spent is CHARGED and only the rest comes back — strictly more honest than
|
|
// the whole-hold refund the settlement branch is forced into, because here there IS a figure.
|
|
//
|
|
// Mutation caught: releasing the hold whole in the orphan branch (the account is paid for work it
|
|
// received); charging the whole hold (it is billed for work nobody can show); leaving the run live.
|
|
func TestAProvenOrphanIsEndedAndChargedWhatTheEngineLastReported(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
// The stream's last word about this attempt: a baseline and a figure above it. Both columns are
|
|
// what the reconciler would have settled from, and the point of the handle is that it cannot.
|
|
var baseline int64
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select coalesce(spend_baseline_micro_usd, 0) from run_attempts where run_id = $1`, run.ID).
|
|
Scan(&baseline); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const attemptSpent = 40_000 // $0.04 of this attempt's own work
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set spend_micro_usd = $2 where run_id = $1`, run.ID, baseline+attemptSpent); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := f.account(t)
|
|
verdict, err := f.store.AbandonRun(f.ctx, proven(t, f, run.ID, "systemd says the unit is gone"))
|
|
if err != nil {
|
|
t.Fatalf("a proven orphan was not ended: %v", err)
|
|
}
|
|
if verdict != pgstore.AbandonedOrphan {
|
|
t.Errorf("verdict %q, want %q — the three verdicts mean different things to whoever typed the command",
|
|
verdict, pgstore.AbandonedOrphan)
|
|
}
|
|
after := f.account(t)
|
|
if after.Reserved != 0 {
|
|
t.Errorf("the hold is still open at %s: the orphan branch has no sweep to hand it to",
|
|
after.Reserved.USD())
|
|
}
|
|
// The hold was DEBITED when it was taken (credits.go: a hold moves the cached balance with the
|
|
// ledger row), so closing it credits the balance back and the charge comes off that.
|
|
if want := before.Balance + before.Reserved - money.MicroUSD(attemptSpent); after.Balance != want {
|
|
t.Errorf("balance %s, want %s — the attempt's own reported spend is charged and the rest comes back",
|
|
after.Balance.USD(), want.USD())
|
|
}
|
|
// The run is OVER on every surface the operator and the client read, and the book owes a reading
|
|
// surface like it does after every other ending: whatever was translated is bought and paid for.
|
|
var runStatus, bookStatus string
|
|
var finished, settled, ended *string
|
|
if err := f.store.Pool().QueryRow(f.ctx, `
|
|
select r.status, b.status, r.finished_at::text, r.settled_at::text,
|
|
(select a.ended_at::text from run_attempts a where a.run_id = r.id order by a.attempt_no desc limit 1)
|
|
from runs r join books b on b.id = r.book_id where r.id = $1`, run.ID).
|
|
Scan(&runStatus, &bookStatus, &finished, &settled, &ended); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if runStatus != "failed" || bookStatus != "failed" {
|
|
t.Errorf("run %q, book %q — an abandoned run and its book must not give a client two answers",
|
|
runStatus, bookStatus)
|
|
}
|
|
if finished == nil || ended == nil {
|
|
t.Error("the run or its attempt is still open after the verdict")
|
|
}
|
|
if settled == nil {
|
|
t.Error("the run's money was closed and `settled_at` was not stamped: the settlement list would " +
|
|
"never see it again and the run would read unsettled for good")
|
|
}
|
|
}
|
|
|
|
// An attempt with NO baseline is the one nothing can price, and the handle says so with money rather
|
|
// than with a guess: the hold comes back whole. It is the settlement branch's own reasoning arriving
|
|
// at the live half — there is no figure this platform could justify charging.
|
|
func TestAProvenOrphanWithNothingToPriceGivesTheHoldBackWhole(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
// A spawned attempt from before the baseline column: the unit's name is the only evidence, and
|
|
// it is the shape `settle` refuses to price at all.
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set spend_baseline_micro_usd = null where run_id = $1`, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := f.account(t)
|
|
verdict, err := f.store.AbandonRun(f.ctx, proven(t, f, run.ID, "systemd says the unit is gone"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if verdict != pgstore.AbandonedOrphan {
|
|
t.Errorf("verdict %q, want %q", verdict, pgstore.AbandonedOrphan)
|
|
}
|
|
after := f.account(t)
|
|
if after.Reserved != 0 {
|
|
t.Errorf("the hold is still open at %s", after.Reserved.USD())
|
|
}
|
|
if after.Balance != before.Balance+before.Reserved {
|
|
t.Errorf("balance %s, want %s — an attempt nothing can price gets its hold back WHOLE",
|
|
after.Balance.USD(), (before.Balance + before.Reserved).USD())
|
|
}
|
|
}
|
|
|
|
// `PD-418`, and it is the same query answering a different question. A run that is LIVE can carry an
|
|
// ENDED attempt whose reservation never closed, and every terminal path used to ask
|
|
// `runs.finished_at` — which says nothing about that. Branching on the PRESENCE of the orphan is the
|
|
// durable cure both rows name.
|
|
//
|
|
// ⚠ The live attempt is NOT touched and the run is NOT ended: what was stuck is one attempt's money,
|
|
// and ending a translation over it would take work away from a user who did not ask for that.
|
|
//
|
|
// Mutation caught: restoring `r.finished_at is not null` to the orphan query (the row becomes
|
|
// unreachable again); stamping `settled_at` on a live run (the settlement list stops seeing the
|
|
// attempt whose money is still open).
|
|
func TestALiveRunsOrphanedAttemptIsSettledWithoutEndingTheRun(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The shape `PD-418` describes and today's code cannot produce on its own: an attempt that ENDED
|
|
// with its reservation open, under a run that is still live. Written directly, because the whole
|
|
// row exists to say what happens if the invariant that keeps it unreachable ever loosens.
|
|
if _, err := f.store.Pool().Exec(f.ctx, `
|
|
update run_attempts set ended_at = $2, reconcile_failures = $3 where run_id = $1`,
|
|
run.ID, f.now, StalledAfter); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := f.account(t)
|
|
if before.Reserved == 0 {
|
|
t.Fatal("the fixture holds nothing: it cannot observe a settlement")
|
|
}
|
|
verdict, err := f.store.AbandonRun(f.ctx, pgstore.AbandonOrder{
|
|
RunID: run.ID, Reason: "its settlement will never close", Now: f.now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("a live run's orphaned attempt could not be given up on: %v", err)
|
|
}
|
|
if verdict != pgstore.AbandonedSettlement {
|
|
t.Errorf("verdict %q, want %q: what was ended is an attempt's money, not the run",
|
|
verdict, pgstore.AbandonedSettlement)
|
|
}
|
|
after := f.account(t)
|
|
if after.Reserved != 0 {
|
|
t.Errorf("the orphaned hold is still open at %s", after.Reserved.USD())
|
|
}
|
|
var finished, settled *string
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select finished_at::text, settled_at::text from runs where id = $1`, run.ID).
|
|
Scan(&finished, &settled); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if finished != nil {
|
|
t.Error("the run was ended over an orphaned attempt's money; the user did not ask for that")
|
|
}
|
|
if settled != nil {
|
|
t.Error("a LIVE run was stamped settled: the settlement list would stop seeing the attempt " +
|
|
"whose money is still the reconciler's to close")
|
|
}
|
|
}
|
|
|
|
// stallTheReconciler puts a run in the population the operator is told about: reconciliation has
|
|
// failed the threshold number of times running. It writes the column directly because producing the
|
|
// failures for real needs an engine that cannot be asked, which is the state under test rather than
|
|
// a fixture this test could build.
|
|
func stallTheReconciler(t *testing.T, f *fixture, runID string) {
|
|
t.Helper()
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set reconcile_failures = $2 where run_id = $1`, runID, StalledAfter); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// The write-off is CAPPED at the hold, and the cap is a money guard rather than arithmetic hygiene.
|
|
//
|
|
// `spend_micro_usd` is what the engine's stream last reported for the attempt, and a run whose
|
|
// journal ran ahead of its settlement can report more than the account ever agreed to hold. Charging
|
|
// that would bill an account past its own consent — the one thing the hold exists to make
|
|
// impossible — so the surplus is the DEPLOYMENT's, exactly as the ordinary settlement decides it
|
|
// (`pgstore.Settle`, «capped at the hold»).
|
|
//
|
|
// Mutation caught: dropping the `spent > held` branch in the orphan write-off.
|
|
func TestTheWriteOffNeverChargesMoreThanTheAccountAgreedToHold(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
before := f.account(t)
|
|
if before.Reserved == 0 {
|
|
t.Fatal("the fixture holds nothing: it cannot observe a cap")
|
|
}
|
|
// The stream reports FAR more than the hold — the shape a journal that ran ahead of its
|
|
// settlement produces.
|
|
var baseline int64
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select coalesce(spend_baseline_micro_usd, 0) from run_attempts where run_id = $1`, run.ID).
|
|
Scan(&baseline); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set spend_micro_usd = $2 where run_id = $1`,
|
|
run.ID, baseline+int64(before.Reserved)*7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.AbandonRun(f.ctx, proven(t, f, run.ID, "systemd says the unit is gone")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after := f.account(t)
|
|
if after.Reserved != 0 {
|
|
t.Errorf("the hold is still open at %s", after.Reserved.USD())
|
|
}
|
|
// The account paid its hold and not a micro-dollar more; the surplus is the deployment's.
|
|
if after.Balance != before.Balance {
|
|
t.Errorf("balance %s, want %s — charging past the hold bills an account beyond its own consent",
|
|
after.Balance.USD(), before.Balance.USD())
|
|
}
|
|
}
|
|
|
|
// proven builds the order `run abandon` presents: the verdict plus the proof's subject and
|
|
// generation, read the way the command reads them (askSystemd → StalledRuns) rather than filled in by
|
|
// hand.
|
|
func proven(t *testing.T, f *fixture, runID, reason string) pgstore.AbandonOrder {
|
|
t.Helper()
|
|
list, err := f.store.StalledRuns(f.ctx, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, r := range list {
|
|
if r.RunID == runID && !r.Settling {
|
|
return pgstore.AbandonOrder{
|
|
RunID: runID, Reason: reason, ProcessGone: true, Now: f.now,
|
|
ProofAttemptID: r.AttemptID, ProofSpawns: r.Spawns,
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("run %s is not a live row this proof could be about", runID)
|
|
return pgstore.AbandonOrder{}
|
|
}
|
|
|
|
// A proof about one attempt is not a proof about another (PD-424, first of two windows). The
|
|
// settlement that wedged the run unblocks, `restart` opens a new attempt and spawns it; the write
|
|
// reads "the live attempt", so without the subject comparison a proof about attempt N is spent on
|
|
// attempt N+1 — an engine mid-call with its hold closed underneath it.
|
|
func TestAProofAboutTheAttemptBeforeThisOneIsRefused(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
order := proven(t, f, run.ID, "systemd says the unit is gone")
|
|
// The run opens its next attempt while the operator is talking to systemd. Written here through
|
|
// the columns rather than through `restart`, because what is being pinned is the WRITE's reaction
|
|
// to a moved world and not the reconciler's route to moving it.
|
|
//
|
|
// ⚠ The old attempt's money is CLOSED as `restart` closes it. Left open it would make attempt 1 an
|
|
// orphaned settlement, and `AbandonRun` answers that branch first and never reaches the live one —
|
|
// so the test would pass for a reason that has nothing to do with what it is named after.
|
|
if _, err := f.store.Pool().Exec(f.ctx,
|
|
`update run_attempts set ended_at = now() where run_id = $1 and ended_at is null`, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.store.Release(f.ctx, pgstore.ReservationKey(run.ID, 1), f.now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
//
|
|
// The new attempt is given the proof's own claim count on purpose: left at its default, the SPAWN
|
|
// comparison catches this case too, and the test then passes with the attempt comparison deleted.
|
|
if _, err := f.store.Pool().Exec(f.ctx, `
|
|
insert into run_attempts (run_id, attempt_no, started_at, unit_name, reconcile_failures, spawns)
|
|
values ($1, 2, now(), 'tm-run-'||$1||'-2', $2, $3)`, run.ID, StalledAfter, order.ProofSpawns); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = f.store.AbandonRun(f.ctx, order)
|
|
if !errors.Is(err, pgstore.ErrProofOvertaken) {
|
|
t.Fatalf("an abandon proved against the PREVIOUS attempt answered %v, want ErrProofOvertaken: "+
|
|
"the run now has a live attempt the operator never asked systemd about", err)
|
|
}
|
|
// Which refusal fired, not merely that one did: both comparisons answer the same sentinel.
|
|
if !strings.Contains(err.Error(), "the proof was about attempt") {
|
|
t.Fatalf("the refusal came from a guard other than the attempt's identity: %v", err)
|
|
}
|
|
}
|
|
|
|
// The second window, and the one no name could witness. ReleaseSpawnClaim returns `unit_name` to NULL
|
|
// when a unit could not be created — which is not "was not created": a `systemd-run` killed after it
|
|
// had already asked leaves an engine running. So the attempt can be claimed again, and afterwards the
|
|
// name reads exactly as the proof saw it. The claim counter does not.
|
|
func TestAnAttemptClaimedWhileSystemdWasBeingAskedIsRefused(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stallTheReconciler(t, f, run.ID)
|
|
order := proven(t, f, run.ID, "systemd says the unit is gone")
|
|
// The unit could not be created, so the claim went back and the sweep took it again — through the
|
|
// real calls, so what is pinned is that the pair leaves the name alone and the counter one higher.
|
|
var attemptID int64
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select id from run_attempts where run_id = $1 and ended_at is null`, run.ID).Scan(&attemptID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := proven(t, f, run.ID, "x")
|
|
if err := f.store.ReleaseSpawnClaim(f.ctx, attemptID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := f.store.RecordSpawn(f.ctx, pgstore.SpawnRecord{
|
|
AttemptID: attemptID, Unit: "tm-run-" + run.ID + "-1", Binary: "/opt/tmctl",
|
|
EngineRunID: run.ID + "#1", Ceiling: money.MicroUSD(1000), CeilingArg: money.MicroUSD(1000),
|
|
})
|
|
if err != nil || !claimed {
|
|
t.Fatalf("the re-claim did not happen (claimed=%v err=%v), so this test proves nothing", claimed, err)
|
|
}
|
|
after := proven(t, f, run.ID, "x")
|
|
if after.ProofSpawns != before.ProofSpawns+1 {
|
|
t.Fatalf("the claim counter went %d → %d: a witness that does not move cannot catch the race",
|
|
before.ProofSpawns, after.ProofSpawns)
|
|
}
|
|
if _, err := f.store.AbandonRun(f.ctx, order); !errors.Is(err, pgstore.ErrProofOvertaken) {
|
|
t.Fatalf("an abandon proved before the attempt was claimed again answered %v, want ErrProofOvertaken: "+
|
|
"the unit name reads exactly as the proof saw it, and an engine may be running", err)
|
|
}
|
|
}
|