package runs import ( "bytes" "encoding/json" "log/slog" "strings" "testing" "time" "textmachine/platform/internal/ingest" "textmachine/platform/internal/money" "textmachine/platform/internal/pgstore" "textmachine/platform/internal/runner" ) // ⛔ THE RESTART LOOP HAS A FLOOR, AND THE MONEY IS CLOSED WHEN IT IS REACHED. // // A unit that ends with no marker is restarted (unified backlog row 398), and where the marker // machinery itself is broken every replacement ends the same way. The loop is money-NEUTRAL per turn // — the old hold comes back whole and an equal one is taken — so no balance ever stops it, and every // turn writes an attempt row and three ledger rows. // // Three things are asserted together because a cure for any one of them alone would be a defect: the // platform STOPS (no further attempt, no further unit), the last attempt's hold is CLOSED rather than // frozen (refusing before the settlement would trade an unbounded loop for a stranded reservation), // and the ending does not tell the client that retrying is the remedy after this platform retried and // gave up. func TestTheRestartLoopStopsAtTheCapWithTheMoneyClosedAndNoNewHold(t *testing.T) { f := newFixture(t, "10", 500) f.svc.Cfg.MaxAttempts = 3 run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } // The shape the row describes: the engine spends nothing, the unit is gone and no marker is ever // written, so every pass reads the same three facts and reaches the same decision. f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil) f.runner.alive = false // Two passes restart, the third meets the cap. The clock moves because a restart stamps the new // attempt with `now` and the spawn grace has to be behind it before the next pass may decide. for i := 1; i <= 3; i++ { f.svc.Now = func() time.Time { return f.now.Add(time.Duration(i) * time.Hour) } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatalf("pass %d: %v", i, err) } } _, card, err := f.store.GetBook(f.ctx, "u1", f.bookID(t)) if err != nil { t.Fatal(err) } if card == nil || card.Status != "failed" || card.FinishedAt == nil { t.Fatalf("the capped run did not end: %+v", card) } // NOT `interrupted`: that word promises a client the retry is the remedy, and this platform has // just performed that retry as many times as the deployment allows. if card.FailureReason != "service_error" { t.Errorf("the capped run is reported as %q, want service_error", card.FailureReason) } var attempts int if err := f.store.Pool().QueryRow(f.ctx, `select count(*) from run_attempts where run_id = $1`, run.ID).Scan(&attempts); err != nil { t.Fatal(err) } if attempts != 3 { t.Errorf("%d attempt rows, want 3: the cap is the number of attempts one run may have", attempts) } if n := len(f.runner.starts()); n != 3 { t.Errorf("%d units started, want 3: a capped run must not be spawned again", n) } // The platform's OWN word, on the attempt it stopped at. It is what tells an operator "we stopped // trying" from "the run kept dying", and systemd cannot produce it. var result string if err := f.store.Pool().QueryRow(f.ctx, `select coalesce(exit_result, '') from run_attempts where run_id = $1 order by attempt_no desc limit 1`, run.ID).Scan(&result); err != nil { t.Fatal(err) } if result != runner.AttemptsExhaustedResult { t.Errorf("the last attempt's exit_result is %q, want %q", result, runner.AttemptsExhaustedResult) } acct := f.account(t) if acct.Reserved != 0 { t.Errorf("the capped run left %s reserved: refusing the restart before the settlement would "+ "trade an unbounded loop for a frozen hold", acct.Reserved.USD()) } if acct.Balance != acct.LedgerSum { t.Errorf("balance %s and ledger %s disagree after the cap", acct.Balance.USD(), acct.LedgerSum.USD()) } // Nothing was spent, so the whole hold came back. if want := money.MicroUSD(10_000_000); acct.Balance != want { t.Errorf("balance %s, want %s — the capped run spent nothing and its hold is not kept", acct.Balance.USD(), want.USD()) } } // ⚠ THE CONTROL THAT KEEPS THE PIN ABOVE FROM MEASURING NOTHING: the SAME fixture, one attempt below // the cap, is still restarted. // // It is also the pin on the choice unified backlog row 396 asked to be re-decided rather than // inherited: an ending nobody recorded an intent for is an INTERRUPTION and comes back, because // reading a reboot as a user's stop would leave every run on a restarted host dead with its budget // unspent (row 138). What the cap changes is not that answer but its price — "one restart of a run // whose owner never asked for it" is now a bounded sentence rather than a loop. func TestBelowTheCapTheSameRunIsStillRestarted(t *testing.T) { f := newFixture(t, "10", 500) f.svc.Cfg.MaxAttempts = 5 run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil) f.runner.alive = false for i := 1; i <= 3; i++ { f.svc.Now = func() time.Time { return f.now.Add(time.Duration(i) * time.Hour) } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatalf("pass %d: %v", i, err) } } live := f.live(t) if live.AttemptNo != 4 { t.Fatalf("attempt %d after three passes under a cap of 5: the run should still be coming back", live.AttemptNo) } if n := len(f.runner.starts()); n != 4 { t.Errorf("%d units started, want 4", n) } } // ⛔ THE CAP IS NOT THE COUNTER A RESTART RESETS, and this test asserts the PREMISE rather than the // consequence — without it the pin above would pass for a cap keyed on the wrong number and nobody // would learn anything. // // `reconcile_failures` is the obvious place to count restarts and it is the wrong one: it lives on // the ATTEMPT, and a restart opens a new attempt row (pgstore/runs.go, `next := attemptNo + 1`), so // it is zero on every turn of this loop by construction. That is measured here — at the very pass // that meets the cap, the counter a cap might have been built on reads zero — which is why a cap // expressed in it would never fire and the defect would stand, wearing the look of a cure (the // alarm's own threshold, StalledAfter, is unreachable for exactly this reason). func TestTheCapIsNotTheCounterARestartResets(t *testing.T) { f := newFixture(t, "10", 500) f.svc.Cfg.MaxAttempts = 3 run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil) f.runner.alive = false // Two passes: the run is now standing at the LAST attempt the cap allows, and the next pass is // the one that ends it. for i := 1; i <= 2; i++ { f.svc.Now = func() time.Time { return f.now.Add(time.Duration(i) * time.Hour) } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatalf("pass %d: %v", i, err) } } live := f.live(t) if live.AttemptNo != 3 { t.Fatalf("attempt %d, want 3: this test has to stand at the cap for its measurement to mean anything", live.AttemptNo) } if live.ReconcileFailures != 0 { t.Fatalf("the deferral counter reads %d at the attempt the cap ends: this fixture no longer "+ "demonstrates that a cap keyed on that counter measures nothing", live.ReconcileFailures) } if live.AttemptNo < f.svc.maxAttempts() { t.Fatalf("attempt %d is below the cap %d: the premise of this test is that the NEXT decision is the cap's", live.AttemptNo, f.svc.maxAttempts()) } } // The ending a cap writes must not be read as a retryable interruption — and the neighbours it sits // beside must not move, which is what the table's other rows are for. // // ⚠ BOTH SHAPES OF THE CAP'S ENDING ARE HERE, because the two call sites of the restart hand over // different evidence: one has a marker the unit left (the code travels, the result word is ours), the // other has none at all. A fix that covered one of them would look complete and be half. func TestAnExhaustedRunIsNotCalledARetryableInterruption(t *testing.T) { at := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC) for _, tc := range []struct { name string marker runner.Marker want string }{ // No marker was ever written: the word this platform would have recorded is its own either // way, and the cap's is the more informative one. {"the cap, with nothing the unit left", runner.Marker{Unit: "u", Result: runner.AttemptsExhaustedResult}, "service_error"}, // The unit left a caught signal. The code travels to the client as it always did and the // answer is the same, so the operator's column keeps both facts. {"the cap, over a unit that exited 5", runner.Marker{Unit: "u", Result: runner.AttemptsExhaustedResult, Code: "exited", Status: "5", At: at}, "service_error"}, // The controls: an ending nobody described is still the one where retrying IS the remedy, and // our own grace is still not a broken deployment (gracekill_test owns that pair). {"a unit that simply vanished, below the cap", runner.Marker{Unit: "u", Result: runner.UnitVanishedResult}, "interrupted"}, {"our own stop deadline", runner.Marker{Unit: "u", Result: "timeout", Code: "killed", Status: "KILL", At: at}, "interrupted"}, } { t.Run(tc.name, func(t *testing.T) { status, _, exit := outcome(pgstore.LiveRun{RunID: "r"}, tc.marker) if status != "failed" { t.Fatalf("this ending came out %q, not failed: the test is about the reason a FAILED run "+ "carries, so it now measures nothing", status) } if got := failureReason(status, exit, tc.marker); got != tc.want { t.Errorf("reported to the client as %q, want %q", got, tc.want) } }) } } // ⚠ THE LIMIT IS VISIBLE BEFORE IT IS REACHED, which the pack made a condition of the cap and not a // nicety: a platform that stops restarting without having said how close it was leaves an operator to // discover the wall by arriving at it. // // The cap rides EVERY restart line rather than a threshold line of its own, and that is deliberate — // a second place where the number is written is a second place for it to go stale against the first. func TestEveryRestartSaysHowCloseTheCapIs(t *testing.T) { f := newFixture(t, "10", 500) f.svc.Cfg.MaxAttempts = 7 var buf bytes.Buffer f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil) f.runner.alive = false f.svc.Now = func() time.Time { return f.now.Add(time.Hour) } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } var said bool for _, l := range strings.Split(buf.String(), "\n") { if !strings.Contains(l, "interrupted run restarted") { continue } said = true var line map[string]any if err := json.Unmarshal([]byte(l), &line); err != nil { t.Fatal(err) } if line["cap"] != float64(7) { t.Errorf("the restart line says cap=%v, want the deployment's 7: an operator cannot see the "+ "wall coming from it", line["cap"]) } if line["attempt"] != float64(2) { t.Errorf("the restart line says attempt=%v, want 2 — the pair is what makes the distance "+ "readable", line["attempt"]) } } if !said { t.Fatalf("no restart was announced at all, so this test measured nothing: %s", buf.String()) } } // The DEFAULT is inside the band its own argument gives — the form `pricing` already uses for a // figure whose provenance is prose rather than a measurement. // // ⛔ WHY A BAND AND NOT AN EQUALITY: the number is a judgement and may be re-judged, but not to any // value. Below the deferral threshold this platform would give up on a run before the alarm an // operator acts on has even fired (StalledAfter), and a cap of one is unified backlog row 138's // defect turned into a setting — every run on a rebooted host dead, budget unspent. Far above fifty, // the loop the cap exists to bound costs about an hour of attempt rows and ledger pairs, which is // the rubbish D39.240 forbids leaving behind. func TestTheDefaultAttemptCapStaysInTheBandItsArgumentGives(t *testing.T) { if DefaultMaxAttempts < StalledAfter { t.Errorf("the default cap is %d, below the deferral threshold %d: a run would be given up on "+ "before an operator is ever told it is troubled", DefaultMaxAttempts, StalledAfter) } if DefaultMaxAttempts > 50 { t.Errorf("the default cap is %d: the loop it bounds writes an attempt row and a ledger pair per "+ "turn, and above fifty it stops being a bound on rubbish", DefaultMaxAttempts) } }