284 lines
13 KiB
Go
284 lines
13 KiB
Go
package runs
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"textmachine/platform/internal/ingest"
|
|
"textmachine/platform/internal/money"
|
|
"textmachine/platform/internal/pgstore"
|
|
"textmachine/platform/internal/runner"
|
|
)
|
|
|
|
// ⛔ THE RESTART LOOP HAS A FLOOR, AND THE MONEY IS CLOSED WHEN IT IS REACHED.
|
|
//
|
|
// A unit that ends with no marker is restarted (unified backlog row 398), and where the marker
|
|
// machinery itself is broken every replacement ends the same way. The loop is money-NEUTRAL per turn
|
|
// — the old hold comes back whole and an equal one is taken — so no balance ever stops it, and every
|
|
// turn writes an attempt row and three ledger rows.
|
|
//
|
|
// Three things are asserted together because a cure for any one of them alone would be a defect: the
|
|
// platform STOPS (no further attempt, no further unit), the last attempt's hold is CLOSED rather than
|
|
// frozen (refusing before the settlement would trade an unbounded loop for a stranded reservation),
|
|
// and the ending does not tell the client that retrying is the remedy after this platform retried and
|
|
// gave up.
|
|
func TestTheRestartLoopStopsAtTheCapWithTheMoneyClosedAndNoNewHold(t *testing.T) {
|
|
f := newFixture(t, "10", 500)
|
|
f.svc.Cfg.MaxAttempts = 3
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The shape the row describes: the engine spends nothing, the unit is gone and no marker is ever
|
|
// written, so every pass reads the same three facts and reaches the same decision.
|
|
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil)
|
|
f.runner.alive = false
|
|
// Two passes restart, the third meets the cap. The clock moves because a restart stamps the new
|
|
// attempt with `now` and the spawn grace has to be behind it before the next pass may decide.
|
|
for i := 1; i <= 3; i++ {
|
|
f.svc.Now = func() time.Time { return f.now.Add(time.Duration(i) * time.Hour) }
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatalf("pass %d: %v", i, err)
|
|
}
|
|
}
|
|
_, card, err := f.store.GetBook(f.ctx, "u1", f.bookID(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if card == nil || card.Status != "failed" || card.FinishedAt == nil {
|
|
t.Fatalf("the capped run did not end: %+v", card)
|
|
}
|
|
// NOT `interrupted`: that word promises a client the retry is the remedy, and this platform has
|
|
// just performed that retry as many times as the deployment allows.
|
|
if card.FailureReason != "service_error" {
|
|
t.Errorf("the capped run is reported as %q, want service_error", card.FailureReason)
|
|
}
|
|
var attempts int
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select count(*) from run_attempts where run_id = $1`, run.ID).Scan(&attempts); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if attempts != 3 {
|
|
t.Errorf("%d attempt rows, want 3: the cap is the number of attempts one run may have", attempts)
|
|
}
|
|
if n := len(f.runner.starts()); n != 3 {
|
|
t.Errorf("%d units started, want 3: a capped run must not be spawned again", n)
|
|
}
|
|
// The platform's OWN word, on the attempt it stopped at. It is what tells an operator "we stopped
|
|
// trying" from "the run kept dying", and systemd cannot produce it.
|
|
var result string
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select coalesce(exit_result, '') from run_attempts where run_id = $1 order by attempt_no desc limit 1`,
|
|
run.ID).Scan(&result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result != runner.AttemptsExhaustedResult {
|
|
t.Errorf("the last attempt's exit_result is %q, want %q", result, runner.AttemptsExhaustedResult)
|
|
}
|
|
acct := f.account(t)
|
|
if acct.Reserved != 0 {
|
|
t.Errorf("the capped run left %s reserved: refusing the restart before the settlement would "+
|
|
"trade an unbounded loop for a frozen hold", acct.Reserved.USD())
|
|
}
|
|
if acct.Balance != acct.LedgerSum {
|
|
t.Errorf("balance %s and ledger %s disagree after the cap", acct.Balance.USD(), acct.LedgerSum.USD())
|
|
}
|
|
// Nothing was spent, so the whole hold came back.
|
|
if want := money.MicroUSD(10_000_000); acct.Balance != want {
|
|
t.Errorf("balance %s, want %s — the capped run spent nothing and its hold is not kept", acct.Balance.USD(), want.USD())
|
|
}
|
|
}
|
|
|
|
// ⚠ THE CONTROL THAT KEEPS THE PIN ABOVE FROM MEASURING NOTHING: the SAME fixture, one attempt below
|
|
// the cap, is still restarted.
|
|
//
|
|
// It is also the pin on the choice unified backlog row 396 asked to be re-decided rather than
|
|
// inherited: an ending nobody recorded an intent for is an INTERRUPTION and comes back, because
|
|
// reading a reboot as a user's stop would leave every run on a restarted host dead with its budget
|
|
// unspent (row 138). What the cap changes is not that answer but its price — "one restart of a run
|
|
// whose owner never asked for it" is now a bounded sentence rather than a loop.
|
|
func TestBelowTheCapTheSameRunIsStillRestarted(t *testing.T) {
|
|
f := newFixture(t, "10", 500)
|
|
f.svc.Cfg.MaxAttempts = 5
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil)
|
|
f.runner.alive = false
|
|
for i := 1; i <= 3; i++ {
|
|
f.svc.Now = func() time.Time { return f.now.Add(time.Duration(i) * time.Hour) }
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatalf("pass %d: %v", i, err)
|
|
}
|
|
}
|
|
live := f.live(t)
|
|
if live.AttemptNo != 4 {
|
|
t.Fatalf("attempt %d after three passes under a cap of 5: the run should still be coming back", live.AttemptNo)
|
|
}
|
|
if n := len(f.runner.starts()); n != 4 {
|
|
t.Errorf("%d units started, want 4", n)
|
|
}
|
|
}
|
|
|
|
// ⛔ THE CAP IS NOT THE COUNTER A RESTART RESETS, and this test asserts the PREMISE rather than the
|
|
// consequence — without it the pin above would pass for a cap keyed on the wrong number and nobody
|
|
// would learn anything.
|
|
//
|
|
// `reconcile_failures` is the obvious place to count restarts and it is the wrong one: it lives on
|
|
// the ATTEMPT, and a restart opens a new attempt row (pgstore/runs.go, `next := attemptNo + 1`), so
|
|
// it is zero on every turn of this loop by construction. That is measured here — at the very pass
|
|
// that meets the cap, the counter a cap might have been built on reads zero — which is why a cap
|
|
// expressed in it would never fire and the defect would stand, wearing the look of a cure (the
|
|
// alarm's own threshold, StalledAfter, is unreachable for exactly this reason).
|
|
func TestTheCapIsNotTheCounterARestartResets(t *testing.T) {
|
|
f := newFixture(t, "10", 500)
|
|
f.svc.Cfg.MaxAttempts = 3
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil)
|
|
f.runner.alive = false
|
|
// Two passes: the run is now standing at the LAST attempt the cap allows, and the next pass is
|
|
// the one that ends it.
|
|
for i := 1; i <= 2; i++ {
|
|
f.svc.Now = func() time.Time { return f.now.Add(time.Duration(i) * time.Hour) }
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatalf("pass %d: %v", i, err)
|
|
}
|
|
}
|
|
live := f.live(t)
|
|
if live.AttemptNo != 3 {
|
|
t.Fatalf("attempt %d, want 3: this test has to stand at the cap for its measurement to mean anything", live.AttemptNo)
|
|
}
|
|
if live.ReconcileFailures != 0 {
|
|
t.Fatalf("the deferral counter reads %d at the attempt the cap ends: this fixture no longer "+
|
|
"demonstrates that a cap keyed on that counter measures nothing", live.ReconcileFailures)
|
|
}
|
|
if live.AttemptNo < f.svc.maxAttempts() {
|
|
t.Fatalf("attempt %d is below the cap %d: the premise of this test is that the NEXT decision is the cap's",
|
|
live.AttemptNo, f.svc.maxAttempts())
|
|
}
|
|
}
|
|
|
|
// The ending a cap writes must not be read as a retryable interruption — and the neighbours it sits
|
|
// beside must not move, which is what the table's other rows are for.
|
|
//
|
|
// ⚠ BOTH SHAPES OF THE CAP'S ENDING ARE HERE, because the two call sites of the restart hand over
|
|
// different evidence: one has a marker the unit left (the code travels, the result word is ours), the
|
|
// other has none at all. A fix that covered one of them would look complete and be half.
|
|
func TestAnExhaustedRunIsNotCalledARetryableInterruption(t *testing.T) {
|
|
at := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC)
|
|
for _, tc := range []struct {
|
|
name string
|
|
marker runner.Marker
|
|
want string
|
|
}{
|
|
// No marker was ever written: the word this platform would have recorded is its own either
|
|
// way, and the cap's is the more informative one.
|
|
{"the cap, with nothing the unit left",
|
|
runner.Marker{Unit: "u", Result: runner.AttemptsExhaustedResult}, "service_error"},
|
|
// The unit left a caught signal. The code travels to the client as it always did and the
|
|
// answer is the same, so the operator's column keeps both facts.
|
|
{"the cap, over a unit that exited 5",
|
|
runner.Marker{Unit: "u", Result: runner.AttemptsExhaustedResult, Code: "exited", Status: "5", At: at}, "service_error"},
|
|
// The controls: an ending nobody described is still the one where retrying IS the remedy, and
|
|
// our own grace is still not a broken deployment (gracekill_test owns that pair).
|
|
{"a unit that simply vanished, below the cap",
|
|
runner.Marker{Unit: "u", Result: runner.UnitVanishedResult}, "interrupted"},
|
|
{"our own stop deadline", runner.Marker{Unit: "u", Result: "timeout", Code: "killed", Status: "KILL", At: at}, "interrupted"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
status, _, exit := outcome(pgstore.LiveRun{RunID: "r"}, tc.marker)
|
|
if status != "failed" {
|
|
t.Fatalf("this ending came out %q, not failed: the test is about the reason a FAILED run "+
|
|
"carries, so it now measures nothing", status)
|
|
}
|
|
if got := failureReason(status, exit, tc.marker); got != tc.want {
|
|
t.Errorf("reported to the client as %q, want %q", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// ⚠ THE LIMIT IS VISIBLE BEFORE IT IS REACHED, which the pack made a condition of the cap and not a
|
|
// nicety: a platform that stops restarting without having said how close it was leaves an operator to
|
|
// discover the wall by arriving at it.
|
|
//
|
|
// The cap rides EVERY restart line rather than a threshold line of its own, and that is deliberate —
|
|
// a second place where the number is written is a second place for it to go stale against the first.
|
|
func TestEveryRestartSaysHowCloseTheCapIs(t *testing.T) {
|
|
f := newFixture(t, "10", 500)
|
|
f.svc.Cfg.MaxAttempts = 7
|
|
var buf bytes.Buffer
|
|
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.StatusReport{TotalUnits: 100, Done: 0, Spend: usd(0), Reserved: usd(0)}, nil)
|
|
f.runner.alive = false
|
|
f.svc.Now = func() time.Time { return f.now.Add(time.Hour) }
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var said bool
|
|
for _, l := range strings.Split(buf.String(), "\n") {
|
|
if !strings.Contains(l, "interrupted run restarted") {
|
|
continue
|
|
}
|
|
said = true
|
|
var line map[string]any
|
|
if err := json.Unmarshal([]byte(l), &line); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if line["cap"] != float64(7) {
|
|
t.Errorf("the restart line says cap=%v, want the deployment's 7: an operator cannot see the "+
|
|
"wall coming from it", line["cap"])
|
|
}
|
|
if line["attempt"] != float64(2) {
|
|
t.Errorf("the restart line says attempt=%v, want 2 — the pair is what makes the distance "+
|
|
"readable", line["attempt"])
|
|
}
|
|
}
|
|
if !said {
|
|
t.Fatalf("no restart was announced at all, so this test measured nothing: %s", buf.String())
|
|
}
|
|
}
|
|
|
|
// The DEFAULT is inside the band its own argument gives — the form `pricing` already uses for a
|
|
// figure whose provenance is prose rather than a measurement.
|
|
//
|
|
// ⛔ WHY A BAND AND NOT AN EQUALITY: the number is a judgement and may be re-judged, but not to any
|
|
// value. Below the deferral threshold this platform would give up on a run before the alarm an
|
|
// operator acts on has even fired (StalledAfter), and a cap of one is unified backlog row 138's
|
|
// defect turned into a setting — every run on a rebooted host dead, budget unspent. Far above fifty,
|
|
// the loop the cap exists to bound costs about an hour of attempt rows and ledger pairs, which is
|
|
// the rubbish D39.240 forbids leaving behind.
|
|
func TestTheDefaultAttemptCapStaysInTheBandItsArgumentGives(t *testing.T) {
|
|
if DefaultMaxAttempts < StalledAfter {
|
|
t.Errorf("the default cap is %d, below the deferral threshold %d: a run would be given up on "+
|
|
"before an operator is ever told it is troubled", DefaultMaxAttempts, StalledAfter)
|
|
}
|
|
if DefaultMaxAttempts > 50 {
|
|
t.Errorf("the default cap is %d: the loop it bounds writes an attempt row and a ledger pair per "+
|
|
"turn, and above fifty it stops being a bound on rubbish", DefaultMaxAttempts)
|
|
}
|
|
}
|