package main import ( "context" "os" "os/exec" "path/filepath" "strings" "testing" "time" "github.com/jackc/pgx/v5" ) // abandon_proof_test.go: the PROOF the orphan branch of `run abandon` stands on. // // The handle closes an account's money over one fact — that no engine process exists for the stuck // attempt — and this file is what stops that fact being obtained wrongly. Nothing pinned it: an // adversarial pass found that `systemctl --user` answers about the CALLING user's manager while the // runs live in the daemon user's, so a unit nobody has ever heard of and a unit that ended look // identical, and the acceptance then found that the CURE was unpinned too — removing the // discriminator outright left the whole battery green. // // ⚠ The three refusals matter as much as the one success. A handle that refuses when it should // proceed costs an operator a support call; a handle that proceeds when it should refuse strands a // live engine spending against money nobody will be charged for. // The discriminator itself, on all four answers. It is a pure function of the environment and the // filesystem, so it is pinned without a database and without systemd — which is the point: this is // the check that must survive every future edit of the command around it. // // Mutation caught: removing the `notTheRunsOwnManager` call from askSystemd; returning "" for a // missing variable; returning "" for a directory owned by somebody else; returning a reason for a // directory that IS ours (the handle would then never work at all). func TestTheDiscriminatorAnswersAllFourWaysAndOnlyOneOfThemIsAProof(t *testing.T) { t.Run("no state directory in the environment", func(t *testing.T) { t.Setenv("TM_PLATFORM_STATE_DIR", "") why := notTheRunsOwnManager() if why == "" { t.Fatal("an unverifiable answer was accepted as a proof: with no state directory there is " + "nothing to tell whose systemd this shell can reach") } // The EMPTY case has its own sentence, and the assertion is on that sentence rather than on // the variable's name: without it, deleting the empty-check still refuses — `os.Stat("")` fails // and the reader is told the directory «could not be read», which is true of nothing and sends // an operator looking for a path that was never set. Measured as a surviving mutant. if !strings.Contains(why, "has no TM_PLATFORM_STATE_DIR") { t.Errorf("the refusal does not say the variable is UNSET (it would read as a broken path): %q", why) } }) t.Run("a state directory that cannot be read", func(t *testing.T) { t.Setenv("TM_PLATFORM_STATE_DIR", filepath.Join(t.TempDir(), "not-there")) if why := notTheRunsOwnManager(); why == "" { t.Fatal("a state directory that does not exist was accepted as a proof") } else if !strings.Contains(why, "could not be read") { t.Errorf("the refusal does not say what went wrong: %q", why) } }) t.Run("a state directory owned by somebody else", func(t *testing.T) { if os.Getuid() == 0 { t.Skip("running as root: every directory is ours, so the mismatch cannot be produced here") } // `/` is root's on every host this deploys to, and the process is not root — so this is a real // ownership mismatch produced without creating one. t.Setenv("TM_PLATFORM_STATE_DIR", "/") why := notTheRunsOwnManager() if why == "" { t.Fatal("a state directory owned by ANOTHER user was accepted as a proof — this is the " + "defect itself: `systemctl --user` would answer about this shell's manager, where a " + "unit that never existed and a unit that ended are the same answer") } // Both numbers, because "run it as the right user" is useless without saying which. if !strings.Contains(why, "uid 0") || !strings.Contains(why, "-M") { t.Errorf("the refusal does not name the owner or the way to ask its manager: %q", why) } }) t.Run("our own state directory", func(t *testing.T) { t.Setenv("TM_PLATFORM_STATE_DIR", t.TempDir()) if why := notTheRunsOwnManager(); why != "" { t.Fatalf("the deployment's own user was refused (%q): the handle would never work", why) } }) } // The same four answers as the OPERATOR meets them, through the command, against a live database and // a run that is genuinely stuck: what the command prints, and — the half that decides money — // whether the run is still live afterwards. // // Mutation caught: any of the discriminator mutations above (the run gets abandoned over an // unverifiable answer); dropping the ProcessGone re-call (the handle stops working entirely). func TestTheOperatorCannotAbandonARunOverAnUnverifiableAnswer(t *testing.T) { live := func(t *testing.T, dsn string) bool { t.Helper() return countRows(t, dsn, `select count(*) from runs where id = 'run_probe' and finished_at is null`) == 1 } t.Run("with no state directory the run is left alone", func(t *testing.T) { dsn := stuckLiveRun(t) t.Setenv("TM_PLATFORM_STATE_DIR", "") var out strings.Builder err := run([]string{"run", "abandon", "--run", "run_probe", "--reason", "the host is gone"}, &out) if err == nil { t.Fatal("a run was abandoned on an answer nothing could verify") } if !strings.Contains(err.Error(), "TM_PLATFORM_STATE_DIR") { t.Errorf("the refusal does not tell the operator what is missing: %v", err) } if !live(t, dsn) { t.Error("the run was ended anyway") } }) t.Run("with somebody else's state directory the run is left alone", func(t *testing.T) { if os.Getuid() == 0 { t.Skip("running as root: the ownership mismatch cannot be produced here") } dsn := stuckLiveRun(t) t.Setenv("TM_PLATFORM_STATE_DIR", "/") var out strings.Builder if err := run([]string{"run", "abandon", "--run", "run_probe", "--reason", "the host is gone"}, &out); err == nil { t.Fatal("a run was abandoned while asking the wrong systemd") } if !live(t, dsn) { t.Error("the run was ended anyway") } }) t.Run("with our own state directory and a unit that is gone the run is ended", func(t *testing.T) { dsn := stuckLiveRun(t) t.Setenv("TM_PLATFORM_STATE_DIR", t.TempDir()) got := capture(t, "run", "abandon", "--run", "run_probe", "--reason", "the host is gone") if !strings.Contains(got, "is gone") { t.Errorf("the command did not say what it established before acting:\n%s", got) } if live(t, dsn) { t.Error("the run is still live: the handle did not work on the population it exists for") } }) } // And the answer no environment variable can fake: systemd says the unit IS there. // // It is the one case that cannot be reasoned about — a real transient unit is created, named exactly // as the stuck attempt's, and the command must refuse over it. Gated on a reachable user manager for // the same reason every live-systemd test in this repository is, and the skip is loud. // // Mutation caught: treating `active` as gone; dropping the Alive call entirely. func TestARunWhoseUnitIsRunningIsRefusedByTheCommandItself(t *testing.T) { if _, err := exec.LookPath("systemd-run"); err != nil { t.Skip("no systemd-run: the live-unit refusal cannot be measured on this host") } if out, err := exec.CommandContext(t.Context(), "systemctl", "--user", "show", "--property=Version").CombinedOutput(); err != nil { t.Skipf("no reachable systemd user manager (%s): the live-unit refusal cannot be measured on this host", strings.TrimSpace(string(out))) } dsn := stuckLiveRun(t) unit := "tm-run-run_probe-1" // A unit of our own, in OUR manager — which is also the manager the command will ask, because the // state directory below is ours. That is the arrangement the discriminator exists to guarantee. if out, err := exec.CommandContext(t.Context(), "systemd-run", "--user", "--unit", unit, "--property=Type=simple", "sleep", "120").CombinedOutput(); err != nil { t.Skipf("this host would not start a transient unit (%s)", strings.TrimSpace(string(out))) } t.Cleanup(func() { // Its OWN context: the test's is already cancelled by the time cleanups run, and a unit left // active would outlive the test and be there for the next one to trip over. ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() _ = exec.CommandContext(ctx, "systemctl", "--user", "stop", unit+".service").Run() _ = exec.CommandContext(ctx, "systemctl", "--user", "reset-failed", unit+".service").Run() }) // systemd-run returns as soon as the job is queued; the unit is active a moment later. for i := 0; i < 50; i++ { out, _ := exec.CommandContext(t.Context(), "systemctl", "--user", "show", unit+".service", "--property=ActiveState", "--value").Output() if strings.TrimSpace(string(out)) == "active" { break } time.Sleep(100 * time.Millisecond) } t.Setenv("TM_PLATFORM_STATE_DIR", t.TempDir()) var out strings.Builder err := run([]string{"run", "abandon", "--run", "run_probe", "--reason", "I think it is gone"}, &out) if err == nil { t.Fatal("a run whose unit is RUNNING was abandoned: its engine would go on spending against " + "money this command has just given back") } if !strings.Contains(err.Error(), "still active") { t.Errorf("the refusal does not say what systemd answered: %v", err) } if countRows(t, dsn, `select count(*) from runs where id = 'run_probe' and finished_at is null`) != 1 { t.Error("the run was ended over a live unit") } } // stuckLiveRun is a live run in the population the handle exists for: its attempt names a unit and // carries a spend baseline (so the store refuses without a proof), and it has failed to reconcile // the threshold number of times (so the floor is satisfied and the proof is the only thing left). func stuckLiveRun(t *testing.T) string { t.Helper() dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedALiveRun(t, dsn) execSQL(t, dsn, `update run_attempts set unit_name = 'tm-run-run_probe-1', spend_baseline_micro_usd = 0, reconcile_failures = 5 where run_id = 'run_probe'`) return dsn } // countRows answers one aggregate against the test's own database. It exists because what these // tests assert is not what the command PRINTED but what it DID to a run — the printing is the // smaller half of a money handle. func countRows(t *testing.T, dsn, sql string) int { t.Helper() ctx := t.Context() conn, err := pgx.Connect(ctx, dsn) if err != nil { t.Fatal(err) } defer conn.Close(ctx) var n int if err := conn.QueryRow(ctx, sql).Scan(&n); err != nil { t.Fatalf("%s: %v", sql, err) } return n }