231 lines
10 KiB
Go
231 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
// abandon_proof_test.go: the PROOF the orphan branch of `run abandon` stands on.
|
|
//
|
|
// The handle closes an account's money over one fact — that no engine process exists for the stuck
|
|
// attempt — and this file is what stops that fact being obtained wrongly. Nothing pinned it: an
|
|
// adversarial pass found that `systemctl --user` answers about the CALLING user's manager while the
|
|
// runs live in the daemon user's, so a unit nobody has ever heard of and a unit that ended look
|
|
// identical, and the acceptance then found that the CURE was unpinned too — removing the
|
|
// discriminator outright left the whole battery green.
|
|
//
|
|
// ⚠ The three refusals matter as much as the one success. A handle that refuses when it should
|
|
// proceed costs an operator a support call; a handle that proceeds when it should refuse strands a
|
|
// live engine spending against money nobody will be charged for.
|
|
|
|
// The discriminator itself, on all four answers. It is a pure function of the environment and the
|
|
// filesystem, so it is pinned without a database and without systemd — which is the point: this is
|
|
// the check that must survive every future edit of the command around it.
|
|
//
|
|
// Mutation caught: removing the `notTheRunsOwnManager` call from askSystemd; returning "" for a
|
|
// missing variable; returning "" for a directory owned by somebody else; returning a reason for a
|
|
// directory that IS ours (the handle would then never work at all).
|
|
func TestTheDiscriminatorAnswersAllFourWaysAndOnlyOneOfThemIsAProof(t *testing.T) {
|
|
t.Run("no state directory in the environment", func(t *testing.T) {
|
|
t.Setenv("TM_PLATFORM_STATE_DIR", "")
|
|
why := notTheRunsOwnManager()
|
|
if why == "" {
|
|
t.Fatal("an unverifiable answer was accepted as a proof: with no state directory there is " +
|
|
"nothing to tell whose systemd this shell can reach")
|
|
}
|
|
// The EMPTY case has its own sentence, and the assertion is on that sentence rather than on
|
|
// the variable's name: without it, deleting the empty-check still refuses — `os.Stat("")` fails
|
|
// and the reader is told the directory «could not be read», which is true of nothing and sends
|
|
// an operator looking for a path that was never set. Measured as a surviving mutant.
|
|
if !strings.Contains(why, "has no TM_PLATFORM_STATE_DIR") {
|
|
t.Errorf("the refusal does not say the variable is UNSET (it would read as a broken path): %q", why)
|
|
}
|
|
})
|
|
|
|
t.Run("a state directory that cannot be read", func(t *testing.T) {
|
|
t.Setenv("TM_PLATFORM_STATE_DIR", filepath.Join(t.TempDir(), "not-there"))
|
|
if why := notTheRunsOwnManager(); why == "" {
|
|
t.Fatal("a state directory that does not exist was accepted as a proof")
|
|
} else if !strings.Contains(why, "could not be read") {
|
|
t.Errorf("the refusal does not say what went wrong: %q", why)
|
|
}
|
|
})
|
|
|
|
t.Run("a state directory owned by somebody else", func(t *testing.T) {
|
|
if os.Getuid() == 0 {
|
|
t.Skip("running as root: every directory is ours, so the mismatch cannot be produced here")
|
|
}
|
|
// `/` is root's on every host this deploys to, and the process is not root — so this is a real
|
|
// ownership mismatch produced without creating one.
|
|
t.Setenv("TM_PLATFORM_STATE_DIR", "/")
|
|
why := notTheRunsOwnManager()
|
|
if why == "" {
|
|
t.Fatal("a state directory owned by ANOTHER user was accepted as a proof — this is the " +
|
|
"defect itself: `systemctl --user` would answer about this shell's manager, where a " +
|
|
"unit that never existed and a unit that ended are the same answer")
|
|
}
|
|
// Both numbers, because "run it as the right user" is useless without saying which.
|
|
if !strings.Contains(why, "uid 0") || !strings.Contains(why, "-M") {
|
|
t.Errorf("the refusal does not name the owner or the way to ask its manager: %q", why)
|
|
}
|
|
})
|
|
|
|
t.Run("our own state directory", func(t *testing.T) {
|
|
t.Setenv("TM_PLATFORM_STATE_DIR", t.TempDir())
|
|
if why := notTheRunsOwnManager(); why != "" {
|
|
t.Fatalf("the deployment's own user was refused (%q): the handle would never work", why)
|
|
}
|
|
})
|
|
}
|
|
|
|
// The same four answers as the OPERATOR meets them, through the command, against a live database and
|
|
// a run that is genuinely stuck: what the command prints, and — the half that decides money —
|
|
// whether the run is still live afterwards.
|
|
//
|
|
// Mutation caught: any of the discriminator mutations above (the run gets abandoned over an
|
|
// unverifiable answer); dropping the ProcessGone re-call (the handle stops working entirely).
|
|
func TestTheOperatorCannotAbandonARunOverAnUnverifiableAnswer(t *testing.T) {
|
|
live := func(t *testing.T, dsn string) bool {
|
|
t.Helper()
|
|
return countRows(t, dsn, `select count(*) from runs where id = 'run_probe' and finished_at is null`) == 1
|
|
}
|
|
|
|
t.Run("with no state directory the run is left alone", func(t *testing.T) {
|
|
dsn := stuckLiveRun(t)
|
|
t.Setenv("TM_PLATFORM_STATE_DIR", "")
|
|
var out strings.Builder
|
|
err := run([]string{"run", "abandon", "--run", "run_probe", "--reason", "the host is gone"}, &out)
|
|
if err == nil {
|
|
t.Fatal("a run was abandoned on an answer nothing could verify")
|
|
}
|
|
if !strings.Contains(err.Error(), "TM_PLATFORM_STATE_DIR") {
|
|
t.Errorf("the refusal does not tell the operator what is missing: %v", err)
|
|
}
|
|
if !live(t, dsn) {
|
|
t.Error("the run was ended anyway")
|
|
}
|
|
})
|
|
|
|
t.Run("with somebody else's state directory the run is left alone", func(t *testing.T) {
|
|
if os.Getuid() == 0 {
|
|
t.Skip("running as root: the ownership mismatch cannot be produced here")
|
|
}
|
|
dsn := stuckLiveRun(t)
|
|
t.Setenv("TM_PLATFORM_STATE_DIR", "/")
|
|
var out strings.Builder
|
|
if err := run([]string{"run", "abandon", "--run", "run_probe", "--reason", "the host is gone"}, &out); err == nil {
|
|
t.Fatal("a run was abandoned while asking the wrong systemd")
|
|
}
|
|
if !live(t, dsn) {
|
|
t.Error("the run was ended anyway")
|
|
}
|
|
})
|
|
|
|
t.Run("with our own state directory and a unit that is gone the run is ended", func(t *testing.T) {
|
|
dsn := stuckLiveRun(t)
|
|
t.Setenv("TM_PLATFORM_STATE_DIR", t.TempDir())
|
|
got := capture(t, "run", "abandon", "--run", "run_probe", "--reason", "the host is gone")
|
|
if !strings.Contains(got, "is gone") {
|
|
t.Errorf("the command did not say what it established before acting:\n%s", got)
|
|
}
|
|
if live(t, dsn) {
|
|
t.Error("the run is still live: the handle did not work on the population it exists for")
|
|
}
|
|
})
|
|
}
|
|
|
|
// And the answer no environment variable can fake: systemd says the unit IS there.
|
|
//
|
|
// It is the one case that cannot be reasoned about — a real transient unit is created, named exactly
|
|
// as the stuck attempt's, and the command must refuse over it. Gated on a reachable user manager for
|
|
// the same reason every live-systemd test in this repository is, and the skip is loud.
|
|
//
|
|
// Mutation caught: treating `active` as gone; dropping the Alive call entirely.
|
|
func TestARunWhoseUnitIsRunningIsRefusedByTheCommandItself(t *testing.T) {
|
|
if _, err := exec.LookPath("systemd-run"); err != nil {
|
|
t.Skip("no systemd-run: the live-unit refusal cannot be measured on this host")
|
|
}
|
|
if out, err := exec.CommandContext(t.Context(), "systemctl", "--user", "show", "--property=Version").CombinedOutput(); err != nil {
|
|
t.Skipf("no reachable systemd user manager (%s): the live-unit refusal cannot be measured on this host",
|
|
strings.TrimSpace(string(out)))
|
|
}
|
|
dsn := stuckLiveRun(t)
|
|
unit := "tm-run-run_probe-1"
|
|
// A unit of our own, in OUR manager — which is also the manager the command will ask, because the
|
|
// state directory below is ours. That is the arrangement the discriminator exists to guarantee.
|
|
if out, err := exec.CommandContext(t.Context(), "systemd-run", "--user", "--unit", unit,
|
|
"--property=Type=simple", "sleep", "120").CombinedOutput(); err != nil {
|
|
t.Skipf("this host would not start a transient unit (%s)", strings.TrimSpace(string(out)))
|
|
}
|
|
t.Cleanup(func() {
|
|
// Its OWN context: the test's is already cancelled by the time cleanups run, and a unit left
|
|
// active would outlive the test and be there for the next one to trip over.
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
_ = exec.CommandContext(ctx, "systemctl", "--user", "stop", unit+".service").Run()
|
|
_ = exec.CommandContext(ctx, "systemctl", "--user", "reset-failed", unit+".service").Run()
|
|
})
|
|
// systemd-run returns as soon as the job is queued; the unit is active a moment later.
|
|
for i := 0; i < 50; i++ {
|
|
out, _ := exec.CommandContext(t.Context(), "systemctl", "--user", "show", unit+".service",
|
|
"--property=ActiveState", "--value").Output()
|
|
if strings.TrimSpace(string(out)) == "active" {
|
|
break
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
|
|
t.Setenv("TM_PLATFORM_STATE_DIR", t.TempDir())
|
|
var out strings.Builder
|
|
err := run([]string{"run", "abandon", "--run", "run_probe", "--reason", "I think it is gone"}, &out)
|
|
if err == nil {
|
|
t.Fatal("a run whose unit is RUNNING was abandoned: its engine would go on spending against " +
|
|
"money this command has just given back")
|
|
}
|
|
if !strings.Contains(err.Error(), "still active") {
|
|
t.Errorf("the refusal does not say what systemd answered: %v", err)
|
|
}
|
|
if countRows(t, dsn, `select count(*) from runs where id = 'run_probe' and finished_at is null`) != 1 {
|
|
t.Error("the run was ended over a live unit")
|
|
}
|
|
}
|
|
|
|
// stuckLiveRun is a live run in the population the handle exists for: its attempt names a unit and
|
|
// carries a spend baseline (so the store refuses without a proof), and it has failed to reconcile
|
|
// the threshold number of times (so the floor is satisfied and the proof is the only thing left).
|
|
func stuckLiveRun(t *testing.T) string {
|
|
t.Helper()
|
|
dsn := freshDB(t)
|
|
t.Setenv("TM_PLATFORM_DSN", dsn)
|
|
seedALiveRun(t, dsn)
|
|
execSQL(t, dsn, `update run_attempts
|
|
set unit_name = 'tm-run-run_probe-1', spend_baseline_micro_usd = 0,
|
|
reconcile_failures = 5
|
|
where run_id = 'run_probe'`)
|
|
return dsn
|
|
}
|
|
|
|
// countRows answers one aggregate against the test's own database. It exists because what these
|
|
// tests assert is not what the command PRINTED but what it DID to a run — the printing is the
|
|
// smaller half of a money handle.
|
|
func countRows(t *testing.T, dsn, sql string) int {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
conn, err := pgx.Connect(ctx, dsn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.Close(ctx)
|
|
var n int
|
|
if err := conn.QueryRow(ctx, sql).Scan(&n); err != nil {
|
|
t.Fatalf("%s: %v", sql, err)
|
|
}
|
|
return n
|
|
}
|