textmachine/platform/internal/runner/runner.go

// Package runner starts one engine run as a TRANSIENT SYSTEMD UNIT and answers what happened to
// it. It is the production half of the seam ratified as D39.106: the platform is NOT the engine's
// parent, because a run has to outlive a deploy or a crash of the control plane and a child cannot.
//
// # Privilege model
//
// Units are created in the platform user's OWN systemd manager (`systemd-run --user`), which needs
// no new privilege at all: a user may always manage their own units. The alternative research/25
// named — a narrow polkit rule over the SYSTEM manager — is not merely heavier, it is unsound for
// this job, and the reason is a property of polkit rather than a preference:
//
//   - polkit authorizes the VERB, not the properties. StartTransientUnit lets the CALLER choose
//     ExecStart= and User=, and a transient unit in the system manager with no User= runs as root.
//     A rule that lets this service create transient system units therefore grants it root, and no
//     rule can narrow that, because properties are not part of the authorization request.
//   - on systemd before v257 the request does not even carry the unit NAME for StartTransientUnit
//     (systemd issue #17224, fixed by PR #34651, merged 2024-10-09), so `action.lookup("unit")` is
//     undefined and the rule cannot be scoped by name either.
//
// The cost of the user manager is one root action at install time (`loginctl enable-linger`) and
// the two lines in deploy/tmplatformd.service that let this service reach its own user bus. See
// STACK_DECISIONS §15.
//
// # Why the unit's own state is not the truth
//
// Units are started with --collect, so systemd unloads them the instant they exit: `systemctl show`
// then reports LoadState=not-found and ExecMainStatus=0 for a run that exited 3 and for a run that
// never existed alike (measured — zone journal, session P4). What the platform reads instead is the
// EXIT MARKER that ExecStopPost writes, and the source of truth on boot is Postgres plus the book's
// directory, never systemd (research/25 §Опс).
package runner

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"log/slog"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

// Slice is the cgroup slice every run unit is placed in.
//
// NOT cosmetic, and not merely tidiness: measured on the stand (systemd 255), a --user unit left in
// the default app.slice gets NO cgroup control files at all — app.slice's cgroup.subtree_control
// stays empty, so MemoryMax= and TasksMax= are accepted by systemd, reported back by `show`, and
// enforce nothing (a process that faulted in 400 MiB survived MemoryMax=64M). Placed in a slice of
// its own the leaf gets memory.max and pids.max for real, and the same process is oom-killed at the
// limit. This is the answer to PD-13: what bounds a run is the run's own cgroup, not the control
// plane's.
const Slice = "tm-runs.slice"

// stopGrace is how long the engine has to shut down after SIGTERM before systemd kills it. Generous
// on purpose (research/25 §Форма): the engine holds an EXCLUSIVE lock on the book's project file
// and finishes the in-flight chunk before exiting, so a short grace buys a leftover lock.
const stopGrace = 10 * time.Minute

// Spec is one run to start.
type Spec struct {
	// Unit is the transient unit's name, without the .service suffix.
	Unit string
	// Binary is the VERSIONED path of the engine binary (row 139). Absolute: the unit has its own
	// working directory and PATH is not part of the contract.
	Binary string
	// Args are the engine's arguments. They carry the book's config path and the run's ceiling, so
	// they are never logged (PD-99).
	Args []string
	// Workdir is the book's project directory.
	Workdir string
	// Env is the child's environment as KEY=VALUE. It carries the run's identity and never provider
	// keys — those reach the engine as the `--keys-file` argument in Args (row 211) — and, like
	// Args, it is never logged.
	Env []string
	// ExitMarker is the file ExecStopPost fills with what systemd saw. It is platform state and
	// lives in the platform's state directory, not in the book's directory — the engine owns that
	// one (D39.110).
	ExitMarker string
	// MarkerArgv is the command that writes ExitMarker, as argv. It runs as ExecStopPost=, in the
	// unit, with $SERVICE_RESULT/$EXIT_CODE/$EXIT_STATUS set by systemd.
	MarkerArgv []string
	// MemoryMax and TasksMax bound the run (PD-13). Empty MemoryMax means "no bound", which is a
	// decision the caller has to make deliberately rather than inherit.
	MemoryMax string
	TasksMax  int
}

// Runner starts and stops run units.
type Runner struct {
	// Log is the platform's own view. Money and book ids never reach it (D39.84, PD-99).
	Log *slog.Logger
	// run executes a command and returns its combined output. Injectable so the argv this package
	// builds can be pinned by a test without systemd, and so the systemd tests can be told apart
	// from the ones that only check the wiring.
	run func(ctx context.Context, name string, args ...string) ([]byte, error)
}

// New builds a Runner that talks to the real systemd.
func New(log *slog.Logger) *Runner { return &Runner{Log: log, run: runCommand} }

// ErrNotSupported is systemd being unreachable: no user manager, no bus, or no systemd-run.
var ErrNotSupported = errors.New("runner: no systemd user manager is reachable")

func runCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, name, args...)
	var out bytes.Buffer
	cmd.Stdout, cmd.Stderr = &out, &out
	err := cmd.Run()
	return out.Bytes(), err
}

// Available reports whether this process can create transient units at all. Called at boot so an
// operator learns it from a startup line instead of from the first run that refuses to start.
func (r *Runner) Available(ctx context.Context) error {
	if _, err := exec.LookPath("systemd-run"); err != nil {
		return fmt.Errorf("%w: %w", ErrNotSupported, err)
	}
	if out, err := r.run(ctx, "systemctl", "--user", "is-system-running"); err != nil {
		// `is-system-running` exits non-zero for degraded and starting states, which are still
		// perfectly able to run a unit. Only an unreachable bus is fatal, and that says so.
		if bytes.Contains(out, []byte("Failed to connect to bus")) {
			return fmt.Errorf("%w: %s", ErrNotSupported, bytes.TrimSpace(out))
		}
	}
	return nil
}

// Start creates the transient unit. It returns once systemd has accepted the unit; the run itself
// then lives entirely on its own.
func (r *Runner) Start(ctx context.Context, s Spec) error {
	if s.Unit == "" || s.Binary == "" || s.Workdir == "" || s.ExitMarker == "" || len(s.MarkerArgv) == 0 {
		return errors.New("runner: unit, binary, workdir and exit marker are all required")
	}
	args := r.startArgv(s)
	out, err := r.run(ctx, "systemd-run", args...)
	if err != nil {
		return fmt.Errorf("runner: systemd-run %s: %w: %s", s.Unit, err, bytes.TrimSpace(out))
	}
	// The command name, never argv: the arguments carry the book's identity and the run's ceiling
	// in dollars, and neither belongs in an INFO stream (PD-99, D39.84).
	r.log().InfoContext(ctx, "run unit started", "unit", s.Unit, "command", s.Binary)
	return nil
}

// startArgv is the argv of systemd-run, built in one place so a test can read it.
func (r *Runner) startArgv(s Spec) []string {
	args := []string{
		"--user",
		"--unit=" + s.Unit,
		// The unit is unloaded as soon as it is done. Without this a failed run stays loaded in
		// "failed" state until someone calls reset-failed, and the next attempt cannot reuse the
		// name. What the platform reads afterwards is the exit marker, not the unit.
		"--collect",
		"--quiet",
		"--property=Description=TextMachine run " + s.Unit,
		"--property=Slice=" + Slice,
		// Restart=no is systemd's default; it is written anyway because it is a RATIFIED property of
		// the seam (D39.106 §2) and a default is not a decision anyone can see.
		"--property=Restart=no",
		"--property=KillSignal=SIGTERM",
		// mixed: SIGTERM to the engine alone so it can finish the chunk it is paying for and release
		// its lock; the final SIGKILL goes to everything left in the cgroup.
		"--property=KillMode=mixed",
		"--property=TimeoutStopSec=" + strconv.Itoa(int(stopGrace.Seconds())),
		"--property=WorkingDirectory=" + s.Workdir,
		"--property=ExecStopPost=" + quoteArgv(s.MarkerArgv),
	}
	if s.MemoryMax != "" {
		// MemorySwapMax=0 goes WITH MemoryMax, always. A memory limit on its own does not bound a
		// run: at the limit the kernel reclaims to swap instead of killing, so the process survives
		// and slows to disk speed — measured here, and it is worse than the failure it replaces,
		// because a translation that thrashes for hours still spends money on every call it does
		// manage to make. With swap barred the limit is what it claims to be.
		args = append(args,
			"--property=MemoryAccounting=yes",
			"--property=MemoryMax="+s.MemoryMax,
			"--property=MemorySwapMax=0")
	}
	if s.TasksMax > 0 {
		args = append(args, "--property=TasksMax="+strconv.Itoa(s.TasksMax))
	}
	for _, kv := range s.Env {
		args = append(args, "--setenv="+kv)
	}
	args = append(args, "--", s.Binary)
	return append(args, s.Args...)
}

// Stop asks the unit to shut down. It returns nil for a unit that is already gone: "stop what is
// not running" is the state the caller wanted, and a reconciler that retries would otherwise treat
// its own success as a failure.
//
// ⛔ `--no-block` IS NOT AN OPTIMISATION. Without it `systemctl stop` waits for the unit to be GONE,
// so the call returns only when the engine has finished shutting down or the stop grace has run out.
// Measured on this stand (systemd 259, a transient unit whose process catches SIGTERM and does not
// exit, TimeoutStopSec=15): the blocking form returned after 15.058 s — by SIGKILL — and
// `--no-block` after 0.014 s, with the unit left in `deactivating`.
//
// Two things pay for that wait, and neither is the one an earlier reading of this named:
//
//   - THE CONTRACT DESCRIBES THIS CALL AS ASYNCHRONOUS. §stopRun promises a `202` that «does not mean
//     the run has stopped»; a handler that blocks until it HAS stopped keeps that promise's letter and
//     breaks its substance. ⚠ It is NOT a cut-off response: the API listener carries no WriteTimeout
//     at all, and deliberately (serve.go, «No WriteTimeout» — it would cut the SSE stream). The
//     30-second one belongs to the METRICS listener. So the damage is a request that hangs and a
//     handler goroutine held with it, not a `202` that never arrives (D39.235 §4 said otherwise and
//     was corrected by its own author, erratum 10.09-з).
//   - THE SWEEP RECORDS A WEDGE THAT IS NOT THERE. `runs.reconcileOne` gives each run its own budget
//     and counts SPENDING it as a failed reconciliation, so a stop that blocks inside a pass buys the
//     run a deferral and, after enough passes, an operator's stalled alarm — for doing exactly what it
//     was asked to do. It also eats that pass's share for every other account (D39.236 §5).
//
// Nothing is lost by not waiting: the reconciler re-issues the stop of any live run it finds with an
// intent on file, which is what closes the case this could not close anyway — a platform that died
// between committing the intent and calling systemd.
//
// ⚠ AND THE RE-ISSUE IS SAFE FOR A MEASURED REASON, not an assumed one. On a unit already in
// `deactivating` a second `stop` sends NO further signal and does NOT restart TimeoutStopSec:
// measured with three stops five seconds apart against a process that ignores SIGTERM — one signal in
// the process's own log, and the unit died at t+6.04 s from the FIRST stop rather than the last. That
// is what keeps the sweep's re-issue from pushing the SIGKILL backstop out of reach on every pass,
// and it is undocumented systemd behaviour, so it is pinned rather than trusted:
// TestARepeatedStopNeitherSignalsNorExtendsTheGrace.
func (r *Runner) Stop(ctx context.Context, unit string) error {
	out, err := r.run(ctx, "systemctl", "--user", "stop", "--no-block", unitName(unit))
	if err == nil {
		return nil
	}
	// The same two spellings answer a `--no-block` stop as answer a blocking one — measured, because a
	// flag that changed the refusal text would turn "the run is already gone" into an error the
	// reconciler retries forever.
	if bytes.Contains(out, []byte("not loaded")) || bytes.Contains(out, []byte("not found")) {
		return nil
	}
	return fmt.Errorf("runner: stop %s: %w: %s", unit, err, bytes.TrimSpace(out))
}

// Alive reports whether the unit is still running.
//
// ⚠ A false here means "systemd does not have this unit", which covers a finished run, a collected
// failure and a name that never existed. It is evidence, never a verdict: what the run DID is read
// from the exit marker and from Postgres.
func (r *Runner) Alive(ctx context.Context, unit string) (bool, error) {
	out, err := r.run(ctx, "systemctl", "--user", "show", unitName(unit), "--property=ActiveState", "--value")
	if err != nil {
		// `show` answers for unknown units too (ActiveState=inactive), so a non-zero exit is the bus
		// being unreachable — which must not be read as "the run is gone".
		return false, fmt.Errorf("runner: show %s: %w: %s", unit, err, bytes.TrimSpace(out))
	}
	switch strings.TrimSpace(string(out)) {
	case "active", "activating", "deactivating", "reloading":
		return true, nil
	default:
		return false, nil
	}
}

func (r *Runner) log() *slog.Logger {
	if r.Log == nil {
		return slog.New(slog.DiscardHandler)
	}
	return r.Log
}

func unitName(u string) string {
	if strings.HasSuffix(u, ".service") {
		return u
	}
	return u + ".service"
}

// quoteArgv renders argv the way a systemd command line takes it: arguments containing whitespace
// are double-quoted, and a literal quote or backslash is escaped. Written out rather than glued
// with spaces because a state directory with a space in it would otherwise turn one argument into
// two and the marker would be written somewhere else, silently.
//
// ⚠ `$` is deliberately NOT escaped, and that is a MEASUREMENT rather than an omission (register row
// PD-165, which raised it as an unverified suspicion). On systemd 259, through the exact call this
// package makes — `systemd-run --user --property=ExecStopPost=…` — a `$dir` in the path arrived
// LITERALLY, and it still did with `--setenv=dir=EXPANDED` on the same unit: the marker landed in
// `…/dollar$dir/`, never in `…/dollarEXPANDED/`. Escaping it to `$$` would therefore be the bug:
// the marker would go to a path with two dollars in it. Same finding as `%` (STACK_DECISIONS §17) —
// values of `--property=` are not put through systemd's own substitutions. If a later systemd
// changes that, this comment is where to start.
func quoteArgv(argv []string) string {
	parts := make([]string, 0, len(argv))
	for _, a := range argv {
		if !strings.ContainsAny(a, " \t\n\"'\\") {
			parts = append(parts, a)
			continue
		}
		esc := strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(a)
		parts = append(parts, `"`+esc+`"`)
	}
	return strings.Join(parts, " ")
}