437 lines
17 KiB
Go
437 lines
17 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
|
|
"textmachine/platform/internal/ingest"
|
|
"textmachine/platform/internal/money"
|
|
"textmachine/platform/internal/pgstore"
|
|
"textmachine/platform/internal/runner"
|
|
)
|
|
|
|
// fakeRunner stands in for systemd. What these tests are about is the reconciler's DECISION — start,
|
|
// wait, restart, finish — and every one of those decisions is taken from three facts: the exit
|
|
// marker, whether the unit is alive, and how long ago the attempt was admitted.
|
|
type fakeRunner struct {
|
|
mu sync.Mutex
|
|
started []runner.Spec
|
|
stopped []string
|
|
alive bool
|
|
aliveErr error
|
|
startErr error
|
|
// onStop runs INSIDE the fake's Stop, which is what makes the ORDER of "write the intent, then
|
|
// ask systemd" assertable: by the time this runs, the database must already know.
|
|
onStop func(unit string)
|
|
}
|
|
|
|
func (f *fakeRunner) Start(_ context.Context, s runner.Spec) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.startErr != nil {
|
|
return f.startErr
|
|
}
|
|
f.started = append(f.started, s)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRunner) Stop(_ context.Context, unit string) error {
|
|
f.mu.Lock()
|
|
hook := f.onStop
|
|
f.stopped = append(f.stopped, unit)
|
|
f.mu.Unlock()
|
|
if hook != nil {
|
|
hook(unit)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRunner) stops() []string {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return append([]string(nil), f.stopped...)
|
|
}
|
|
|
|
func (f *fakeRunner) Alive(context.Context, string) (bool, error) { return f.alive, f.aliveErr }
|
|
|
|
func (f *fakeRunner) starts() []runner.Spec {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return append([]runner.Spec(nil), f.started...)
|
|
}
|
|
|
|
// fakeEngine stands in for `tmctl status --json`. Guarded because the spawn reads the book's spend
|
|
// baseline, and several spawners legitimately race.
|
|
type fakeEngine struct {
|
|
mu sync.Mutex
|
|
report ingest.StatusReport
|
|
err error
|
|
calls int
|
|
// hangs is the workdir the engine refuses to answer about.
|
|
hangs string
|
|
// onStatus runs INSIDE the call, which is how a test opens the window the reconciler decides in:
|
|
// a status call is seconds of the engine's CPU, and a user can press stop inside it.
|
|
onStatus func()
|
|
}
|
|
|
|
func (f *fakeEngine) Status(ctx context.Context, _, workdir string) (ingest.StatusReport, error) {
|
|
f.mu.Lock()
|
|
f.calls++
|
|
hang := f.hangs != "" && f.hangs == workdir
|
|
hook := f.onStatus
|
|
rep, err := f.report, f.err
|
|
f.mu.Unlock()
|
|
if hook != nil {
|
|
hook()
|
|
}
|
|
if hang {
|
|
// An engine that never answers, which is what a hung `tmctl status` looks like from here. It
|
|
// ends when the CALLER's budget does, and that is the property being measured.
|
|
<-ctx.Done()
|
|
return ingest.StatusReport{}, ctx.Err()
|
|
}
|
|
return rep, err
|
|
}
|
|
|
|
// block makes the engine hang for one book's directory. The run id is taken only to make the call
|
|
// site read like the thing it models.
|
|
func (f *fakeEngine) block(_ string, workdir string) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.hangs = workdir
|
|
}
|
|
|
|
func (f *fakeEngine) set(rep ingest.StatusReport, err error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.report, f.err = rep, err
|
|
}
|
|
|
|
func (f *fakeEngine) called() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.calls
|
|
}
|
|
|
|
// usd is a figure the engine DID report, as opposed to one it left out. The status report carries
|
|
// both money figures as pointers precisely so the two can be told apart (PD-40), so a test that
|
|
// wants "the engine said zero" has to say so with a pointer.
|
|
func usd(v money.MicroUSD) *money.MicroUSD { return &v }
|
|
|
|
func decideOutcome(t *testing.T, paused string, m runner.Marker) string {
|
|
t.Helper()
|
|
status, _ := outcome(pgstore.LiveRun{PausedReason: paused}, m)
|
|
return status
|
|
}
|
|
|
|
// The engine's exit contract, and the two ways a process can end that have no exit code at all.
|
|
func TestWhatTheUnitDidBecomesTheProductStatus(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
paused string
|
|
marker runner.Marker
|
|
want string
|
|
}{
|
|
{"clean", "", runner.Marker{Result: "exit-code", Code: "exited", Status: "0"}, "ready"},
|
|
{"completed with flagged units", "", runner.Marker{Result: "exit-code", Code: "exited", Status: "2"}, "ready"},
|
|
{"bank signing stop", "", runner.Marker{Result: "exit-code", Code: "exited", Status: "3"}, "awaiting_bank"},
|
|
{"infra failure", "", runner.Marker{Result: "exit-code", Code: "exited", Status: "1"}, "failed"},
|
|
{"we asked it to stop", "", runner.Marker{Result: "success", Code: "killed", Status: "TERM"}, "stopped"},
|
|
{"the kernel took it", "", runner.Marker{Result: "oom-kill", Code: "killed", Status: "TERM"}, "failed"},
|
|
{"it ran out of time", "", runner.Marker{Result: "timeout", Code: "killed", Status: "KILL"}, "failed"},
|
|
// A ceiling halt is `paused`, never `failed`, however the process then ended: the stop is
|
|
// resumable and saying "failed" lies about that (contract §BookStatus).
|
|
{"a ceiling halt survives any exit", "credit_exhausted",
|
|
runner.Marker{Result: "exit-code", Code: "exited", Status: "1"}, "paused"},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := decideOutcome(t, tc.paused, tc.marker); got != tc.want {
|
|
t.Errorf("%s: %q, want %q", tc.name, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func service(t *testing.T, rn UnitRunner, eng EngineStatus, now time.Time) *Service {
|
|
t.Helper()
|
|
tpl, err := runner.ParseCeilingTemplate("--ceiling-usd {{usd}}")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return &Service{
|
|
Runner: rn, Engine: eng,
|
|
Cfg: Config{
|
|
StateDir: t.TempDir(), EngineBinary: "/opt/engine/2026.08.01/tmctl",
|
|
MarkerArgv: []string{"/usr/local/bin/tmplatformctl", "exit-marker"},
|
|
Ceiling: tpl, ResyncEvery: time.Minute,
|
|
},
|
|
Now: func() time.Time { return now },
|
|
}
|
|
}
|
|
|
|
// "No unit and no marker" is two different situations and the discriminator is time: systemd has not
|
|
// got to it yet, or the whole manager went away with the machine. Reading the second as the first
|
|
// means an interrupted run never comes back; reading the first as the second means a second engine
|
|
// on the same book.
|
|
func TestAnAdmittedRunIsGivenTimeBeforeItIsPresumedLost(t *testing.T) {
|
|
now := time.Now()
|
|
rn := &fakeRunner{}
|
|
svc := service(t, rn, nil, now)
|
|
// The grace belongs to the ATTEMPT: a restarted attempt inherits a run start hours old, and a
|
|
// grace measured from the run would expire before systemd had a chance to create anything.
|
|
fresh := pgstore.LiveRun{RunID: "r1", UnitName: "tm-run-r1-1",
|
|
StartedAt: now.Add(-3 * time.Hour), AttemptStartedAt: now.Add(-5 * time.Second)}
|
|
if err := svc.reconcile(t.Context(), fresh); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(rn.starts()) != 0 {
|
|
t.Errorf("a run admitted five seconds ago was already presumed lost")
|
|
}
|
|
}
|
|
|
|
// A unit that was never recorded means the platform stopped between admitting the run and creating
|
|
// it — or the queue entry was lost. The reconciler is the backstop for both, which is what makes the
|
|
// queue's own retry unnecessary.
|
|
func TestTheUnitAnAttemptGetsCarriesItsPinnedBinaryAndItsCeiling(t *testing.T) {
|
|
now := time.Now()
|
|
svc := service(t, &fakeRunner{}, nil, now)
|
|
l := pgstore.LiveRun{
|
|
RunID: "r1", BookID: "bk1", AttemptID: 7, AttemptNo: 1, Workdir: t.TempDir(),
|
|
Ceiling: 3_000_000, VerifyBank: true, StartedAt: now,
|
|
}
|
|
// $3 of increment on a book that has already committed $1, with $0.50 left reserved by a process
|
|
// that died: the engine is told the CUMULATIVE cap it actually judges against, and the leftover is
|
|
// NOT part of it — the engine clears it on the write open (meter.bookCap, PD-158).
|
|
spec, err := svc.spec(l, meter{committed: 1_000_000, reserved: 500_000}.bookCap(l.Ceiling))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if spec.Unit != "tm-run-r1-1" {
|
|
t.Errorf("unit %q: the attempt number is part of the name, or a resume cannot reuse it", spec.Unit)
|
|
}
|
|
if spec.Binary != "/opt/engine/2026.08.01/tmctl" {
|
|
t.Errorf("the attempt was not pinned to the configured engine build: %q", spec.Binary)
|
|
}
|
|
if len(spec.Env) != 0 {
|
|
t.Errorf("the platform passed an environment to the engine: %v — provider keys travel in the book's own .env", spec.Env)
|
|
}
|
|
wantCeiling := false
|
|
for i, a := range spec.Args {
|
|
if a == "--ceiling-usd" && i+1 < len(spec.Args) && spec.Args[i+1] == "4.000000" {
|
|
wantCeiling = true
|
|
}
|
|
}
|
|
if !wantCeiling {
|
|
t.Errorf("the run's ceiling did not reach the engine as a cumulative cap: %v", spec.Args)
|
|
}
|
|
if spec.MarkerArgv[len(spec.MarkerArgv)-1] != spec.Unit || spec.MarkerArgv[len(spec.MarkerArgv)-2] != spec.ExitMarker {
|
|
t.Errorf("the marker command does not name its own file and unit: %v", spec.MarkerArgv)
|
|
}
|
|
}
|
|
|
|
// A stale marker from an earlier attempt of the same name would be read as this attempt's ending the
|
|
// moment the reconciler looked — settling money against a result nobody produced.
|
|
func TestEachAttemptHasAMarkerOfItsOwn(t *testing.T) {
|
|
svc := service(t, &fakeRunner{}, nil, time.Now())
|
|
if svc.markerPath("r1", 1) == svc.markerPath("r1", 2) {
|
|
t.Fatal("two attempts of one run share an exit marker: the first one's ending would be read as the second's")
|
|
}
|
|
if !strings.HasPrefix(svc.markerPath("r1", 1), svc.Cfg.StateDir) {
|
|
t.Errorf("the marker is not in the platform's state directory: %s", svc.markerPath("r1", 1))
|
|
}
|
|
}
|
|
|
|
// A run cannot be started at all while the engine has no way to be told its ceiling: it would spend
|
|
// under the BOOK's limit instead of the account's (row 145).
|
|
func TestNoRunIsSpawnedUntilTheEngineCanBeToldItsCeiling(t *testing.T) {
|
|
now := time.Now()
|
|
rn := &fakeRunner{}
|
|
svc := service(t, rn, nil, now)
|
|
svc.Cfg.Ceiling = nil
|
|
_, err := svc.spec(pgstore.LiveRun{RunID: "r1", AttemptNo: 1, Workdir: t.TempDir(), Ceiling: 1_000_000}, 1_000_000)
|
|
if !errors.Is(err, runner.ErrCeilingNotWired) {
|
|
t.Fatalf("building a unit without a ceiling argument gave %v", err)
|
|
}
|
|
if len(rn.starts()) != 0 {
|
|
t.Fatal("a unit was started with no ceiling")
|
|
}
|
|
}
|
|
|
|
// A deployment that cannot start runs at all must find that out for FREE. The spawn asks the engine
|
|
// for the book's money meter, and that call re-ingests and re-chunks the source (seconds of CPU per
|
|
// call, unified backlog row 100) — paying it for every run on every sweep before arriving at a
|
|
// refusal that depends on nothing but configuration is a cost with no purchase.
|
|
func TestAMisconfiguredDeploymentIsRefusedWithoutAskingTheEngine(t *testing.T) {
|
|
eng := &fakeEngine{report: ingest.StatusReport{Spend: usd(0), Reserved: usd(0)}}
|
|
svc := service(t, &fakeRunner{}, eng, time.Now())
|
|
svc.Cfg.Ceiling = nil
|
|
err := svc.spawnAttempt(t.Context(),
|
|
pgstore.LiveRun{RunID: "r1", AttemptNo: 1, Workdir: t.TempDir(), Ceiling: 1_000_000})
|
|
if !errors.Is(err, runner.ErrCeilingNotWired) {
|
|
t.Fatalf("spawning under a deployment with no ceiling argument gave %v", err)
|
|
}
|
|
if eng.called() != 0 {
|
|
t.Errorf("the engine was asked %d times before a refusal that needed no answer", eng.called())
|
|
}
|
|
}
|
|
|
|
// "I could not ask systemd" must never be read as "the run is gone": that mistake starts a second
|
|
// engine against a project file the first one holds an exclusive lock on.
|
|
func TestAnUnreachableBusNeverCausesARestart(t *testing.T) {
|
|
now := time.Now()
|
|
rn := &fakeRunner{aliveErr: errors.New("Failed to connect to bus")}
|
|
svc := service(t, rn, nil, now)
|
|
l := pgstore.LiveRun{RunID: "r1", UnitName: "tm-run-r1-1", AttemptNo: 1,
|
|
StartedAt: now.Add(-time.Hour), AttemptStartedAt: now.Add(-time.Hour)}
|
|
if err := svc.reconcile(t.Context(), l); err == nil {
|
|
t.Fatal("an unreachable bus was treated as a decision")
|
|
}
|
|
if len(rn.starts()) != 0 {
|
|
t.Fatal("a run was restarted on the strength of a question that was never answered")
|
|
}
|
|
}
|
|
|
|
// The resync is a SLOW poll and the interval is what makes it affordable: every call re-ingests and
|
|
// re-chunks the source. A reconciler sweeping every fifteen seconds must not call it every time.
|
|
func TestALiveRunIsResyncedAtMostOncePerInterval(t *testing.T) {
|
|
now := time.Now()
|
|
eng := &fakeEngine{report: ingest.StatusReport{TotalUnits: 10, Done: 3}}
|
|
rn := &fakeRunner{alive: true}
|
|
svc := service(t, rn, eng, now)
|
|
if !svc.dueForResync("r1") {
|
|
t.Fatal("a run that was never resynced is not due")
|
|
}
|
|
if svc.dueForResync("r1") {
|
|
t.Fatal("a second sweep inside the interval asked the engine again")
|
|
}
|
|
if !svc.dueForResync("r2") {
|
|
t.Fatal("the interval is per run, not global")
|
|
}
|
|
svc.Now = func() time.Time { return now.Add(2 * time.Minute) }
|
|
if !svc.dueForResync("r1") {
|
|
t.Fatal("the interval never expires")
|
|
}
|
|
// With no engine wired there is nothing to ask, and asking is what costs.
|
|
svc.Engine = nil
|
|
if svc.dueForResync("r3") {
|
|
t.Fatal("a resync was scheduled with no engine to ask")
|
|
}
|
|
_ = eng
|
|
_ = rn
|
|
}
|
|
|
|
// A status call that fails is not a run that failed.
|
|
func TestAFailedResyncIsNotAFailedRun(t *testing.T) {
|
|
now := time.Now()
|
|
eng := &fakeEngine{err: errors.New("tmctl: config: no such file")}
|
|
svc := service(t, &fakeRunner{alive: true}, eng, now)
|
|
// The store is never reached: a status call that failed has nothing to materialize, so this
|
|
// exercises the branch that must not turn a repair-channel outage into a run failure.
|
|
if err := svc.maybeResync(t.Context(), pgstore.LiveRun{RunID: "r1", Workdir: t.TempDir()}); err != nil {
|
|
t.Fatalf("a failed status call was reported as a reconciliation failure: %v", err)
|
|
}
|
|
if eng.called() != 1 {
|
|
t.Fatalf("the engine was asked %d times", eng.called())
|
|
}
|
|
}
|
|
|
|
// The journal that does not exist yet is the ordinary case today: the emitter is engine work (row
|
|
// 103), and every sweep of every run meets it.
|
|
func TestAnAbsentJournalIsNotAReconciliationFailure(t *testing.T) {
|
|
now := time.Now()
|
|
svc := service(t, &fakeRunner{alive: true}, nil, now)
|
|
l := pgstore.LiveRun{RunID: "r1", BookID: "bk1", Workdir: t.TempDir()}
|
|
if _, err := svc.drainJournal(t.Context(), l); err != nil {
|
|
t.Fatalf("an absent journal: %v", err)
|
|
}
|
|
}
|
|
|
|
// A quarantined attempt is not read at all: materialization stopped on purpose and the run's
|
|
// freshness now comes from the resync channel.
|
|
func TestAQuarantinedAttemptIsNotTailed(t *testing.T) {
|
|
now := time.Now()
|
|
svc := service(t, &fakeRunner{alive: true}, nil, now)
|
|
dir := t.TempDir()
|
|
// A journal that would fail loudly if it were read.
|
|
if err := os.WriteFile(filepath.Join(dir, ingest.JournalFile), []byte("not json at all\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
l := pgstore.LiveRun{RunID: "r1", BookID: "bk1", Workdir: dir, Quarantined: true}
|
|
if _, err := svc.drainJournal(t.Context(), l); err != nil {
|
|
t.Fatalf("a quarantined attempt was tailed anyway: %v", err)
|
|
}
|
|
}
|
|
|
|
// "The journal could not be materialized" is two facts, and treating them alike is expensive in both
|
|
// directions: retrying a malformed line forever wedges a run, and quarantining on a lock Postgres
|
|
// broke blinds the projection of a live, paying run permanently.
|
|
//
|
|
// ⚠ The second half was measured by acceptance: a deadlock out of the materializer reached the
|
|
// reconciler as an unreadable journal.
|
|
func TestOnlyAJournalWeCannotReadStopsTheProjection(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
err error
|
|
want bool
|
|
}{
|
|
{"a gap in the stream", fmt.Errorf("%w: seq 9 after 7", ingest.ErrStreamGap), true},
|
|
{"the same seq saying something else", ingest.ErrPayloadConflict, true},
|
|
{"a line that is not an event", errors.New("ingest: decode line 4: invalid character"), true},
|
|
{"our own shutdown", context.Canceled, false},
|
|
{"a sweep that ran out of time", fmt.Errorf("tail: %w", context.DeadlineExceeded), false},
|
|
{"a deadlock Postgres broke", &pgconn.PgError{Code: "40P01", Message: "deadlock detected"}, false},
|
|
{"a serialization failure", fmt.Errorf("pgstore: move cursor: %w",
|
|
&pgconn.PgError{Code: "40001", Message: "could not serialize access"}), false},
|
|
// A managed database is RESTARTED on a schedule, and every shape that arrives in is one the
|
|
// next sweep reads through. Missing them left the same permanent blindness the deadlock did.
|
|
{"the server is shutting down", &pgconn.PgError{Code: "57P01", Message: "terminating connection due to administrator command"}, false},
|
|
{"a sibling backend crashed", &pgconn.PgError{Code: "57P02", Message: "crash shutdown"}, false},
|
|
{"the server is not accepting connections yet", &pgconn.PgError{Code: "57P03", Message: "cannot connect now"}, false},
|
|
{"the connection failed", &pgconn.PgError{Code: "08006", Message: "connection failure"}, false},
|
|
{"the connection was reset", fmt.Errorf("pgstore: lock attempt: %w",
|
|
&net.OpError{Op: "read", Err: errors.New("connection reset by peer")}), false},
|
|
{"a column that does not exist", &pgconn.PgError{Code: "42703", Message: "no such column"}, true},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := quarantines(tc.err); got != tc.want {
|
|
t.Errorf("%s: quarantines=%v, want %v", tc.name, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The unit name carries the attempt: systemd will not accept a name that is still loaded, and a
|
|
// resume is a new process.
|
|
func TestTheUnitNameCarriesTheAttempt(t *testing.T) {
|
|
if got := unitName("run_ABC", 2); got != "tm-run-run_ABC-2" {
|
|
t.Errorf("unit name %q", got)
|
|
}
|
|
}
|
|
|
|
// The journal offset an attempt starts from is the size of the file when it was admitted: the
|
|
// journal is per BOOK and append-only, so its own lines begin after everything already there.
|
|
func TestAnAttemptStartsWhereTheJournalAlreadyEnds(t *testing.T) {
|
|
dir := t.TempDir()
|
|
if got, err := journalSize(dir); err != nil || got != 0 {
|
|
t.Fatalf("an absent journal gave %d, %v; want 0 and no error", got, err)
|
|
}
|
|
body := []byte("{\"seq\":1}\n{\"seq\":2}\n")
|
|
if err := os.WriteFile(filepath.Join(dir, ingest.JournalFile), body, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := journalSize(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got != int64(len(body)) {
|
|
t.Errorf("journal size %d, want %d", got, len(body))
|
|
}
|
|
}
|