package pgstore import ( "context" "errors" "fmt" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "textmachine/platform/internal/money" ) // Tx is a transaction of this store. Exported so a caller can join its own write to one — the queue // does, because a run row and its queue entry must be written together — without every package in // the zone importing pgx to name the type. type Tx = pgx.Tx var ( // ErrRunInFlight is a second live run on one book. The database refuses it (the partial unique // index runs_one_live_per_book) rather than the API explaining it: the engine holds an EXCLUSIVE // lock on the project file, so two live runs is not a state anything downstream can represent. ErrRunInFlight = errors.New("pgstore: the book already has a live run") // ErrNoRun is a run that does not exist or does not belong to the caller. ErrNoRun = errors.New("pgstore: no such run") ) // StartRunInput is one accepted run request. type StartRunInput struct { UserID string BookID string VerifyBank bool OrderedChapters int // Ceiling is what those chapters are worth and what is HELD before anything is spawned. The hold // and the ceiling handed to the engine are the same number by construction: the hold makes the // credit unavailable to any other run, and the engine stops itself there, so an overspend is // impossible even while the platform is blind (D39.100). Ceiling money.MicroUSD Now time.Time // Resnapshot/AcceptRebill are the re-pass consents, decided ONCE at admission and stored on the // run the same way VerifyBank is: every spawn of every attempt reads the row, so argv is stable // across respawns and a flag cannot appear mid-run (P10 §3.1). AcceptRebill is the CONCRETE sum // the platform consents to (`--accept-rebill=`) — the projection the user was shown, never // a blanket yes (D20.2-Q2). Resnapshot bool AcceptRebill money.MicroUSD // OrderedUnits is set for every run whose order was phrased in CHARACTERS, and nil for every other // shape. Nil is the ordinary case and leaves the run's bar counted in chapters, exactly as before // this pack. See migration 00033 for why such a run needs a bar of its own: counted in chapters an // order that stops inside a chapter reads `0/N` for its whole life, which is the state the // admission refuses a book for. // // ⚠ THE CONDITION IS THE PHRASING, NOT WHERE THE ORDER LANDED. An earlier edition of this line // said «only for a run whose order does not close whole chapters», and that is measurably false — // see pricing.Quote.UnitShaped for the measurement and for why the phrasing is the right test. OrderedUnits *int // BondFunded is whether this run's hold has room for the book-level consistency passes on top of // the work it bought. Recorded on the RUN because the order form's answer is a QUOTE and the // balance moves under it — see migration 00033. BondFunded bool // Order is THE ORDER, and it is written on the BOOK rather than on the run (unified backlog // row 280). // // It lived on the run because a ceiling was a property of one process — and that is exactly what // made «raise the limit» mean «start a NEW run with a bigger ceiling», a sentence nobody could // explain to a buyer. On the book, continuing after a top-up is the same order again, and the // re-pass door stands beside it as a second KIND of order rather than as a special case. // // ⛔ NIL MEANS «LEAVE THE BOOK'S ORDER ALONE», and it is not a convenience. The RE-PASS buys no // volume — it re-makes what a correction touched — so it has no order to record; and the zero // VALUE of BookOrder means «the whole book», which written over an existing boundary silently // WIDENS a paid order: «bought through chapter 12» became «bought the whole book», with nothing // said and nothing charged for the difference. A purchase that buys no volume must not be able to // change what was bought. Order *BookOrder } // BookOrder is what a book is bought through: an IDENTITY of the boundary, never a count and never // an ordinal. See migration 00033 for the whole of why — in one line, an ordinal survives a re-cut // by silently naming different text, and that is a change of what was bought that nobody sees. type BookOrder struct { // ThroughChapterID is the last chapter bought, for an order phrased in chapters. Empty for the // whole book and for a character order. ThroughChapterID string // ThroughUnitID is the last output unit bought, for an order phrased in characters — which stops // inside a chapter, so no chapter identity can express where. Empty otherwise. ThroughUnitID string // ThroughChapterNumber is the ordinal AT THE MOMENT OF THE ORDER: a label for the screen, and the // thing whose disagreement with the identity is what makes a shift visible. Zero when there is // none. ThroughChapterNumber int } // Whole reports the whole-book order — the ratified default, and the only order with no boundary. func (o BookOrder) Whole() bool { return o.ThroughChapterID == "" && o.ThroughUnitID == "" } // StartedRun is what the caller needs after a successful start. type StartedRun struct { Run AttemptID int64 // JournalOffset is the size of the book's journal at the moment this attempt was admitted. The // journal is per BOOK and append-only, so an attempt's own lines begin after everything already // in it; without this the tailer would adopt the previous attempt's handshake. JournalOffset int64 } // StartRun admits a run: the run row, its first attempt, the hold and the queue entry, all in ONE // transaction. // // One transaction because each of them alone is a way to lose money or work. A hold without a run // row is credit reserved for nothing; a run row without a hold is a run that spends credit nobody // set aside; a queue entry without either is a worker spawning an engine against a book whose // bookkeeping does not exist. research/25 names the first of those explicitly ("a hold can leak // before the directory exists"), and the answer is not a compensating sweep but not splitting the // write in the first place. // // enqueue is handed the transaction so the queue's own insert joins it. It is a callback rather than // a second call because River owns its SQL and this package owns its own: neither reaches into the // other, and the atomicity is still real. func (s *Store) StartRun(ctx context.Context, in StartRunInput, journalOffset int64, enqueue func(context.Context, Tx, string) error) (StartedRun, error) { // Zero is ONE legal shape: the re-pass run (P10), which buys no chapters — it re-walks the book // under the consents, so Resnapshot is what marks it. Everything else still needs a positive // ceiling; the bar of a zero-ceiling row switches to the re-pass form (readmodel.runTotal), so // the 0/0 frame stays unreachable. if in.OrderedChapters < 0 || (in.OrderedChapters == 0 && !in.Resnapshot) { return StartedRun{}, fmt.Errorf("pgstore: a run needs a positive chapter ceiling, got %d", in.OrderedChapters) } out := StartedRun{JournalOffset: journalOffset} err := s.inTx(ctx, func(tx pgx.Tx) error { // The book first, before the hold — see lockBook for why the order is global and not local. // Here it also makes the race between two admissions of one book a queue instead of a // collision: the loser still gets ErrRunInFlight, it just gets it after waiting. if err := lockBook(ctx, tx, in.BookID); err != nil { return err } runID := newID("run") // BOTH baselines are captured HERE, in the same statement that creates the run: what the book // had already finished is what each half of this run's bar is measured from (see runDone) — // `chapters_before` on the EDIT column, `draft_before` on the DRAFT one. Neither is re-based // afterwards: the bar is one monotonic fraction through both waves, and a base that moved // mid-run is how it used to restart from zero at the signing stop. // // ⚠ FIXED predicates, deliberately NOT the flag-following finishedUnits: `edit_wave` can flip // false→true AFTER this insert (the engine announces its wave shape with its first progress // event), and a baseline captured through the flag would sit on the draft column while the // numerator moved to the edit one — the bar then read 0/N forever (reviewer's blocker, P9). // Captured flag-free, each baseline is subtracted only from the numerator of its own column // (runDone pairs them at READ time), so no flip can strand the bar. const insertRun = ` insert into runs (id, book_id, status, verify_bank, ceiling_chapters, started_at, revision, resnapshot, accept_rebill_micro, bond_funded, ordered_units, chapters_before, draft_before, units_before, draft_units_before) select $1, $2, 'translating', $3, $4, $5, b.revision + 1, $7, $8, $9, $10, (select count(*) from chapters c where c.book_id = b.id and c.units_total > 0 and c.units_edit_done >= c.units_total), (select count(*) from chapters c where c.book_id = b.id and c.units_total > 0 and c.units_draft_done >= c.units_total), ` + bookUnitsEditDone + `, ` + bookUnitsDraftDone + ` from books b where b.id = $2 and b.owner_id = $6 returning id, book_id, revision, status, verify_bank, (case when ordered_units is not null then null else ceiling_chapters end), ordered_units, coalesce(paused_reason, ''), started_at, finished_at` err := tx.QueryRow(ctx, insertRun, runID, in.BookID, in.VerifyBank, in.OrderedChapters, in.Now, in.UserID, in.Resnapshot, int64(in.AcceptRebill), in.BondFunded, in.OrderedUnits). Scan(&out.ID, &out.BookID, &out.Revision, &out.Status, &out.VerifyBank, &out.OrderedChapters, &out.OrderedUnits, &out.PausedReason, &out.StartedAt, &out.FinishedAt) if errors.Is(err, pgx.ErrNoRows) { return ErrNoBook // the book is missing, or it is not this account's } if isUnique(err, "runs_one_live_per_book") { return ErrRunInFlight } if err != nil { return fmt.Errorf("pgstore: insert run: %w", err) } // The receipt's bar is READ rather than zeroed: the numerator opens at 0 by construction // (both halves measure THIS run's work from the baselines the same statement just took), but // the DENOMINATOR is the run's own — a continuation run over a drafted backlog owes fewer // draft passes (runTotal folds draftWork in), and the screen reads its scale against it. if err := tx.QueryRow(ctx, `select `+runDone+`, `+runTotal+`, `+runStage+` from books b `+lastRun+` where b.id = $1`, in.BookID). Scan(&out.Progress.Done, &out.Progress.Total, &out.Progress.Stage); err != nil { return fmt.Errorf("pgstore: read the new run's bar: %w", err) } if err := tx.QueryRow(ctx, ` insert into run_attempts (run_id, attempt_no, started_at, last_offset, engine_run_id) values ($1, 1, $2, $3, $4) returning id`, out.ID, in.Now, journalOffset, EngineStreamID(out.ID, 1)).Scan(&out.AttemptID); err != nil { return fmt.Errorf("pgstore: insert attempt: %w", err) } // The hold is taken here, BEFORE anything is spawned, and it is the enforcement half of the // money design rather than an accounting note. if err := holdTx(ctx, tx, in.UserID, in.BookID, engineRunKey(out.ID, 1), in.Ceiling, in.Now); err != nil { return err } // THE ORDER IS RECORDED WITH THE ADMISSION, in the transaction that took the money for it. A // later write would leave a window in which a run is live and the book cannot say what was // bought — and that is precisely the question every spawn of every attempt asks, to compute // `--max-units`. // ⚠ THE ORDER IS WRITTEN ONLY WHEN THERE IS ONE. A purchase that buys no volume (the re-pass) // leaves the book's boundary exactly as it stands: writing its zero value over an existing one // would turn «bought through chapter 12» into «bought the whole book» — see StartRunInput.Order. if in.Order != nil { if _, err := tx.Exec(ctx, ` update books set ordered_at = $2, ordered_through_chapter_id = nullif($3, ''), ordered_through_unit_id = nullif($4, ''), ordered_through_chapter_number = nullif($5, 0) where id = $1`, in.BookID, in.Now, in.Order.ThroughChapterID, in.Order.ThroughUnitID, in.Order.ThroughChapterNumber); err != nil { return fmt.Errorf("pgstore: record the order: %w", err) } } if _, err := tx.Exec(ctx, ` update books set status = 'translating', revision = `+nextRevisionOfThisBooksLibrary+` where id = $1`, in.BookID); err != nil { return fmt.Errorf("pgstore: mark book translating: %w", err) } if err := emitStatus(ctx, tx, in.BookID); err != nil { return err } if enqueue != nil { return enqueue(ctx, tx, out.ID) } return nil }) if err != nil { return StartedRun{}, err } return out, nil } // engineRunKey is the reservation key of an attempt. // // ⚠ It is the PLATFORM's attempt identity, not the engine's `engine_run_id`: the engine mints its // own id per invocation and the platform only learns it from the handshake, which arrives after the // money has already been reserved. Reservations are keyed by attempt for exactly that reason — the // hold has to exist before there is anything to key it by on the engine's side. func engineRunKey(runID string, attempt int) string { return fmt.Sprintf("%s#%d", runID, attempt) } // EngineStreamID is the identity this attempt's engine announces its event stream under — the // `engine_run_id` half of the ratified idempotency key. The PLATFORM chooses it and hands it to the // engine in the environment (TM_TRACE_ID, unified backlog row 102). // // ⚠ It is written when the ATTEMPT ROW IS CREATED and not when the unit is, and that is the whole // point. The journal is per BOOK, and a reader with no name of its own adopts the first handshake it // meets at its offset — so between admission and spawn (seconds inside a `tmctl status` call, or a // whole sweep interval) a stream belonging to somebody else could be adopted, its `ceiling` event // materialized onto this attempt, and the attempt's own handshake then refused as a sequence gap. // Naming the stream at creation closes the window rather than narrowing it. // // Per ATTEMPT and not per run: seq restarts at 1 in every process. The shape is what the engine // accepts without rewriting it — printable ASCII, no spaces, well under its 64-character bound. func EngineStreamID(runID string, attempt int) string { return fmt.Sprintf("tm-stream-%s-%d", runID, attempt) } func isUnique(err error, constraint string) bool { var pg *pgconn.PgError return errors.As(err, &pg) && pg.Code == "23505" && pg.ConstraintName == constraint } // LiveRun is a run the reconciler has to make a decision about. type LiveRun struct { RunID string BookID string UserID string Workdir string AttemptID int64 AttemptNo int UnitName string EngineRunID string Position Position Quarantined bool // Parked is what the last sweep recorded about this attempt's projection. Carried on the snapshot // so a pass whose verdict is unchanged issues no write at all. Parked bool VerifyBank bool // Resnapshot/AcceptRebill are the run's re-pass consents (P10): read by every spawn so the argv // is the admission's decision, never a sweep's re-derivation. AcceptRebill == 0 means no consent // was given (the flag is not passed); a consent is always a concrete sum. Resnapshot bool AcceptRebill money.MicroUSD // OrderedChapters is the run's whole budget, in the unit the user chose; Ceiling is what THIS // attempt was allowed to spend. They differ after a restart, which gets what is left. OrderedChapters int Ceiling money.MicroUSD // ⚠ THE ORDER IS DELIBERATELY NOT HERE. It used to be five more columns on this struct, and // this struct is read by `runColumns` — the reconciler's own list, for EVERY live run on EVERY // sweep. Resolving an order means walking the book's units, so carrying it here put three passes // over every unit of every live book into a query whose job is to say which runs exist, for // figures only the SPAWN reads. The spawn is rare and the sweep is not: it is read there, once, // by ReadOrderForSpawn. Found by an adversarial pass on the plan, not by a failure. // EngineBinary is the VERSIONED path this attempt was pinned to (unified backlog row 139). Read // back and USED — for the resume and for the repair channel — because a pin nothing reads is a // column, not a pin: the engine ships more often than a run finishes, and asking the CURRENT // binary about a book an older one is translating is a different question. EngineBinary string // SpendBaseline is what the BOOK had already cost when this attempt began. Nil means it was never // captured, which no current path produces (the spawn refuses without it). SpendBaseline *money.MicroUSD // CeilingArg is the limit a PREVIOUS claim of this attempt already handed the engine. Read back // because a retry must hand the same one: recomputing it from a counter that the first claim's own // engine has been moving gives that engine a second, larger limit and leaves the column that is // supposed to answer "what limit did that process have" describing neither. Zero means no claim // has been recorded (the column predates nothing else being able to tell). CeilingArg money.MicroUSD // PausedReason is what the stream already said about this run. A run that reported a ceiling halt // is `paused` however its process then ended (contract §BookStatus: never `failed`). PausedReason string // Status is the run's product status. The reconciler works on live runs, where it is // `translating`; the resume path reads a run that has already ended and decides by it. Status string // StopRequestedAt is when THIS platform asked the run to stop, and it is the only thing that can // tell a stop from a crash: the engine catches SIGTERM and exits 1, so the marker says // `exit-code/exited/1` for both (register row PD-152). Nil means nobody asked. StopRequestedAt *time.Time // ReconcileFailures is how many passes in a row have failed to reconcile THIS attempt. It is read // back so the next deferral can be computed from it, and it is on the attempt rather than the run // because a restart opens a new process that deserves a clean slate. ReconcileFailures int // StartedAt is the RUN's start; AttemptStartedAt is THIS attempt's. The reconciler's grace is // measured against the second: a restarted attempt inherits a start time hours old, and the // grace then expires before systemd has had a chance to create anything. StartedAt time.Time AttemptStartedAt time.Time } // Position mirrors ingest.Position without importing it: this package owns the columns, and the // dependency runs the other way. type Position struct { Offset int64 LastSeq int64 LastHash []byte } // ListLiveRuns returns every run that has not finished, with its live attempt. // // This is the reconciler's source of truth, together with the book's directory — NOT systemd // (research/25 §Опс). A transient unit does not survive a reboot and is unloaded the moment it // exits, so asking systemd "what is running" answers a different question than "what did this // platform promise a user". func (s *Store) ListLiveRuns(ctx context.Context) ([]LiveRun, error) { return s.queryRuns(ctx, ` join run_attempts a on a.run_id = r.id and a.ended_at is null left join reservations res on res.engine_run_id = r.id || '#' || a.attempt_no and res.state = 'open' where r.finished_at is null order by r.started_at`) } // RunsToReconcile is ListLiveRuns minus the attempts that are serving a deferral. // // The two lists are deliberately different and the difference is the whole of the starvation fix. // This one is what the sweep WORKS through, and an attempt the last pass could not finish is not in // it until its deferral lapses. ListLiveRuns stays whole because its other caller is telemetry // (`Lag`), and a run hidden from the number that says how far behind the projection is would be a // run nobody can see is stuck. // // Ordering is unchanged — oldest first — and it is only safe BECAUSE of the filter: a list ordered // the same way on every pass hands the head to whichever item is oldest, so an item that can never // succeed held that head forever. Deferral is what takes it out of the head rather than a different // ordering, because "oldest first" is right for everything that is merely slow. // // ⚠ A STOP OUTRANKS A DEFERRAL: the deferral is a statement about the past, and the sweep is the only // reader of live runs, so a stopped run sat at `translating` with its hold reserved until the backoff // lapsed — minutes on a run already deferred a few times, and against Stop's own promise to re-issue // on the next pass. func (s *Store) RunsToReconcile(ctx context.Context, now time.Time) ([]LiveRun, error) { return s.queryRuns(ctx, ` join run_attempts a on a.run_id = r.id and a.ended_at is null left join reservations res on res.engine_run_id = r.id || '#' || a.attempt_no and res.state = 'open' where r.finished_at is null and (a.reconcile_after is null or a.reconcile_after <= $1 or r.stop_requested_at is not null) order by r.started_at`, now) } // DeferRun records that a pass could not finish this attempt, and when the next one may try. // // It returns the number of CONSECUTIVE failures including this one, so the caller can decide the // next deadline from it and can say out loud when an attempt has stopped being merely slow. The // count lives in the database and not in the process because the process restarts — several times a // week by design (research/25) — and a counter that restarts with it never reaches any threshold. // // The reason goes through `truncateReason` — repaired to valid UTF-8 and then cut on a rune // boundary — and it is an operator's field, never a client's: it is a Go error string, which may // name a path, a systemd unit or the engine's own stderr, while the wire's vocabulary of failure is // a closed enum decided elsewhere (`failureReason`). See truncateReason for why the repair is not // decoration: a write that Postgres refuses here is a failure that never gets counted, and the // attempt then holds the head of the list forever. // ⚠ NOT restricted to live attempts: a finished one whose MONEY is still open holds the head of the // settlement list the same way, and the two phases select on opposite sides of `ended_at`. func (s *Store) DeferRun(ctx context.Context, attemptID int64, after time.Time, reason string) (int, error) { var failures int err := s.pool.QueryRow(ctx, ` update run_attempts set reconcile_failures = reconcile_failures + 1, reconcile_after = $2, reconcile_error = $3 where id = $1 returning reconcile_failures`, attemptID, after, truncateReason(reason)).Scan(&failures) if errors.Is(err, pgx.ErrNoRows) { // No such attempt. Nothing to defer, and nothing wrong. return 0, nil } if err != nil { return 0, fmt.Errorf("pgstore: defer the reconciliation of a run: %w", err) } return failures, nil } // AttemptEnded reports whether an attempt is over. It exists for one decision and says so: when the // RECONCILIATION phase writes a deferral after a failure, the attempt it is deferring may already // have ended during that same pass — `finish`, `finishStopped` and `restart` all close it and then // settle — and an ended attempt belongs to the SETTLEMENT phase's list, whose deferral is capped // shorter because it gates the user's own resume (runs.settlementBackoffCap). func (s *Store) AttemptEnded(ctx context.Context, attemptID int64) (bool, error) { var ended bool if err := s.pool.QueryRow(ctx, `select ended_at is not null from run_attempts where id = $1`, attemptID).Scan(&ended); err != nil { if errors.Is(err, pgx.ErrNoRows) { return false, nil } return false, fmt.Errorf("pgstore: read whether the attempt ended: %w", err) } return ended, nil } // ClearRunDeferral forgets the failures of an attempt that has just been reconciled successfully. // // Conditional on there being something to forget: the overwhelming majority of passes succeed, and // an unconditional write would put a row update on every live run on every tick — the cost the // deferral exists to avoid, paid back on the healthy path. func (s *Store) ClearRunDeferral(ctx context.Context, attemptID int64) error { if _, err := s.pool.Exec(ctx, ` update run_attempts set reconcile_failures = 0, reconcile_after = null, reconcile_error = null where id = $1 and (reconcile_failures <> 0 or reconcile_after is not null)`, attemptID); err != nil { return fmt.Errorf("pgstore: clear the deferral of a run: %w", err) } return nil } // StalledRun is a run the reconciler keeps failing on, as an operator needs to see it. type StalledRun struct { RunID string Title string // AttemptID and Spawns are the subject and generation of the proof `run abandon` takes from this // listing. See AbandonOrder.ProofAttemptID. AttemptID int64 Spawns int UnitName string AttemptNo int Status string Failures int NextTry *time.Time LastError string // HeldMicroUSD is what is still reserved against this run, and HeldSeconds how long. It is the // operator's whole question: a stalled run is money that is not moving and a book that cannot // start another run. HeldMicroUSD int64 HeldSeconds float64 // SpentMicroUSD is what the engine has reported spending on THIS attempt, as the stream last said. // It is here because of the decision this table exists to inform: `run abandon --release-hold` // gives the hold back WHOLE, and without this number that choice is made blind to how much work is // being written off. // // ⚠ IT IS A DIFFERENCE, not the column: `spend_micro_usd` holds the engine's figure verbatim, and // that figure is the BOOK's lifetime total (ingest.Spend, "CUMULATIVE"), so printing it raw told an // operator what the book had cost since upload and called it this attempt's. Same arithmetic as // `attemptSpend`, clamp included. // // ⚠ NIL IS "NOT KNOWN", never zero: an attempt with no baseline is the one `settle` refuses to // price at all, and a zero here is a figure an operator could write a hold off against. SpentMicroUSD *int64 // QuarantineReason is why the projection of this attempt stopped being materialized, or empty. // Here because the lift (`run unquarantine`) is a decision, and the reason is what it is made on. QuarantineReason string // ParkedAt is when this attempt's projection parked: the journal continued under a foreign stream // id and the tailer stopped there (ingest.ErrForeignStreamAhead), so freshness comes from the // repair channel alone. // // Separate from QuarantineReason because the remedies differ: a quarantine is lifted by a person // (`run unquarantine`), a park clears itself on the pass whose verdict changes (PD-438). ParkedAt *time.Time // Settling says which half of the stall this row is: an attempt still LIVE that the reconciler // cannot finish, or one that has ENDED whose money never closed. They are one list because they // are one operator question — "what is stuck and what is it holding" — and one counter // (reconcile_failures, through the shared deferItem); they are told apart because the remedy // differs and because the second half was invisible until this column existed (PD-385). Settling bool } // StalledRuns lists what the reconciler cannot finish, at `atLeast` consecutive failures or more. // Zero lists every live run, which is what the operator's plain `runs` asks for. // // It exists because the metric that showed this was a counter with no names in it: an operator could // see that a pass did not finish and could not see WHICH run, why, or what it was holding. A number // without a handle is half a mechanism. // // ⚠ TWO POPULATIONS, and until PD-385 this listed only the first of them. The reconciler has two // phases sharing one failure counter through `deferItem`, and this query was built for LIVE runs // alone — `a.ended_at is null` joined with `r.finished_at is null`. So a run that ENDED and whose // money never closed was invisible here, invisible to the gauge built on the same predicate, and // refused by `run abandon`; the ERROR logged at the threshold sent an operator to this very command // and it answered "no run is failing to reconcile" over a frozen hold. Measured on state grown // through the ordinary paths: five failures, an open reservation of 90000 micro, and all three // surfaces silent. // // The second half is keyed exactly as the settlement worklist is (UnsettledRuns): the ATTEMPT is // over and its reservation is still open. What it is NOT keyed on is the run being unfinished, which // is the whole bug. // // ⚠ The floor for the settling half is `greatest($1, 1)` and that is not a fudge: an ordinary // settlement closes on its first pass and never records a failure, so at `atLeast` zero — the plain // `runs` — every settlement in flight would appear for the seconds it lives, and a table that lists // healthy work is one an operator stops reading. One failure is the smallest number that means // something went wrong. The live half keeps its own floor of zero, because a live run IS the answer // to plain `runs`. The literal `>= 1` written beside it is not redundant: it is the CONSTANT the // planner needs to prove the partial index applies, which `greatest($1, 1)` cannot give it under a // generic plan. func (s *Store) StalledRuns(ctx context.Context, atLeast int) ([]StalledRun, error) { // ⚠ A UNION OF TWO ARMS and not one WHERE with an OR, and the honest reason is PREDICTABILITY of // the plan — NOT the buffer count, which an earlier version of this comment claimed and a // re-measurement refused. Measured on 200 000 attempts, four cells, because there are two // variables and the first draft named one: // // without 00028 with 00028 // one WHERE with an OR 3647 buffers 10 buffers (BitmapOr over both partial indexes) // UNION of two arms 3653 buffers 18 buffers (each arm on its own index) // // So the catastrophe — a parallel sequential scan of every attempt ever, on a command the runbook // recommends from a cron line and on a twin gauge the daemon runs every fifteen seconds forever — // is removed by the INDEX (00028), which both shapes need equally. What the UNION buys is that // each arm carries its access path STRUCTURALLY: `run_attempts_live_idx` (00009) for the live // half, `run_attempts_settling_idx` (00028) for the settling one. The OR shape's BitmapOr is the // planner's choice, taken on statistics — and the statistics of a HEALTHY deployment are an empty // stalled population, which is the case least like the one it was measured on. Eight buffers is // what that costs; the top-left cell is what getting it wrong costs. rows, err := s.pool.Query(ctx, ` with stalled as ( select r.id as run_id, b.title, a.id as attempt_id, a.spawns, a.parked_at, coalesce(a.unit_name, '') as unit, a.attempt_no, r.status, a.reconcile_failures, a.reconcile_after, coalesce(a.reconcile_error, '') as last_error, coalesce(a.quarantine_reason, '') as quarantine, a.spend_micro_usd, a.spend_baseline_micro_usd, r.started_at, false as settling from run_attempts a join runs r on r.id = a.run_id and r.finished_at is null join books b on b.id = r.book_id where a.ended_at is null and a.reconcile_failures >= $1 union all select r.id, b.title, a.id, a.spawns, a.parked_at, coalesce(a.unit_name, ''), a.attempt_no, r.status, a.reconcile_failures, a.reconcile_after, coalesce(a.reconcile_error, ''), coalesce(a.quarantine_reason, ''), a.spend_micro_usd, a.spend_baseline_micro_usd, r.started_at, true from run_attempts a join runs r on r.id = a.run_id join books b on b.id = r.book_id where a.ended_at is not null and a.reconcile_failures >= 1 and a.reconcile_failures >= greatest($1, 1) and exists (select 1 from reservations res where res.engine_run_id = a.run_id || '#' || a.attempt_no and res.state = 'open') ) select s.run_id, s.title, s.attempt_id, s.spawns, s.parked_at, s.unit, s.attempt_no, s.status, s.reconcile_failures, s.reconcile_after, s.last_error, s.quarantine, coalesce(res.amount_micro_usd, 0), coalesce(extract(epoch from (now() - res.opened_at)), 0), case when s.spend_baseline_micro_usd is null then null else greatest(coalesce(s.spend_micro_usd, 0) - s.spend_baseline_micro_usd, 0) end, s.settling from stalled s left join reservations res on res.engine_run_id = s.run_id || '#' || s.attempt_no and res.state = 'open' order by s.reconcile_failures desc, s.started_at`, atLeast) if err != nil { return nil, fmt.Errorf("pgstore: list stalled runs: %w", err) } defer rows.Close() var out []StalledRun for rows.Next() { var v StalledRun if err := rows.Scan(&v.RunID, &v.Title, &v.AttemptID, &v.Spawns, &v.ParkedAt, &v.UnitName, &v.AttemptNo, &v.Status, &v.Failures, &v.NextTry, &v.LastError, &v.QuarantineReason, &v.HeldMicroUSD, &v.HeldSeconds, &v.SpentMicroUSD, &v.Settling); err != nil { return nil, fmt.Errorf("pgstore: scan stalled run: %w", err) } out = append(out, v) } return out, rows.Err() } // ErrRunMayHaveAProcess is an abandon asked for on a run whose attempt still carries evidence that a // process exists. Refused rather than forced: closing a run over a live engine leaves that engine // spending against a run nothing looks at any more, which is the one outcome worse than the stall. // // ⚠ There are TWO pieces of such evidence and only one of them is the unit's name. `ReleaseSpawnClaim` // clears the name when the unit could not be CREATED — and "could not be created" is not "was not // created": a `systemd-run` killed after it had already asked leaves an engine running, which is why // the spend baseline is deliberately kept on such an attempt (RecordSpawn). That baseline is the // tombstone, and the reconciler's own close of this case reads it rather than the name // (runs.finishStopped). This guard reads both, because it is the same question. var ErrRunMayHaveAProcess = errors.New("pgstore: the run's attempt still carries evidence of a process") // AbandonOrder is one operator's terminal verdict together with the proof it stands on. // // A struct and not five positional arguments because two of the five are booleans that mean opposite // kinds of thing — one is about WHEN the money moves, the other about whether a process exists — and // a call site where those two sit next to each other unnamed is a call site nobody can read. type AbandonOrder struct { RunID string // Reason is required by the caller and written to the attempt and the ledger: this row is the // only account anyone will ever have of why a paid-for run was declared over. Reason string // ReleaseHold gives the hold back inside this transaction instead of on the next sweep. It is a // convenience for an operator working with the daemon STOPPED and never a money decision — see // the note below. Ignored by the two branches that have no sweep to hand the money to. ReleaseHold bool // ProcessGone is the caller's PROOF that the stuck attempt has no engine process, and it is the // whole of what makes the orphan branch reachable. // // ⚠ IT IS A PROOF AND NOT A PREFERENCE, and the caller owes the asking. What this package can see // is only the attempt's own columns — a unit name and a spend baseline, both of which say a // process was CREATED and neither of which says whether it is still there. Whether it is running // is systemd's to answer, so the answer has to come from whoever can ask (cmd/tmplatformctl, // `run abandon`, which asks and refuses on anything but a clear "gone", including on systemd // being unreachable). // // ⚠ WHAT IT COSTS IF THE PROOF IS WRONG, said plainly because it is a destructive operator // surface over money: a live engine whose run is abandoned keeps spending against its own book // ceiling, and the account's hold is closed here — so that spend is charged to nobody and the // DEPLOYMENT pays the provider for it. It is bounded (by the engine's `--ceiling-usd`, which is // this run's own) and it is not exploitable by an account: reaching this state takes a run the // deployment's own reconciler could not finish. // // The proof is a SNAPSHOT taken before this transaction opens, and `RecordSpawn` takes no book // lock, so the world can move under it. The two ways below are named, re-read under the lock and // refused. // // ⚠ A THIRD IS NOT CAUGHT, and it is named rather than left to be discovered. `RecordSpawn` // commits the claim — and the counter with it — BEFORE the unit is created (runs.spawnAttempt, // where `Runner.Start` follows the record). A proof taken inside that window reads the counter // already bumped and hears «gone» from a systemd the unit has not reached yet, so both // comparisons agree and the write-off lands on an attempt whose unit is about to rise. Catching // it needs the store to know that a spawn is IN FLIGHT, which is a state nothing records today — // register row PD-465. The window is the milliseconds between two statements of one function, and // it costs what every wrong proof costs (see the paragraph on ProcessGone below). ProcessGone bool // ProofAttemptID is which attempt the caller asked systemd about. The write below reads "the live // attempt", so a restart in the window would otherwise spend a proof about attempt N on attempt // N+1 — a fresh unit with an engine mid-call, and its hold closed underneath it. ProofAttemptID int64 // ProofSpawns is `run_attempts.spawns` as the caller read it. `unit_name` cannot serve here: // ReleaseSpawnClaim returns it to NULL, so a claim-then-release inside the window leaves the name // exactly as the proof saw it while an engine may be running. The counter never goes back. ProofSpawns int Now time.Time } // ErrProofOvertaken is an abandon whose proof stopped being about the run in front of it. Refused // rather than resolved here: whether a process exists is systemd's answer and the caller's to ask. var ErrProofOvertaken = errors.New("pgstore: the attempt moved while its absence was being proved") // AbandonRun is the OPERATOR's terminal verdict on a run the reconciler cannot finish. // // It is not a policy this platform applies by itself, and that boundary is deliberate. The // reconciler's rule everywhere else is that "I could not ask" is never "the run is gone"; a run that // has failed N times is exactly a run nobody could ask about, so deciding on its behalf would be // that same mistake with a counter in front of it. What the counter buys is that an operator is TOLD // (StalledRuns, the stalled gauge); what this buys is that being told is worth something. // // ⚠ THE HOLD COMES BACK WHOLE EITHER WAY, and the flag only decides WHEN. The guard below admits // only an attempt that never reached the engine, so nothing was spent, so the ordinary settlement // releases it whole on its next pass — `--release-hold` does it inside this transaction instead, // which is what an operator working with the daemon stopped actually needs. What the flag never was // is a money decision: the doc used to promise the hold "stays open while there is any chance the // engine will answer", which the guard has already made impossible. Neither branch is the escrow // design (unified backlog row 136, zone backlog П-18). func (s *Store) AbandonRun(ctx context.Context, o AbandonOrder) (AbandonVerdict, error) { runID, reason, giveTheHoldBack, now := o.RunID, o.Reason, o.ReleaseHold, o.Now var verdict AbandonVerdict err := s.inTx(ctx, func(tx pgx.Tx) error { // ⚠ THE BOOK'S ROW FIRST, before the run and its attempt. That is the package's global order // (lockBook) and it is not a preference: this used to take the run and the attempt together // through `for update of r, a` and the book afterwards, which is the inversion `RestartRun` // names as the pair that deadlocked — measured here at 7 aborted transactions in 60 concurrent // pairs against `FinishRun` and 3 against `PauseRun`, both of which are on the settlement path. // Postgres breaks the cycle by aborting one side, so the price is a failed sweep or an operator // command that answers with raw driver text. // // The book id is read WITHOUT a lock to find out which book to lock; everything the write // depends on is then re-read under it. var bookID string switch err := tx.QueryRow(ctx, `select book_id from runs where id = $1`, runID).Scan(&bookID); { case errors.Is(err, pgx.ErrNoRows): return ErrNoRun case err != nil: return fmt.Errorf("pgstore: read the run to abandon: %w", err) } if err := lockBook(ctx, tx, bookID); err != nil { return err } // ⚠ WHICH OF THE TWO VERDICTS THIS IS, read UNDER THE BOOK LOCK and not before it. The // unlocked read above is only for finding out which book to lock, and reading the branch from // it was a real race with a money answer: `RestartRun` takes this same book lock, clears // `finished_at` and opens a NEW attempt with a NEW hold, so an abandon that queued behind a // resume would come through holding a snapshot that says "finished" and act on a run that is // live again. Everything the write depends on is re-read here, which is what the paragraph // above already promised and this line now keeps. var finished *time.Time if err := tx.QueryRow(ctx, `select finished_at from runs where id = $1`, runID).Scan(&finished); err != nil { return fmt.Errorf("pgstore: read the run to abandon: %w", err) } if finished != nil { verdict = AbandonedSettlement return abandonSettlement(ctx, tx, runID, reason, true, now) } // ⚠ THE RUN IS LIVE AND THAT DOES NOT MEAN ITS MONEY IS. Before anything is decided about the // live attempt, the run is asked whether it carries an ORPHANED one — an attempt that ended // with its reservation still open. That population is `PD-418`, and the reason it had no // handle is exactly this branch: everything terminal used to ask `runs.finished_at`, which // says nothing about whether some earlier attempt's hold is stuck. Branching on the PRESENCE // of the orphan instead is the durable cure both rows name, and it is one query. switch err := abandonSettlement(ctx, tx, runID, reason, false, now); { case err == nil: verdict = AbandonedSettlement return nil case !errors.Is(err, errNoOrphanedAttempt): return err } verdict = AbandonedRun var unit string var attemptID int64 var attemptNo int var baseline *int64 var reported int64 var failures int var spawns int err := tx.QueryRow(ctx, ` select a.id, a.attempt_no, coalesce(a.unit_name, ''), a.spend_baseline_micro_usd, a.spend_micro_usd, a.reconcile_failures, a.spawns from runs r join run_attempts a on a.run_id = r.id and a.ended_at is null where r.id = $1 and r.finished_at is null for update of r, a`, runID). Scan(&attemptID, &attemptNo, &unit, &baseline, &reported, &failures, &spawns) if errors.Is(err, pgx.ErrNoRows) { // It FINISHED while this transaction was taking the book — the unlocked read above said // live and the locked one does not. That is not "there is no such run", and answering it // as one is the very confusion PD-385 is about: the operator is looking at the run. It is // the settlement case, one pass late, and it is handled as the settlement case. verdict = AbandonedSettlement return abandonSettlement(ctx, tx, runID, reason, true, now) } if err != nil { return fmt.Errorf("pgstore: read the attempt to abandon: %w", err) } var writeOff *money.MicroUSD if unit != "" || baseline != nil { // Whether a process is still RUNNING is systemd's to answer, and this refuses the case where // nobody asked. The unit's name is derivable from the run and the attempt even when the // column is empty, so the message names it: that is where the operator has to go. if !o.ProcessGone { return fmt.Errorf("%w: tm-run-%s-%d", ErrRunMayHaveAProcess, runID, attemptNo) } // Both halves are re-read above under the book lock. A second look from the CLI would be // one more snapshot with one more window after it (PD-424). switch { case o.ProofAttemptID != attemptID: return fmt.Errorf("%w: the proof was about attempt %d and this run's live attempt is now #%d", ErrProofOvertaken, o.ProofAttemptID, attemptNo) case o.ProofSpawns != spawns: return fmt.Errorf("%w: attempt #%d was claimed for a spawn (%d claims at the proof, %d now)", ErrProofOvertaken, attemptNo, o.ProofSpawns, spawns) } // The caller brought the proof, and the floor is the same one the settlement branch waits // for: SEEING a run go wrong and being allowed to end it are different permissions. Below // the threshold the reconciler is still retrying, and a run it would have closed correctly // on its own must not be written off a tick early. if failures < abandonAfter { return fmt.Errorf("%w: %d of %d", ErrRunNotStalledYet, failures, abandonAfter) } verdict = AbandonedOrphan // ⚠ WHAT IT COSTS, and the two cases are not the same money. `spend_micro_usd` is the BOOK's // lifetime total as the stream last reported it, so the attempt's own work is the DIFFERENCE // against its baseline — the same arithmetic `StalledRuns` prints and `attemptSpend` // settles by, clamp included. A baseline plus that figure is the last thing the engine SAID // about this attempt, and charging it is // strictly more honest than handing the whole hold back — the run demonstrably did that // much work. Without a baseline there is no figure this platform could justify, and the // hold goes back whole, which is the settlement branch's own reasoning arriving here. if baseline != nil { spent := money.MicroUSD(max(reported-*baseline, 0)) writeOff = &spent } } if _, err := tx.Exec(ctx, ` update runs set status = 'failed', failure_reason = 'service_error', finished_at = $2, paused_reason = null, revision = (select revision + 1 from books where id = runs.book_id) where id = $1`, runID, now); err != nil { return fmt.Errorf("pgstore: abandon run: %w", err) } // ⚠ THE DEFERRAL GOES WITH THE VERDICT, and leaving it was PD-391. `UnsettledRuns` filters on // `reconcile_after`, and a run that reached an operator is by construction one that has been // deferred — `deferItem` sets now + backoff, and backoff at five failures is pinned at its // thirty-minute cap. So for the WHOLE population this command exists for, the CLI's "its hold // comes back whole on the next sweep" and the runbook's copy of it were false: the credit // stayed debited, `tm_platform_oldest_open_hold_seconds` kept rising AFTER the operator acted, // and what they read was "I did it and nothing happened". Measured: forty-five seconds and // three sweeps with the hold open; clearing the column closes it on the next one. if _, err := tx.Exec(ctx, `update run_attempts set ended_at = $2, reconcile_error = $3, reconcile_after = null where id = $1`, attemptID, now, truncateReason("abandoned by an operator: "+reason)); err != nil { return fmt.Errorf("pgstore: end the abandoned attempt: %w", err) } // The book owes a reading surface like it does after every other ending: whatever the run did // translate before it stalled is bought and paid for, and this is the one write that puts it // back in front of a reader. if _, err := tx.Exec(ctx, ` update books set status = 'failed', `+owesAReadingSurface+` revision = `+nextRevisionOfThisBooksLibrary+` where id = $1`, bookID); err != nil { return fmt.Errorf("pgstore: mark the abandoned run's book: %w", err) } if err := emitStatus(ctx, tx, bookID); err != nil { return err } if !giveTheHoldBack && verdict != AbandonedOrphan { return nil // the settlement phase closes it on its next pass } // ⚠ THE ORPHAN BRANCH CLOSES ITS MONEY INLINE WHATEVER THE FLAG SAYS, for the same reason the // settlement branch does: the attempt this ends is one no sweep will ever settle — that is // what made it an orphan — so leaving the hold for a pass that is not coming is the defect // this handle exists to end, not a policy. key := ReservationKey(runID, attemptNo) state := "released" if writeOff != nil { state = "settled" } userID, held, err := closeReservation(ctx, tx, key, state, now) if errors.Is(err, ErrNoReservation) { return nil // already settled or released; the abandon still stands } if err != nil { return err } if err := releaseHold(ctx, tx, userID, key, held, now); err != nil { return err } if writeOff != nil { // The last figure the engine gave for THIS attempt, charged rather than forgiven. Capped at // the hold by the same rule the ordinary settlement uses: the hold is what the account // agreed to, and a meter above it is the engine's business to explain, never this // account's to pay. spent := *writeOff // The basis is HALTED without asking anything: an abandon is by definition an attempt cut // off mid-work, and its figure is the last one the engine managed to report. See // SettlementBasis for why every settlement says which floor it is (PD-441). note := BasisHalted.note() + "; abandoned by an operator over an orphaned attempt" if spent > held { note += "; capped at the hold" spent = held } applied, err := appendLedger(ctx, tx, userID, "settlement", -spent, "run_settle", key, note, now) if err != nil { return err } if !applied { // Reported rather than applied, exactly as the ordinary settlement reports it: a spent // key here would return the hold and charge nothing. return fmt.Errorf("%w: %s", ErrSettlementKeySpent, key) } } // The run's money is RESOLVED, and saying so is this branch's own job: it takes the run out of // the settlement list before that list ever sees it, so nothing else would ever stamp the // column and the run stayed "unsettled" for good. _, err = tx.Exec(ctx, `update runs set settled_at = $2 where id = $1 and settled_at is null`, runID, now) return err }) if err != nil { return "", err } return verdict, nil } // AbandonVerdict is which of the two terminal verdicts AbandonRun applied. Returned rather than left // for the caller to re-derive: the two mean different things to whoever typed the command — one // ended a run, the other gave up on money — and a command that printed the same sentence for both // would be the sentence that hid the second one (PD-385). type AbandonVerdict string const ( // AbandonedRun is the live case: a run the reconciler could not finish is declared over. AbandonedRun AbandonVerdict = "run" // AbandonedSettlement is the other half: the run had already ended and its MONEY was stuck. // // ⚠ Since the orphan handle it is ALSO the verdict of a LIVE run that carries an ended attempt // whose hold never closed — the population of `PD-418`. The run itself is untouched there: what // was stuck was one attempt's money, and ending a run over it would take a translation away from // a user who did not ask. AbandonedSettlement AbandonVerdict = "settlement" // AbandonedOrphan is the third and the newest: a run that is LIVE, whose attempt still names an // engine process, and whose process the caller has PROVEN is gone. It is the half of `PD-424` // that stood open through two packs — the run was visible, counted and un-actionable, and the // only exit was the user pressing Stop without being told so. AbandonedOrphan AbandonVerdict = "orphan" ) // ErrRunNotStalledYet is an orphan abandon asked for on a live run the reconciler has not given up // on. The floor is the same `abandonAfter` the settlement branch waits for, and for the same reason // written out there: seeing a run go wrong and being allowed to end it are different permissions. var ErrRunNotStalledYet = errors.New("pgstore: the run has not failed to reconcile enough times to be written off") // errNoOrphanedAttempt says a run carries no ended attempt with an open reservation. Unexported: it // is a fall-through between the branches of one function and never an answer to a caller — for a // FINISHED run the absence is `ErrMoneyAlreadyClosed` or `ErrSettlementNotStuck`, and for a live one // it simply means the live attempt is the thing to look at. var errNoOrphanedAttempt = errors.New("pgstore: the run carries no orphaned attempt") // ErrMoneyAlreadyClosed is an abandon asked for on a run that has finished and whose money is // already resolved. Told apart from ErrNoRun deliberately: "there is nothing to do" and "there is no // such run" send an operator to different places, and answering the first with the second is what // made the settlement stall unactionable (PD-385). var ErrMoneyAlreadyClosed = errors.New("pgstore: the run has finished and its money is already closed") // ErrSettlementNotStuck is an abandon asked for on a finished run whose money is open but whose // settlement has not failed even once — that is, one the next sweep is about to close correctly. // Refused rather than obeyed: the hold this command returns is returned WHOLE, so obeying would be // giving away whatever the run actually spent, and the operator has no way to know that from the // outside. It is also the state the operator's own table deliberately does not show. var ErrSettlementNotStuck = errors.New("pgstore: the run's settlement has not failed; it is still being retried") // abandonSettlement is the operator's terminal verdict on the OTHER half of the stall: a run that // ended and whose settlement will never complete. // // It exists because there was no handle at all for this population, by construction: every terminal // path read `finished_at is null` and answered ErrNoRun, so a hold behind a settlement that could // not be computed — the engine binary gone at a rollout, an attempt from before the baseline column — // was frozen until somebody wrote SQL in production. (A project REPLACED under the platform is // deliberately NOT in that list, though the first draft of this paragraph had it: the reconciler // catches that one by the meter reading below the attempt's own baseline and settles it at nothing, // so it never reaches this handle.) // // ⚠ ONLY A SETTLEMENT THAT IS STALLED, and the floor here is the THRESHOLD and not the one failure // the operator's table shows from. The two numbers are deliberately different, and the difference is // see-early / act-late: // // - `StalledRuns` lists the settling half from ONE failure, because an operator wants to see a // settlement going wrong while it is only going wrong; // - this command REFUNDS THE WHOLE HOLD, so it waits for `abandonAfter` — the same // `runs.StalledAfter` the threshold and the gauge use. // // The first draft of this branch admitted from one failure, matching the list, and acceptance // measured what that costs: an engine unavailable for a single tick puts a row in the plain table // whose CLI comment points at this command, and running it then writes off the run's real spend — // measured at $1.234567 spent, ordinary sweep charging $0.300000 and this command a tick earlier // charging $0.000000. Seeing a thing and being allowed to destroy it are different permissions. // // ⚠ THE HOLD COMES BACK WHOLE, and that is a decision rather than an omission. What is missing is // the engine's committed figure — that is the definition of this state — so there is no number this // platform could justify charging, and the two alternatives are worse in both directions: charging // the ceiling bills for work nobody can show, and leaving it open is today's defect. It is not // exploitable by the account either: reaching the admitted set takes a settlement the deployment's // own engine could not answer, never something a user can ask for. // // ⚠ It does NOT touch the run's status. The run already ended with an outcome of its own, and // overwriting it with `failed` would erase what actually happened to a run that may well have // succeeded. Nor does it bump a revision or emit a frame: the ORDINARY settlement does neither // (MarkSettled), and this is that settlement reaching its end by another road. // // The evidence-of-a-process guard the live branch applies is satisfied here by the ending itself and // not skipped: `ended_at` is written only where the platform established the process is over — // `finish` off a marker, or `finishStopped`, which asks systemd before closing an attempt that // carries a spend baseline (runs.finishStopped). // abandonAfter is how many consecutive settlement failures make a run's money an operator's to write // off. It is `runs.StalledAfter` written here rather than imported: `internal/runs` depends on this // package, so the constant cannot travel the other way, and a second literal is worse than a named // one. The two are pinned equal by TestTheAbandonFloorIsTheStalledThreshold. const abandonAfter = 5 func abandonSettlement(ctx context.Context, tx pgx.Tx, runID, reason string, runFinished bool, now time.Time) error { // ⚠ EVERY orphaned attempt of this run, not the oldest one. A run holds at most one open // reservation on the ordinary path — `reopen` refuses to start attempt N+1 while attempt N's is // open — but a run needing this command is by definition one whose ordinary path came apart, and // the state this package already worries about out loud is exactly "an attempt that was // interrupted and replaced leaves its reservation open while its run goes on" (UnsettledRuns). // Taking one row and then stamping the RUN settled would leave the second hold debited under a // message that says the money came back whole. // ⚠ NO `r.finished_at is not null`, and its removal is the durable cure of `PD-418`. An orphaned // attempt is orphaned by the PRESENCE of its own open reservation over its own `ended_at`, and // whether the RUN has since finished says nothing about that — a live run whose previous attempt // never settled is precisely the population the runbook promised this command served and the code // answered about the process instead. The caller says which shape it is in `runFinished`, and // that decides only what an EMPTY result means. rows, err := tx.Query(ctx, ` select a.id, a.attempt_no from runs r join run_attempts a on a.run_id = r.id and a.ended_at is not null and a.reconcile_failures >= $2 join reservations res on res.engine_run_id = r.id || '#' || a.attempt_no and res.state = 'open' where r.id = $1 order by a.attempt_no for update of r, a`, runID, abandonAfter) if err != nil { return fmt.Errorf("pgstore: read the settlement to abandon: %w", err) } type orphan struct { id int64 attemptNo int } var orphans []orphan for rows.Next() { var o orphan if err := rows.Scan(&o.id, &o.attemptNo); err != nil { rows.Close() return fmt.Errorf("pgstore: scan the settlement to abandon: %w", err) } orphans = append(orphans, o) } rows.Close() if err := rows.Err(); err != nil { return fmt.Errorf("pgstore: read the settlement to abandon: %w", err) } if len(orphans) == 0 { if !runFinished { // A LIVE run with no orphan is not a state to answer about: its own attempt is what the // caller came for, and this is a fall-through rather than a verdict. return errNoOrphanedAttempt } // Two states, told apart, because the operator's next move differs: money already resolved // (nothing to do) versus money still open on a settlement that has not failed yet (wait — the // next sweep closes it, and giving it back now would be a gift of whatever the run spent). var pending bool if err := tx.QueryRow(ctx, ` select exists ( select 1 from run_attempts a join reservations res on res.engine_run_id = a.run_id || '#' || a.attempt_no and res.state = 'open' where a.run_id = $1 and a.ended_at is not null)`, runID).Scan(&pending); err != nil { return fmt.Errorf("pgstore: read the settlement to abandon: %w", err) } if pending { return ErrSettlementNotStuck } return ErrMoneyAlreadyClosed } for _, o := range orphans { key := ReservationKey(runID, o.attemptNo) userID, held, err := closeReservation(ctx, tx, key, "released", now) switch { case errors.Is(err, ErrNoReservation): // The settlement sweep closed it between this transaction's read and this write. Tolerated // rather than reported, exactly as the live branch tolerates it: the money is resolved, // which is what the operator asked for, and a raw "no open reservation" would read as a // failure of the command. continue case err != nil: return err } if err := releaseHold(ctx, tx, userID, key, held, now); err != nil { return err } // The reason on the record, and the deferral cleared for the same reason the live branch // clears it (PD-391): a row that stays deferred is one the sweep keeps stepping over. if _, err := tx.Exec(ctx, ` update run_attempts set reconcile_error = $2, reconcile_after = null where id = $1`, o.id, truncateReason("settlement abandoned by an operator: "+reason)); err != nil { return fmt.Errorf("pgstore: record the abandoned settlement: %w", err) } } if !runFinished { // ⚠ A LIVE run is NOT stamped settled: `settled_at` is a statement about the WHOLE run's // money, and the live attempt's hold is still open and still the reconciler's to close. // // ⚠ AND THE COLUMN HAS NO READER TODAY, which an adversarial pass established and an earlier // edition of this comment did not know: `UnsettledRuns` keys on an OPEN RESERVATION // (`sink.go`), not on this column, and the ordinary sweep already stamps it on live runs // through `MarkSettled`. So what this branch protects is the column's MEANING and not a // worklist — worth keeping (a field that lies is a field the next reader believes) and not // worth claiming more for. return nil } // Said here and nowhere else: this takes the run out of the settlement worklist before that list // sees it again, so nothing else would ever stamp the column and the run would read "unsettled" // for good — the same trap the `--release-hold` branch above names. Stamped only once every // orphan is closed, so the column cannot claim a resolution one of them contradicts. if _, err := tx.Exec(ctx, `update runs set settled_at = $2 where id = $1 and settled_at is null`, runID, now); err != nil { return fmt.Errorf("pgstore: mark the abandoned settlement settled: %w", err) } return nil } // runColumns is the reconciler's view of a run. Written once because the two queries that use it // differ only in which runs they select, and a scan list copied twice is a scan list that drifts. // SpawnOrder is the book's order RESOLVED against the tree as it stands, for one spawn. // // ⚠ THREE STATES AND THEY ARE NOT INTERCHANGEABLE. A whole-book order has no boundary and needs none // (`Order.Whole()`); a partial order that RESOLVES gives a unit count; a partial order that does NOT // resolve is a book cut again under a purchase, and the honest answer there is neither a count nor // «the whole book» — it is a refusal to spawn, because either number would be a volume nobody // bought. // // Resolved at every spawn rather than frozen at admission, for the same reason the boundary is // stored as an identity: what «through chapter 12» is worth in units is a property of the CURRENT // cut, and a restart of a run that already delivered half its order must not hand the engine the // whole of it again. type SpawnOrder struct { Order BookOrder // Units is the order in output units; Resolved says whether its boundary could be found at all. Units int Resolved bool // Delivered is how many of the units the order runs THROUGH are already handed over — bounded by // the same boundary the order is. Counted book-wide, work delivered BEYOND the boundary would // cancel work still owed inside it (see deliveredWithinOrder). Delivered int // UnitsLeft is what is undelivered book-WIDE: the allowance a WHOLE-BOOK order is given on a // continuation. Giving one at all is not belt-and-braces — with no ceiling in force the engine // builds no volume scope, and the scope is what carries its protective order of work (new book // before re-made book, backend/internal/pipeline/volume.go). Without it a continuation carrying // `--resnapshot` re-pays the beginning of the book before editing what was just bought. UnitsLeft int // HasPriorRun is whether this book has EVER had another run — the condition for `--resnapshot` // beside the correction door's own flag (PD-422): the auto-bank grows by MINING during an // ordinary run, mining moves the ENRICHED memory version, and that version is folded into the // EDIT wave's snapshot alone (backend/internal/pipeline/snapshot.go, snapshotIDForWave). So the // second purchase of a mining book meets its own already-pinned edit jobs under a moved snapshot // and the drift guard stops the engine — after the hold was taken, with nothing translated. HasPriorRun bool } // ReadOrderForSpawn resolves a run's order against the tree as it stands now. // // ⛔ ITS OWN QUERY, ON THE SPAWN PATH, and that placement is the whole of it. Resolving an order // means walking the book's units three ways, and these figures were once columns of `runColumns` — // the reconciler's list of live runs, read for every live run on every sweep, for numbers only a // spawn reads. A spawn happens once per attempt; a sweep happens every few seconds. func (s *Store) ReadOrderForSpawn(ctx context.Context, runID string) (SpawnOrder, error) { var out SpawnOrder err := s.pool.QueryRow(ctx, ` select coalesce(b.ordered_through_chapter_id, ''), coalesce(b.ordered_through_unit_id, ''), coalesce(b.ordered_through_chapter_number, 0), `+orderedUnits+`, `+orderResolved+`, `+deliveredWithinOrder+`, `+bookUnitsLeft+`, exists (select 1 from runs pr where pr.book_id = b.id and pr.id <> r.id) from runs r join books b on b.id = r.book_id where r.id = $1`, runID). Scan(&out.Order.ThroughChapterID, &out.Order.ThroughUnitID, &out.Order.ThroughChapterNumber, &out.Units, &out.Resolved, &out.Delivered, &out.UnitsLeft, &out.HasPriorRun) if errors.Is(err, pgx.ErrNoRows) { return SpawnOrder{}, ErrNoRun } if err != nil { return SpawnOrder{}, fmt.Errorf("pgstore: read the run's order: %w", err) } return out, nil } const runColumns = ` select r.id, r.book_id, b.owner_id, b.workdir, r.verify_bank, r.resnapshot, r.accept_rebill_micro, r.ceiling_chapters, coalesce(r.paused_reason, ''), r.started_at, r.status, r.stop_requested_at, a.id, a.attempt_no, coalesce(a.unit_name, ''), coalesce(a.engine_run_id, ''), a.last_offset, a.last_seq, a.last_line_sha256, a.quarantine_reason is not null, a.parked_at is not null, coalesce(res.amount_micro_usd, 0), a.engine_binary, a.spend_baseline_micro_usd, a.ceiling_arg_micro_usd, a.started_at, a.reconcile_failures from runs r join books b on b.id = r.book_id` func (s *Store) queryRuns(ctx context.Context, tail string, args ...any) ([]LiveRun, error) { rows, err := s.pool.Query(ctx, runColumns+tail, args...) if err != nil { return nil, fmt.Errorf("pgstore: list runs: %w", err) } defer rows.Close() var out []LiveRun for rows.Next() { var l LiveRun var ceiling, ceilingArg, acceptRebill int64 var baseline *int64 if err := rows.Scan(&l.RunID, &l.BookID, &l.UserID, &l.Workdir, &l.VerifyBank, &l.Resnapshot, &acceptRebill, &l.OrderedChapters, &l.PausedReason, &l.StartedAt, &l.Status, &l.StopRequestedAt, &l.AttemptID, &l.AttemptNo, &l.UnitName, &l.EngineRunID, &l.Position.Offset, &l.Position.LastSeq, &l.Position.LastHash, &l.Quarantined, &l.Parked, &ceiling, &l.EngineBinary, &baseline, &ceilingArg, &l.AttemptStartedAt, &l.ReconcileFailures); err != nil { return nil, fmt.Errorf("pgstore: scan run: %w", err) } l.Ceiling = money.MicroUSD(ceiling) l.CeilingArg = money.MicroUSD(ceilingArg) l.AcceptRebill = money.MicroUSD(acceptRebill) if baseline != nil { v := money.MicroUSD(*baseline) l.SpendBaseline = &v } out = append(out, l) } return out, rows.Err() } // RunSpent is what a run has actually been charged so far, across all its attempts. Settlements are // ledger rows keyed by attempt, so the sum is over the run's own key space and nothing else's. func (s *Store) RunSpent(ctx context.Context, runID string) (money.MicroUSD, error) { var v int64 err := s.pool.QueryRow(ctx, ` select coalesce(-sum(amount_micro_usd), 0) from credit_ledger where source = 'run_settle' and split_part(source_id, '#', 1) = $1`, runID).Scan(&v) if err != nil { return 0, fmt.Errorf("pgstore: read run spend: %w", err) } return money.MicroUSD(v), nil } // ErrNoFirstHold is a run whose first attempt's reservation cannot be found. Every admission opens // that row in the same transaction as the run (StartRun → holdTx), and closing a reservation keeps // the row and changes its state — so this is a broken invariant rather than a state, and the caller // is told by name instead of being handed a figure the rate would give. var ErrNoFirstHold = errors.New("pgstore: the run's first hold is missing, so what it was sold for cannot be read") // RunBudget is what the run was SOLD for: the hold its FIRST attempt took, read back from the // reservation rather than quoted again. // // The two are one number on the day of the admission and different afterwards: an order's quote is // made of the engine's projection (re-derived on every re-cut of the book) and of a cushion the // deployment sets, and both move, while the hold is the figure the buyer agreed to. Quoting it again // would re-price a paid run in both directions — up, and the continuation holds more than the buyer // was ever shown; down, and the remainder goes negative and a run with chapters left pauses as // exhausted (PD-168: with the setting doubled, $5.50 held for a $2.50 remainder). // // `amount_micro_usd` and not `ceiling_micro_usd`: the two carry the same figure at open // (OpenReservation writes one argument into both), but the amount is what the ledger debited and what // settlement gives back, i.e. the money fact, while the ceiling column's own comment describes a // meaning the engine's flag does not have (D39.122, PD-377's class). func (s *Store) RunBudget(ctx context.Context, runID string) (money.MicroUSD, error) { var v int64 err := s.pool.QueryRow(ctx, `select amount_micro_usd from reservations where engine_run_id = $1`, engineRunKey(runID, 1)).Scan(&v) if errors.Is(err, pgx.ErrNoRows) { return 0, fmt.Errorf("%w: run %s", ErrNoFirstHold, runID) } if err != nil { return 0, fmt.Errorf("pgstore: read run budget: %w", err) } return money.MicroUSD(v), nil } // SpendBound is the UPPER bound on what an attempt can have cost: the smallest meter reading any // LATER attempt of the same book recorded before it started. // // It exists because settlement reads a lifetime counter of the BOOK at the moment it retries, and // that retry can happen after another run of the same book has already moved it. A settlement is // allowed to defer — the engine has to be asked and asking can fail — and a deferred one is not // blocked from being overtaken: the earlier run is finished, so nothing stops the account starting // another. Measured: a run that cost $0.10 was charged $2.10, the difference being what its // successor had spent by then, and the successor then paid that same amount again. // // A later attempt's baseline is exactly the right bound, because it was read BEFORE that attempt // added anything and AFTER this one had stopped. Nil means no later attempt has been spawned, and // then the counter has not been moved by anyone else. func (s *Store) SpendBound(ctx context.Context, bookID string, attemptID int64) (*money.MicroUSD, error) { var v *int64 err := s.pool.QueryRow(ctx, ` select min(a.spend_baseline_micro_usd) from run_attempts a join runs r on r.id = a.run_id where r.book_id = $1 and a.id > $2 and a.spend_baseline_micro_usd is not null`, bookID, attemptID).Scan(&v) if err != nil { return nil, fmt.Errorf("pgstore: read spend bound: %w", err) } if v == nil { return nil, nil } bound := money.MicroUSD(*v) return &bound, nil } // ErrRunNotLive is a run that cannot be stopped because it is already over. Distinct from ErrNoRun, // which is a run this account cannot see at all: the first is the contract's 409 and the second its // 404, and answering the wrong one either tells a stranger that a run exists or tells an owner that // theirs does not. var ErrRunNotLive = errors.New("pgstore: the run is not live") // ErrStopRequested is a restart refused because the run it would restart has been asked to stop. Not // an error of the caller: the reconciler ends the run instead. var ErrStopRequested = errors.New("pgstore: the run has been asked to stop") // RequestStop records that THIS platform asked a run to stop, and hands back the unit to ask. // // The order is the whole design (migration 00014): the intent is COMMITTED before systemd is // touched, so a platform that dies between the two still knows on its next sweep that the run it // finds ended was stopped on purpose — and, just as important, that it must not restart it. // // Idempotent by coalesce: pressing stop twice is one intent with the FIRST timestamp, because the // timestamp is evidence about which of the two events came first and a later one would erase that. // // The revision it answers with is the BOOK's — see ReadRun for why every run-carrying response uses // one counter. func (s *Store) RequestStop(ctx context.Context, userID, runID string, now time.Time) (Run, string, error) { // The bar comes from the shared fragment and is not spelled out again here: a second copy is how // a projection drifts, and this one already needed the segment predicate twice. const q = ` update runs r set stop_requested_at = coalesce(r.stop_requested_at, $3) from books b where r.id = $1 and b.id = r.book_id and b.owner_id = $2 and r.finished_at is null returning ` + runRow + `, coalesce((select a.unit_name from run_attempts a where a.run_id = r.id and a.ended_at is null order by a.attempt_no desc limit 1), '')` var out Run var unit string err := scanRun(s.pool.QueryRow(ctx, q, runID, userID, now), &out, &unit) if errors.Is(err, pgx.ErrNoRows) { // Nothing matched, and the two reasons need different answers. Asked separately and only on // this path, so the ordinary stop stays one round trip. return Run{}, "", s.whyNotLive(ctx, userID, runID) } if err != nil { return Run{}, "", fmt.Errorf("pgstore: request stop: %w", err) } return out, unit, nil } func (s *Store) whyNotLive(ctx context.Context, userID, runID string) error { var visible bool if err := s.pool.QueryRow(ctx, ` select exists (select 1 from runs r join books b on b.id = r.book_id where r.id = $1 and b.owner_id = $2)`, runID, userID).Scan(&visible); err != nil { return fmt.Errorf("pgstore: read run: %w", err) } if visible { return ErrRunNotLive } return ErrNoRun } // ReadRunForResume loads a run and its LAST attempt, whether or not either is still live. // // The reconciler's own list is deliberately not reusable here: it selects the attempt that has not // ended, and every run this call is about has ended. What resume needs is the state the run stopped // in and the attempt whose money and journal position it stopped at. // LatestRun answers which run of the book every book-scoped read resolves — a LIVE run first and // then the newest-started, the `newestRun` order the read model's `lastRun` splices — and whether // that row is still live. The resume gate compares against it: re-opening any OTHER run leaves the // card and every progress frame quoting the wrong row, and the two reasons carry different words — // a LIVE newer run is «the book is being translated», a finished one is «resume the latest». // // ⚠ The order is SPLICED and not written out again, and that is the half of PD-402 the read fix // could not do alone: this query used to hand-write `order by started_at desc` while the read model // spliced a constant, and the two are the same question. Answered differently they invert the // defect instead of closing it — every surface would follow the live run while this gate refused to // resume it, calling it "not the latest". func (s *Store) LatestRun(ctx context.Context, bookID string) (id string, live bool, err error) { err = s.pool.QueryRow(ctx, ` select id, finished_at is null from runs where book_id = $1`+newestRun, bookID).Scan(&id, &live) if errors.Is(err, pgx.ErrNoRows) { return "", false, ErrNoRun } if err != nil { return "", false, fmt.Errorf("pgstore: latest run: %w", err) } return id, live, nil } func (s *Store) ReadRunForResume(ctx context.Context, userID, runID string) (LiveRun, error) { rows, err := s.queryRuns(ctx, ` join run_attempts a on a.run_id = r.id and a.attempt_no = (select max(attempt_no) from run_attempts where run_id = r.id) left join reservations res on res.engine_run_id = r.id || '#' || a.attempt_no and res.state = 'open' where r.id = $1 and b.owner_id = $2`, runID, userID) if err != nil { return LiveRun{}, err } if len(rows) == 0 { return LiveRun{}, ErrNoRun } return rows[0], nil } // ReadRun is the run row as the contract projects it, read under the caller's ownership. // // ⚠ The revision is the BOOK's, not the `runs.revision` column, and that is the contract rather than // a shortcut: "the counter is PER BOOK — every book-scoped read and the id of every stream frame of // that book's run carry the same number" (§Revision), and a client MUST DROP a read whose revision is // below one it has applied. The book card already answers this way (httpapi getBook); a handle that // answered from the run's own column would hand the client a number below the card's and the client, // obeying the contract, would drop the answer to the button it just pressed. func (s *Store) ReadRun(ctx context.Context, userID, runID string) (Run, error) { // The BAR travels with every receipt and not only with the book card: a client that reads // `progress.total` against `ceiling_chapters` (canon §Run) would otherwise see 0/0 on the answer // to its own stop or resume, stamped with a fresh revision — which passes its staleness guard and // rolls the bar it was watching back to zero. const q = ` select ` + runRow + ` from books b join runs r on r.book_id = b.id where r.id = $1 and b.owner_id = $2` var out Run err := scanRun(s.pool.QueryRow(ctx, q, runID, userID), &out) if errors.Is(err, pgx.ErrNoRows) { return Run{}, ErrNoRun } if err != nil { return Run{}, fmt.Errorf("pgstore: read run: %w", err) } return out, nil } // RestartInput is one interrupted attempt being replaced. type RestartInput struct { RunID string AttemptID int64 UserID string BookID string // Ceiling is what is LEFT of the run's budget. A restart that reserved the full ceiling again // would let one run spend it twice. Ceiling money.MicroUSD Offset int64 // EngineBinary overrides the pinned path. Nil means INHERIT — which is the default, because a // resume is the same run continuing and row 139 lets another version in only on purpose. EngineBinary *string // Resnapshot/AcceptRebill GRANT the re-pass consents on re-open (a resume of a run the bank // moved under — P10 §3.1). Widening only: the row keeps a consent it already carries, and the // sum only grows (a smaller fresh projection is covered by the larger consent already given). // The reconciler's restarts pass zero values and change nothing. Resnapshot bool AcceptRebill money.MicroUSD // OnlyIfLive refuses to re-open a run that has already FINISHED. The reconciler sets it and a // resume does not, and the difference is money. // // A sweep decides from a snapshot and writes seconds later, and another generation of the sweep // can close the run inside that window (register row PD-181's class). Without this, the second // pass would clear `finished_at`, take a fresh hold and spawn an engine for a run whose owner has // already been told it ended — and the run the user sees finished starts spending again. A resume // does exactly that on purpose, which is why the guard is the caller's to ask for. OnlyIfLive bool Now time.Time } // ErrRunFinished is a restart refused because another pass has already ended the run. Not an error of // the caller: the pass that holds the stale snapshot simply has nothing left to do. var ErrRunFinished = errors.New("pgstore: the run has already finished") // RestartRun closes an interrupted attempt and opens the next one, with its own hold, in ONE // transaction — the same reason StartRun is one transaction. // // The old attempt's reservation is NOT carried over: it was taken for a process that no longer // exists, and it is closed by the settlement that runs before this. func (s *Store) RestartRun(ctx context.Context, in RestartInput) (LiveRun, error) { var out LiveRun err := s.inTx(ctx, func(tx Tx) error { // Book first, as in every other transaction that touches both (lockBook). This one took the // attempt first and the materializer takes the book first, which is the pair that deadlocked. if err := lockBook(ctx, tx, in.BookID); err != nil { return err } // The RUN row second — the order lockBook writes down, which this transaction used to take // last, after the money. Taking it here also does the work below: what is read from the row is // read under the lock that will do the writing. var stopRequested, finished *time.Time if err := tx.QueryRow(ctx, `select stop_requested_at, finished_at from runs where id = $1 for update`, in.RunID). Scan(&stopRequested, &finished); err != nil { if errors.Is(err, pgx.ErrNoRows) { return ErrNoRun } return fmt.Errorf("pgstore: lock run: %w", err) } // A stop asked for on a run that is STILL LIVE outranks a restart, and the check belongs here // rather than in the caller's snapshot: the reconciler decides to restart, spends seconds // settling (a `tmctl status` call), and the user presses stop inside that window. Restarting // then cleared the fresh intent, opened a second attempt and took a new hold — the user had a // 202 for a stop that never happened and paid for the work they had just cancelled. // // On a FINISHED run the same column is history: a resume re-opens a run that was stopped, and // the intent belongs to the life that ended — it is cleared below with the rest of what said // the run was over. // // A stop arriving DURING this transaction is not lost either: it waits on this row lock and // lands on the re-opened run, which the next sweep then stops. if stopRequested != nil && finished == nil { return ErrStopRequested } // The reconciler asked about a run it believed was still going. If it is not, this pass is // holding a snapshot another generation has already acted on, and re-opening would resurrect a // finished run with a fresh hold — see OnlyIfLive. if in.OnlyIfLive && finished != nil { return ErrRunFinished } // The previous attempt is closed if it is still open, and left exactly as it is if it is not. // Both callers arrive here: the reconciler replaces an attempt that was INTERRUPTED and is // still open, while a resume continues a run whose attempt already ended with a verdict of its // own — and overwriting that verdict would erase how the run the user stopped actually ended. // // What serializes two callers is no longer this row but the next one: `unique (run_id, // attempt_no)` lets exactly one of them insert attempt N+1, and the loser gets the same // ErrNoRun it always got. var attemptNo int if err := tx.QueryRow(ctx, ` update run_attempts set ended_at = coalesce(ended_at, $2), exit_result = coalesce(exit_result, 'interrupted') where id = $1 returning attempt_no`, in.AttemptID, in.Now).Scan(&attemptNo); err != nil { if errors.Is(err, pgx.ErrNoRows) { return ErrNoRun // the attempt is gone } return fmt.Errorf("pgstore: close interrupted attempt: %w", err) } next := attemptNo + 1 var id int64 var pinned string // The new attempt INHERITS the binary the run was pinned to (unified backlog row 139): the // engine ships more often than a run finishes, so resuming on whatever was deployed since is a // different program continuing someone else's work. Changing it is allowed, but only by saying // so — see runs.Config.AllowEngineVersionChange. if err := tx.QueryRow(ctx, ` insert into run_attempts (run_id, attempt_no, started_at, last_offset, engine_binary, engine_run_id) select $1, $2, $3, $4, coalesce($5, prev.engine_binary, ''), $7 from run_attempts prev where prev.id = $6 returning id, engine_binary`, in.RunID, next, in.Now, in.Offset, in.EngineBinary, in.AttemptID, EngineStreamID(in.RunID, next)). Scan(&id, &pinned); err != nil { if isUnique(err, "run_attempts_run_id_attempt_no_key") { return ErrNoRun // another caller opened this attempt first } return fmt.Errorf("pgstore: open next attempt: %w", err) } if err := holdTx(ctx, tx, in.UserID, in.BookID, engineRunKey(in.RunID, next), in.Ceiling, in.Now); err != nil { return err } // finished_at, settled_at and stop_requested_at are CLEARED, and each for its own reason. The // reconciler's restart works on a run where all three are already null, so there it changes // nothing; a RESUME re-opens a run that ended, and leaving them would mean a live run the // one-live-per-book index does not see, money marked resolved that this attempt has not spent // yet, and a stop request from the previous life that would classify this attempt's ending as // a stop nobody asked for. // ⚠ The bar's baselines are NOT re-taken here, and that is the through-bar's one rule (row // 200): the bar counts both waves against the baselines of the run's START, so lifting the // stop moves nothing — the draft half stands at what the first segment did and the last-pass // half continues from where the run began. The re-basing that used to live here is what made // the bar restart from zero at the signing stop. if _, err := tx.Exec(ctx, ` update runs r set status = 'translating', paused_reason = null, finished_at = null, settled_at = null, stop_requested_at = null, resnapshot = r.resnapshot or $2, accept_rebill_micro = greatest(r.accept_rebill_micro, $3), revision = b.revision + 1 from books b where r.id = $1 and b.id = r.book_id`, in.RunID, in.Resnapshot, int64(in.AcceptRebill)); err != nil { // Clearing finished_at puts the run back under the one-live-run-per-book index, and the // book may already have a NEWER live run — nothing stops an account starting one after it // stopped this one. That is a conflict and not a failure: the whole transaction rolls back, // so the hold taken three lines above is undone with it, and the caller gets the same error // a second admission would have got. if isUnique(err, "runs_one_live_per_book") { return ErrRunInFlight } return fmt.Errorf("pgstore: reopen run: %w", err) } if _, err := tx.Exec(ctx, ` update books set status = 'translating', revision = `+nextRevisionOfThisBooksLibrary+` where id = $1`, in.BookID); err != nil { return fmt.Errorf("pgstore: mark book translating: %w", err) } // A restart moves the book back to `translating` — a status change like any other, and one a // watching client has no other way to learn: without the frame the stream stays silent and a // reconnect can even be answered 204 ("stop reconnecting") on a run that is going again. if err := emitStatus(ctx, tx, in.BookID); err != nil { return err } out = LiveRun{RunID: in.RunID, BookID: in.BookID, UserID: in.UserID, AttemptID: id, AttemptNo: next, Ceiling: in.Ceiling, StartedAt: in.Now, AttemptStartedAt: in.Now, EngineBinary: pinned, Position: Position{Offset: in.Offset}} return nil }) if err != nil { return LiveRun{}, err } // The fields the caller needs to spawn but this transaction did not read. // // ⚠ THE ORDER IS DELIBERATELY NOT AMONG THEM. It is resolved on the spawn path, by // ReadOrderForSpawn, once per spawn — see runColumns for why it is not carried on every LiveRun. // What THIS list must never lose is a field the spawn reads FROM the LiveRun: a field left out // here reaches the spawn as its zero value, which for the order used to mean «no boundary» and // therefore «the whole book» (found by TestTheVolumeAllowanceIsWhatIsLeftOfTheOrderAtEverySpawn, // before the order moved off this struct). const q = `select b.workdir, r.verify_bank, r.resnapshot, r.accept_rebill_micro, r.ceiling_chapters from runs r join books b on b.id = r.book_id where r.id = $1` var acceptRebill int64 if err := s.pool.QueryRow(ctx, q, in.RunID).Scan(&out.Workdir, &out.VerifyBank, &out.Resnapshot, &acceptRebill, &out.OrderedChapters); err != nil { return LiveRun{}, fmt.Errorf("pgstore: read restarted run: %w", err) } out.AcceptRebill = money.MicroUSD(acceptRebill) return out, nil } // ReleaseSpawnClaim gives an attempt back after a unit could NOT be created, so the next sweep // retries this attempt instead of reading a recorded-but-absent unit as an interrupted run. func (s *Store) ReleaseSpawnClaim(ctx context.Context, attemptID int64) error { _, err := s.pool.Exec(ctx, `update run_attempts set unit_name = null where id = $1`, attemptID) if err != nil { return fmt.Errorf("pgstore: release spawn claim: %w", err) } return nil } // AttemptReservationOpen reports whether an attempt's money is still reserved. The restart path asks // before it opens a SECOND reservation: settling can legitimately fail (the engine's figure could not // be read), and proceeding then holds the ceiling twice and strands the first hold where no sweep // looks for it again. func (s *Store) AttemptReservationOpen(ctx context.Context, runID string, attempt int) (bool, error) { var open bool err := s.pool.QueryRow(ctx, ` select exists (select 1 from reservations where engine_run_id = $1 and state = 'open')`, engineRunKey(runID, attempt)).Scan(&open) if err != nil { return false, fmt.Errorf("pgstore: read reservation state: %w", err) } return open, nil } // RunPausedReason reads what the run's projection currently says: its STATUS and its ceiling reason. // // It exists because the reconciler's snapshot is taken BEFORE the journal is drained, and the events // that decide how an ending is classified can arrive in that very drain — the ceiling event, and // (since the bank-stop protection of unified-backlog row 240) the bank-stop event, which writes // `awaiting_bank` on the row and nothing the snapshot carries. Judging the ending from the snapshot answered `failed` for a run whose own stream had just // said `ceiling` — one sweep of staleness, on the one branch where being wrong is contractually // visible (acceptance of D39.131, п.3). // // An empty string is "nothing said so", not an error: most runs never pause. func (s *Store) RunPausedReason(ctx context.Context, runID string) (status, reason string, err error) { err = s.pool.QueryRow(ctx, `select status, coalesce(paused_reason, '') from runs where id = $1`, runID).Scan(&status, &reason) if errors.Is(err, pgx.ErrNoRows) { return "", "", nil // the run is gone; the caller's next write refuses on its own } if err != nil { return "", "", fmt.Errorf("pgstore: read run state: %w", err) } return status, reason, nil } // PauseRun records a run that cannot go on, without pretending it failed. // // paused is false when the write did not apply — the run was already over, or the attempt it was // asked to close is no longer the run's live one. It is a RESULT and not a silent nil because the // caller logs "the run was paused" from it, and reporting a pause that did not happen is how a run // that someone else finished got announced twice (register row PD-141). func (s *Store) PauseRun(ctx context.Context, runID string, attemptID int64, reason string, now time.Time) (paused bool, err error) { if !validPauseReason(reason) { return false, fmt.Errorf("pgstore: %q is not a pause reason", reason) } err = s.inTx(ctx, func(tx Tx) error { // Book first, as everywhere else that touches both (see FinishRun). var bookID string if err := tx.QueryRow(ctx, `select id from books where id = (select book_id from runs where id = $1) for update`, runID).Scan(&bookID); err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil } return fmt.Errorf("pgstore: lock book: %w", err) } // A stop the user asked for outranks a pause, and the check belongs inside this TRANSACTION, // under the same locks: the reconciler decides to pause after settling — seconds of a // `tmctl status` call — and a stop landing inside that window would otherwise be answered with // `paused/credit_exhausted`, which // says the money ran out when what happened is that its owner stopped it. var stopRequested *time.Time if err := tx.QueryRow(ctx, `select stop_requested_at from runs where id = $1 for update`, runID).Scan(&stopRequested); err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil } return fmt.Errorf("pgstore: lock run: %w", err) } if stopRequested != nil { return ErrStopRequested } // The attempt being closed must still be a LIVE attempt OF THIS RUN — the third path to carry // the guard `FinishRun` (PD-181) and `FinishUnspawnedStop` (FP5-2) already have, and it was // missing here. Without it a pass holding an old snapshot pauses a run whose attempt has since // been restarted: the run reads `finished`, the SECOND attempt's hold stays open and falls out // of both `ListLiveRuns` (the run is finished) and `UnsettledRuns` (the attempt is not), and an // engine keeps spending under a run its owner is told ran out of money. if err := tx.QueryRow(ctx, ` update runs set status = 'paused', paused_reason = $2, finished_at = $3, revision = (select revision + 1 from books where id = runs.book_id) where id = $1 and finished_at is null and exists (select 1 from run_attempts a where a.id = $4 and a.run_id = runs.id and a.ended_at is null) returning book_id`, runID, reason, now, attemptID).Scan(&bookID); err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil // already over, or this is not the run's live attempt any more } return fmt.Errorf("pgstore: pause run: %w", err) } paused = true if _, err := tx.Exec(ctx, ` update run_attempts set ended_at = coalesce(ended_at, $3) where id = $1 and run_id = $2`, attemptID, runID, now); err != nil { return fmt.Errorf("pgstore: end attempt: %w", err) } // Owing a materialization like every other ending: a run that halted at a ceiling has translated // everything up to it, and that text is bought and paid for. if _, err := tx.Exec(ctx, ` update books set status = 'paused', `+owesAReadingSurface+` revision = `+nextRevisionOfThisBooksLibrary+` where id = $1`, bookID); err != nil { return fmt.Errorf("pgstore: pause book: %w", err) } // This is the reconciler's pause — the real credit-exhausted one — and it sets `finished_at`, // which is what makes the book read as AT REST. Without a frame the stream ends and the next // reconnect is answered 204 while the client still holds `translating`. return emitStatus(ctx, tx, bookID) }) if err != nil { return false, err } return paused, nil } // RunForSpawn is what the worker needs to spawn an attempt. type RunForSpawn struct { LiveRun // AlreadySpawned is a unit name that was recorded for this attempt. The worker must NOT spawn a // second engine for it: a queue job can be retried after a platform restart, and the run it // refers to may well still be running. AlreadySpawned bool } // ReadRunForSpawn loads the live attempt of a run. func (s *Store) ReadRunForSpawn(ctx context.Context, runID string) (RunForSpawn, error) { live, err := s.ListLiveRuns(ctx) if err != nil { return RunForSpawn{}, err } for _, l := range live { if l.RunID == runID { return RunForSpawn{LiveRun: l, AlreadySpawned: l.UnitName != ""}, nil } } return RunForSpawn{}, ErrNoRun } // SpawnRecord is what is written down about an attempt just before its unit is created. type SpawnRecord struct { AttemptID int64 Unit string // Binary is the versioned engine path this attempt is pinned to (unified backlog row 139). Binary string // EngineRunID is the stream identity the platform gives this attempt's engine, in the same write // that claims the right to start it. Recorded BEFORE the handshake rather than learned from it: // the journal is per book, so a reader that adopts the first hello it meets adopts whatever // happens to be written at its offset (runs.engineStreamID). EngineRunID string // Ceiling is the attempt's own budget: the increment the user bought and the amount held. Ceiling money.MicroUSD // CeilingArg is what the engine is actually told, which is the book's cumulative cap and // therefore a different number from the second run of a book onwards (D39.122). CeilingArg money.MicroUSD // Baseline is where the book's lifetime meter stood before this attempt added to it. Baseline money.MicroUSD } // RecordSpawn claims the right to start this attempt, and writes down what is about to run: the // unit, the binary version it is pinned to, the budget it carries and the ceiling it is given. // Written BEFORE the unit is created, so a crash between the two leaves a record to reconcile rather // than an unattributable process. // // claimed is false when the attempt already has a unit name, when the attempt has ENDED, or when its // run is over. It is a COMPARE-AND-SET rather than a plain update because two callers legitimately // reach here at once — the queue worker that was handed the run and the reconciler that found it // unspawned — and the loser must not start a second engine. systemd would refuse the duplicate NAME, // so that accident was survivable; surviving by someone else's uniqueness rule is not the same as // being correct, and the day a resume changes the naming it stops holding. // // ⚠ The three conditions BESIDES the unit name are money, and they were not here until a run could be // stopped before it ever spawned. The window: a worker sits inside `bookMeter` for the seconds a // `tmctl status` takes, the user stops the run, the sweep ends it and gives the whole hold back // (nothing was spawned) — and the worker then wakes up and creates a unit for a run that is finished // and settled. That engine would spend against its own book cap with NO open reservation, and // nothing would ever look at it: the reconciler lists runs by `finished_at is null` and settlements // by an open reservation, so it is in neither list. The stop INTENT is asked about one step earlier // for the same reason: it lands while the worker is inside that same `bookMeter`, before any sweep // could have finished the run, and a claim granted then starts an engine for work its owner // cancelled before it began. All three are asked in the statement that claims, because a check made // before it is a check with a window after it. func (s *Store) RecordSpawn(ctx context.Context, r SpawnRecord) (claimed bool, err error) { // What a PREVIOUS claim of this attempt decided is kept. The claim is given back when the unit // could not be created (ReleaseSpawnClaim), and "could not be created" is not the same as "was not // created": a systemd-run that was killed after it had already asked for the unit reports a failure // and leaves an engine running. The next claim would then overwrite the baseline with a meter that // engine has been moving, and the attempt would be billed for the difference from a figure that // already includes its own work — an underpayment nothing later looks for. Where the unit really // was not created the two values are identical, so keeping the first costs nothing. // engine_run_id is ASSIGNED and not coalesced. It is derived from (run, attempt), so a retry of the // same claim computes the same string and the "keep what the first claim decided" rule buys // nothing — what a coalesce could keep is a FOREIGN id an early drain adopted before the row was // named (see EngineStreamID), and keeping that would blind the run for its whole life. Attempts // created since EngineStreamID moved into their INSERT already carry it; this is the backfill for // the ones that do not. tag, err := s.pool.Exec(ctx, ` update run_attempts set unit_name = $2, engine_binary = $3, ceiling_micro_usd = $4, -- Incremented inside the claim itself: a count written anywhere else would have a -- window beside it. Read by AbandonOrder.ProofSpawns (PD-424). spawns = spawns + 1, ceiling_arg_micro_usd = case when spend_baseline_micro_usd is null then $5 else ceiling_arg_micro_usd end, spend_baseline_micro_usd = coalesce(spend_baseline_micro_usd, $6), engine_run_id = coalesce(nullif($7, ''), engine_run_id) where id = $1 and unit_name is null and ended_at is null and exists (select 1 from runs r where r.id = run_attempts.run_id and r.finished_at is null and r.stop_requested_at is null)`, r.AttemptID, r.Unit, r.Binary, int64(r.Ceiling), int64(r.CeilingArg), int64(r.Baseline), r.EngineRunID) if err != nil { return false, fmt.Errorf("pgstore: record spawn: %w", err) } return tag.RowsAffected() == 1, nil } // MarkParked records that this attempt's projection has parked, or that it no longer has. // // ⚠ SET ONCE AND CLEARED ONCE, which is what makes the timestamp mean «since when» rather than «as of // the last sweep». The park is re-derived from (cursor, journal) on every pass, so a plain write // would move the stamp forward forever and the one question an operator asks — how long has this run // been quiet — would have no answer anywhere. // // The caller only reaches here when its verdict DIFFERS from the snapshot's, so the steady state // costs no statement at all; the guards below are the second half of that, for the two sweeps that // disagree at once. func (s *Store) MarkParked(ctx context.Context, attemptID int64, parked bool, now time.Time) error { // Two whole statements rather than one built from pieces: the battery plans every statement in // this package against the migrated schema, and it can only do that for a CONSTANT // (sqlgate_test.go). A query assembled at run time is a query nothing checks. var err error if parked { _, err = s.pool.Exec(ctx, `update run_attempts set parked_at = $2 where id = $1 and parked_at is null`, attemptID, now) } else { _, err = s.pool.Exec(ctx, `update run_attempts set parked_at = null where id = $1 and parked_at is not null`, attemptID) } if err != nil { return fmt.Errorf("pgstore: mark the attempt parked: %w", err) } return nil } // SaveCursor persists the tailer's position when nothing was materialized — a re-read of lines the // cursor already covers still moves the byte hint, and losing that means re-reading them forever. func (s *Store) SaveCursor(ctx context.Context, attemptID int64, p Position) error { _, err := s.pool.Exec(ctx, ` update run_attempts set last_offset = $2 where id = $1 and last_offset < $2`, attemptID, p.Offset) if err != nil { return fmt.Errorf("pgstore: save cursor: %w", err) } return nil } // Quarantine stops materializing an attempt without touching the run. // // The engine is NOT stopped: it is spending money the account has already reserved, and our // inability to read its journal is not a reason to throw that away. What stops is the projection — // after this the run's state is only as fresh as the resync channel makes it, and the reason says // so out loud rather than leaving a screen that quietly stopped moving. func (s *Store) Quarantine(ctx context.Context, attemptID int64, reason string) error { _, err := s.pool.Exec(ctx, `update run_attempts set quarantine_reason = $2 where id = $1 and quarantine_reason is null`, attemptID, reason) if err != nil { return fmt.Errorf("pgstore: quarantine attempt: %w", err) } return nil } var ( // ErrNotQuarantined is a lift asked for on a live attempt that is not quarantined. Its own word, // apart from "no such run": the operator was right about the run and wrong about its state, and // the answer is "nothing to do" rather than "look elsewhere". ErrNotQuarantined = errors.New("pgstore: the run's live attempt is not quarantined") // ErrNoLiveAttempt is a lift asked for on a run with no attempt still open. A finished run's // journal is not materialized by anyone, so there is nothing a lift could resume. ErrNoLiveAttempt = errors.New("pgstore: the run has no live attempt") ) // LiftedQuarantine is what Unquarantine found and cleared: the attempt, the reason it carried, and // the cursor the next sweep reads on from. type LiftedQuarantine struct { AttemptNo int Reason string Position Position } // Unquarantine clears the quarantine of a run's LIVE attempt, so the next sweep materializes its // journal again from the cursor the projection stopped at. // // It is the operator's half of Quarantine (PD-426): the state it stands for — a journal this build // could not read from here on — is not always permanent (an operator's stray tmctl stops writing; a // build is replaced; a release relaxes a reader's rule), and a column nothing clears would make it // so. What is NOT touched is the cursor: it is the record of what was applied, and the tailer // resumes from it. If the same bytes are still unreadable the next sweep quarantines again with the // same reason, which is the honest answer and is visible in the runs listing. No money moves and no // process is touched. func (s *Store) Unquarantine(ctx context.Context, runID string) (LiftedQuarantine, error) { var out LiftedQuarantine err := s.inTx(ctx, func(tx pgx.Tx) error { // The RUN row first, in the order every transaction on a run takes (lockBook: book, then run, // then its attempts). RestartRun closes the live attempt and opens the next one under this // same row lock, so a lift that queued behind it reads the attempt that replaced the old one // rather than a snapshot in which the run has no live attempt at all. var one int if err := tx.QueryRow(ctx, `select 1 from runs where id = $1 for update`, runID).Scan(&one); err != nil { if errors.Is(err, pgx.ErrNoRows) { return ErrNoRun } return fmt.Errorf("pgstore: lock run: %w", err) } var id int64 var reason *string err := tx.QueryRow(ctx, ` select id, attempt_no, quarantine_reason, last_offset, last_seq from run_attempts where run_id = $1 and ended_at is null order by attempt_no desc limit 1 for update`, runID). Scan(&id, &out.AttemptNo, &reason, &out.Position.Offset, &out.Position.LastSeq) if errors.Is(err, pgx.ErrNoRows) { return ErrNoLiveAttempt } if err != nil { return fmt.Errorf("pgstore: read live attempt: %w", err) } if reason == nil { return ErrNotQuarantined } out.Reason = *reason if _, err := tx.Exec(ctx, `update run_attempts set quarantine_reason = null where id = $1`, id); err != nil { return fmt.Errorf("pgstore: lift quarantine: %w", err) } return nil }) if err != nil { return LiftedQuarantine{}, err } return out, nil }