1964 lines
107 KiB
Go
1964 lines
107 KiB
Go
package pgstore
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
|
|
"textmachine/platform/internal/money"
|
|
)
|
|
|
|
// Tx is a transaction of this store. Exported so a caller can join its own write to one — the queue
|
|
// does, because a run row and its queue entry must be written together — without every package in
|
|
// the zone importing pgx to name the type.
|
|
type Tx = pgx.Tx
|
|
|
|
var (
|
|
// ErrRunInFlight is a second live run on one book. The database refuses it (the partial unique
|
|
// index runs_one_live_per_book) rather than the API explaining it: the engine holds an EXCLUSIVE
|
|
// lock on the project file, so two live runs is not a state anything downstream can represent.
|
|
ErrRunInFlight = errors.New("pgstore: the book already has a live run")
|
|
// ErrNoRun is a run that does not exist or does not belong to the caller.
|
|
ErrNoRun = errors.New("pgstore: no such run")
|
|
)
|
|
|
|
// StartRunInput is one accepted run request.
|
|
type StartRunInput struct {
|
|
UserID string
|
|
BookID string
|
|
VerifyBank bool
|
|
OrderedChapters int
|
|
// Ceiling is what those chapters are worth and what is HELD before anything is spawned. The hold
|
|
// and the ceiling handed to the engine are the same number by construction: the hold makes the
|
|
// credit unavailable to any other run, and the engine stops itself there, so an overspend is
|
|
// impossible even while the platform is blind (D39.100).
|
|
Ceiling money.MicroUSD
|
|
Now time.Time
|
|
// Resnapshot/AcceptRebill are the re-pass consents, decided ONCE at admission and stored on the
|
|
// run the same way VerifyBank is: every spawn of every attempt reads the row, so argv is stable
|
|
// across respawns and a flag cannot appear mid-run (P10 §3.1). AcceptRebill is the CONCRETE sum
|
|
// the platform consents to (`--accept-rebill=<usd>`) — the projection the user was shown, never
|
|
// a blanket yes (D20.2-Q2).
|
|
Resnapshot bool
|
|
AcceptRebill money.MicroUSD
|
|
// OrderedUnits is set for every run whose order was phrased in CHARACTERS, and nil for every other
|
|
// shape. Nil is the ordinary case and leaves the run's bar counted in chapters, exactly as before
|
|
// this pack. See migration 00033 for why such a run needs a bar of its own: counted in chapters an
|
|
// order that stops inside a chapter reads `0/N` for its whole life, which is the state the
|
|
// admission refuses a book for.
|
|
//
|
|
// ⚠ THE CONDITION IS THE PHRASING, NOT WHERE THE ORDER LANDED. An earlier edition of this line
|
|
// said «only for a run whose order does not close whole chapters», and that is measurably false —
|
|
// see pricing.Quote.UnitShaped for the measurement and for why the phrasing is the right test.
|
|
OrderedUnits *int
|
|
// BondFunded is whether this run's hold has room for the book-level consistency passes on top of
|
|
// the work it bought. Recorded on the RUN because the order form's answer is a QUOTE and the
|
|
// balance moves under it — see migration 00033.
|
|
BondFunded bool
|
|
// Order is THE ORDER, and it is written on the BOOK rather than on the run (unified backlog
|
|
// row 280).
|
|
//
|
|
// It lived on the run because a ceiling was a property of one process — and that is exactly what
|
|
// made «raise the limit» mean «start a NEW run with a bigger ceiling», a sentence nobody could
|
|
// explain to a buyer. On the book, continuing after a top-up is the same order again, and the
|
|
// re-pass door stands beside it as a second KIND of order rather than as a special case.
|
|
//
|
|
// ⛔ NIL MEANS «LEAVE THE BOOK'S ORDER ALONE», and it is not a convenience. The RE-PASS buys no
|
|
// volume — it re-makes what a correction touched — so it has no order to record; and the zero
|
|
// VALUE of BookOrder means «the whole book», which written over an existing boundary silently
|
|
// WIDENS a paid order: «bought through chapter 12» became «bought the whole book», with nothing
|
|
// said and nothing charged for the difference. A purchase that buys no volume must not be able to
|
|
// change what was bought.
|
|
Order *BookOrder
|
|
}
|
|
|
|
// BookOrder is what a book is bought through: an IDENTITY of the boundary, never a count and never
|
|
// an ordinal. See migration 00033 for the whole of why — in one line, an ordinal survives a re-cut
|
|
// by silently naming different text, and that is a change of what was bought that nobody sees.
|
|
type BookOrder struct {
|
|
// ThroughChapterID is the last chapter bought, for an order phrased in chapters. Empty for the
|
|
// whole book and for a character order.
|
|
ThroughChapterID string
|
|
// ThroughUnitID is the last output unit bought, for an order phrased in characters — which stops
|
|
// inside a chapter, so no chapter identity can express where. Empty otherwise.
|
|
ThroughUnitID string
|
|
// ThroughChapterNumber is the ordinal AT THE MOMENT OF THE ORDER: a label for the screen, and the
|
|
// thing whose disagreement with the identity is what makes a shift visible. Zero when there is
|
|
// none.
|
|
ThroughChapterNumber int
|
|
}
|
|
|
|
// Whole reports the whole-book order — the ratified default, and the only order with no boundary.
|
|
func (o BookOrder) Whole() bool { return o.ThroughChapterID == "" && o.ThroughUnitID == "" }
|
|
|
|
// StartedRun is what the caller needs after a successful start.
|
|
type StartedRun struct {
|
|
Run
|
|
AttemptID int64
|
|
// JournalOffset is the size of the book's journal at the moment this attempt was admitted. The
|
|
// journal is per BOOK and append-only, so an attempt's own lines begin after everything already
|
|
// in it; without this the tailer would adopt the previous attempt's handshake.
|
|
JournalOffset int64
|
|
}
|
|
|
|
// StartRun admits a run: the run row, its first attempt, the hold and the queue entry, all in ONE
|
|
// transaction.
|
|
//
|
|
// One transaction because each of them alone is a way to lose money or work. A hold without a run
|
|
// row is credit reserved for nothing; a run row without a hold is a run that spends credit nobody
|
|
// set aside; a queue entry without either is a worker spawning an engine against a book whose
|
|
// bookkeeping does not exist. research/25 names the first of those explicitly ("a hold can leak
|
|
// before the directory exists"), and the answer is not a compensating sweep but not splitting the
|
|
// write in the first place.
|
|
//
|
|
// enqueue is handed the transaction so the queue's own insert joins it. It is a callback rather than
|
|
// a second call because River owns its SQL and this package owns its own: neither reaches into the
|
|
// other, and the atomicity is still real.
|
|
func (s *Store) StartRun(ctx context.Context, in StartRunInput, journalOffset int64, enqueue func(context.Context, Tx, string) error) (StartedRun, error) {
|
|
// Zero is ONE legal shape: the re-pass run (P10), which buys no chapters — it re-walks the book
|
|
// under the consents, so Resnapshot is what marks it. Everything else still needs a positive
|
|
// ceiling; the bar of a zero-ceiling row switches to the re-pass form (readmodel.runTotal), so
|
|
// the 0/0 frame stays unreachable.
|
|
if in.OrderedChapters < 0 || (in.OrderedChapters == 0 && !in.Resnapshot) {
|
|
return StartedRun{}, fmt.Errorf("pgstore: a run needs a positive chapter ceiling, got %d", in.OrderedChapters)
|
|
}
|
|
out := StartedRun{JournalOffset: journalOffset}
|
|
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
|
// The book first, before the hold — see lockBook for why the order is global and not local.
|
|
// Here it also makes the race between two admissions of one book a queue instead of a
|
|
// collision: the loser still gets ErrRunInFlight, it just gets it after waiting.
|
|
if err := lockBook(ctx, tx, in.BookID); err != nil {
|
|
return err
|
|
}
|
|
runID := newID("run")
|
|
// BOTH baselines are captured HERE, in the same statement that creates the run: what the book
|
|
// had already finished is what each half of this run's bar is measured from (see runDone) —
|
|
// `chapters_before` on the EDIT column, `draft_before` on the DRAFT one. Neither is re-based
|
|
// afterwards: the bar is one monotonic fraction through both waves, and a base that moved
|
|
// mid-run is how it used to restart from zero at the signing stop.
|
|
//
|
|
// ⚠ FIXED predicates, deliberately NOT the flag-following finishedUnits: `edit_wave` can flip
|
|
// false→true AFTER this insert (the engine announces its wave shape with its first progress
|
|
// event), and a baseline captured through the flag would sit on the draft column while the
|
|
// numerator moved to the edit one — the bar then read 0/N forever (reviewer's blocker, P9).
|
|
// Captured flag-free, each baseline is subtracted only from the numerator of its own column
|
|
// (runDone pairs them at READ time), so no flip can strand the bar.
|
|
const insertRun = `
|
|
insert into runs (id, book_id, status, verify_bank, ceiling_chapters, started_at, revision,
|
|
resnapshot, accept_rebill_micro, bond_funded, ordered_units,
|
|
chapters_before, draft_before, units_before, draft_units_before)
|
|
select $1, $2, 'translating', $3, $4, $5, b.revision + 1, $7, $8, $9, $10,
|
|
(select count(*) from chapters c
|
|
where c.book_id = b.id and c.units_total > 0 and c.units_edit_done >= c.units_total),
|
|
(select count(*) from chapters c
|
|
where c.book_id = b.id and c.units_total > 0 and c.units_draft_done >= c.units_total),
|
|
` + bookUnitsEditDone + `, ` + bookUnitsDraftDone + `
|
|
from books b where b.id = $2 and b.owner_id = $6
|
|
returning id, book_id, revision, status, verify_bank,
|
|
(case when ordered_units is not null then null else ceiling_chapters end),
|
|
ordered_units, coalesce(paused_reason, ''), started_at, finished_at`
|
|
err := tx.QueryRow(ctx, insertRun, runID, in.BookID, in.VerifyBank, in.OrderedChapters, in.Now, in.UserID,
|
|
in.Resnapshot, int64(in.AcceptRebill), in.BondFunded, in.OrderedUnits).
|
|
Scan(&out.ID, &out.BookID, &out.Revision, &out.Status, &out.VerifyBank, &out.OrderedChapters,
|
|
&out.OrderedUnits, &out.PausedReason, &out.StartedAt, &out.FinishedAt)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrNoBook // the book is missing, or it is not this account's
|
|
}
|
|
if isUnique(err, "runs_one_live_per_book") {
|
|
return ErrRunInFlight
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("pgstore: insert run: %w", err)
|
|
}
|
|
// The receipt's bar is READ rather than zeroed: the numerator opens at 0 by construction
|
|
// (both halves measure THIS run's work from the baselines the same statement just took), but
|
|
// the DENOMINATOR is the run's own — a continuation run over a drafted backlog owes fewer
|
|
// draft passes (runTotal folds draftWork in), and the screen reads its scale against it.
|
|
if err := tx.QueryRow(ctx, `select `+runDone+`, `+runTotal+`, `+runStage+`
|
|
from books b `+lastRun+` where b.id = $1`, in.BookID).
|
|
Scan(&out.Progress.Done, &out.Progress.Total, &out.Progress.Stage); err != nil {
|
|
return fmt.Errorf("pgstore: read the new run's bar: %w", err)
|
|
}
|
|
if err := tx.QueryRow(ctx, `
|
|
insert into run_attempts (run_id, attempt_no, started_at, last_offset, engine_run_id)
|
|
values ($1, 1, $2, $3, $4) returning id`,
|
|
out.ID, in.Now, journalOffset, EngineStreamID(out.ID, 1)).Scan(&out.AttemptID); err != nil {
|
|
return fmt.Errorf("pgstore: insert attempt: %w", err)
|
|
}
|
|
// The hold is taken here, BEFORE anything is spawned, and it is the enforcement half of the
|
|
// money design rather than an accounting note.
|
|
if err := holdTx(ctx, tx, in.UserID, in.BookID, engineRunKey(out.ID, 1), in.Ceiling, in.Now); err != nil {
|
|
return err
|
|
}
|
|
// THE ORDER IS RECORDED WITH THE ADMISSION, in the transaction that took the money for it. A
|
|
// later write would leave a window in which a run is live and the book cannot say what was
|
|
// bought — and that is precisely the question every spawn of every attempt asks, to compute
|
|
// `--max-units`.
|
|
// ⚠ THE ORDER IS WRITTEN ONLY WHEN THERE IS ONE. A purchase that buys no volume (the re-pass)
|
|
// leaves the book's boundary exactly as it stands: writing its zero value over an existing one
|
|
// would turn «bought through chapter 12» into «bought the whole book» — see StartRunInput.Order.
|
|
if in.Order != nil {
|
|
if _, err := tx.Exec(ctx, `
|
|
update books set ordered_at = $2,
|
|
ordered_through_chapter_id = nullif($3, ''),
|
|
ordered_through_unit_id = nullif($4, ''),
|
|
ordered_through_chapter_number = nullif($5, 0)
|
|
where id = $1`,
|
|
in.BookID, in.Now, in.Order.ThroughChapterID, in.Order.ThroughUnitID,
|
|
in.Order.ThroughChapterNumber); err != nil {
|
|
return fmt.Errorf("pgstore: record the order: %w", err)
|
|
}
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
update books set status = 'translating',
|
|
revision = `+nextRevisionOfThisBooksLibrary+` where id = $1`,
|
|
in.BookID); err != nil {
|
|
return fmt.Errorf("pgstore: mark book translating: %w", err)
|
|
}
|
|
if err := emitStatus(ctx, tx, in.BookID); err != nil {
|
|
return err
|
|
}
|
|
if enqueue != nil {
|
|
return enqueue(ctx, tx, out.ID)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return StartedRun{}, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// engineRunKey is the reservation key of an attempt.
|
|
//
|
|
// ⚠ It is the PLATFORM's attempt identity, not the engine's `engine_run_id`: the engine mints its
|
|
// own id per invocation and the platform only learns it from the handshake, which arrives after the
|
|
// money has already been reserved. Reservations are keyed by attempt for exactly that reason — the
|
|
// hold has to exist before there is anything to key it by on the engine's side.
|
|
func engineRunKey(runID string, attempt int) string {
|
|
return fmt.Sprintf("%s#%d", runID, attempt)
|
|
}
|
|
|
|
// EngineStreamID is the identity this attempt's engine announces its event stream under — the
|
|
// `engine_run_id` half of the ratified idempotency key. The PLATFORM chooses it and hands it to the
|
|
// engine in the environment (TM_TRACE_ID, unified backlog row 102).
|
|
//
|
|
// ⚠ It is written when the ATTEMPT ROW IS CREATED and not when the unit is, and that is the whole
|
|
// point. The journal is per BOOK, and a reader with no name of its own adopts the first handshake it
|
|
// meets at its offset — so between admission and spawn (seconds inside a `tmctl status` call, or a
|
|
// whole sweep interval) a stream belonging to somebody else could be adopted, its `ceiling` event
|
|
// materialized onto this attempt, and the attempt's own handshake then refused as a sequence gap.
|
|
// Naming the stream at creation closes the window rather than narrowing it.
|
|
//
|
|
// Per ATTEMPT and not per run: seq restarts at 1 in every process. The shape is what the engine
|
|
// accepts without rewriting it — printable ASCII, no spaces, well under its 64-character bound.
|
|
func EngineStreamID(runID string, attempt int) string {
|
|
return fmt.Sprintf("tm-stream-%s-%d", runID, attempt)
|
|
}
|
|
|
|
func isUnique(err error, constraint string) bool {
|
|
var pg *pgconn.PgError
|
|
return errors.As(err, &pg) && pg.Code == "23505" && pg.ConstraintName == constraint
|
|
}
|
|
|
|
// LiveRun is a run the reconciler has to make a decision about.
|
|
type LiveRun struct {
|
|
RunID string
|
|
BookID string
|
|
UserID string
|
|
Workdir string
|
|
AttemptID int64
|
|
AttemptNo int
|
|
UnitName string
|
|
EngineRunID string
|
|
Position Position
|
|
Quarantined bool
|
|
// Parked is what the last sweep recorded about this attempt's projection. Carried on the snapshot
|
|
// so a pass whose verdict is unchanged issues no write at all.
|
|
Parked bool
|
|
VerifyBank bool
|
|
// Resnapshot/AcceptRebill are the run's re-pass consents (P10): read by every spawn so the argv
|
|
// is the admission's decision, never a sweep's re-derivation. AcceptRebill == 0 means no consent
|
|
// was given (the flag is not passed); a consent is always a concrete sum.
|
|
Resnapshot bool
|
|
AcceptRebill money.MicroUSD
|
|
// OrderedChapters is the run's whole budget, in the unit the user chose; Ceiling is what THIS
|
|
// attempt was allowed to spend. They differ after a restart, which gets what is left.
|
|
OrderedChapters int
|
|
Ceiling money.MicroUSD
|
|
// ⚠ THE ORDER IS DELIBERATELY NOT HERE. It used to be five more columns on this struct, and
|
|
// this struct is read by `runColumns` — the reconciler's own list, for EVERY live run on EVERY
|
|
// sweep. Resolving an order means walking the book's units, so carrying it here put three passes
|
|
// over every unit of every live book into a query whose job is to say which runs exist, for
|
|
// figures only the SPAWN reads. The spawn is rare and the sweep is not: it is read there, once,
|
|
// by ReadOrderForSpawn. Found by an adversarial pass on the plan, not by a failure.
|
|
// EngineBinary is the VERSIONED path this attempt was pinned to (unified backlog row 139). Read
|
|
// back and USED — for the resume and for the repair channel — because a pin nothing reads is a
|
|
// column, not a pin: the engine ships more often than a run finishes, and asking the CURRENT
|
|
// binary about a book an older one is translating is a different question.
|
|
EngineBinary string
|
|
// SpendBaseline is what the BOOK had already cost when this attempt began. Nil means it was never
|
|
// captured, which no current path produces (the spawn refuses without it).
|
|
SpendBaseline *money.MicroUSD
|
|
// CeilingArg is the limit a PREVIOUS claim of this attempt already handed the engine. Read back
|
|
// because a retry must hand the same one: recomputing it from a counter that the first claim's own
|
|
// engine has been moving gives that engine a second, larger limit and leaves the column that is
|
|
// supposed to answer "what limit did that process have" describing neither. Zero means no claim
|
|
// has been recorded (the column predates nothing else being able to tell).
|
|
CeilingArg money.MicroUSD
|
|
// PausedReason is what the stream already said about this run. A run that reported a ceiling halt
|
|
// is `paused` however its process then ended (contract §BookStatus: never `failed`).
|
|
PausedReason string
|
|
// Status is the run's product status. The reconciler works on live runs, where it is
|
|
// `translating`; the resume path reads a run that has already ended and decides by it.
|
|
Status string
|
|
// StopRequestedAt is when THIS platform asked the run to stop, and it is the only thing that can
|
|
// tell a stop from a crash: the engine catches SIGTERM and exits 1, so the marker says
|
|
// `exit-code/exited/1` for both (register row PD-152). Nil means nobody asked.
|
|
StopRequestedAt *time.Time
|
|
// ReconcileFailures is how many passes in a row have failed to reconcile THIS attempt. It is read
|
|
// back so the next deferral can be computed from it, and it is on the attempt rather than the run
|
|
// because a restart opens a new process that deserves a clean slate.
|
|
ReconcileFailures int
|
|
// StartedAt is the RUN's start; AttemptStartedAt is THIS attempt's. The reconciler's grace is
|
|
// measured against the second: a restarted attempt inherits a start time hours old, and the
|
|
// grace then expires before systemd has had a chance to create anything.
|
|
StartedAt time.Time
|
|
AttemptStartedAt time.Time
|
|
}
|
|
|
|
// Position mirrors ingest.Position without importing it: this package owns the columns, and the
|
|
// dependency runs the other way.
|
|
type Position struct {
|
|
Offset int64
|
|
LastSeq int64
|
|
LastHash []byte
|
|
}
|
|
|
|
// ListLiveRuns returns every run that has not finished, with its live attempt.
|
|
//
|
|
// This is the reconciler's source of truth, together with the book's directory — NOT systemd
|
|
// (research/25 §Опс). A transient unit does not survive a reboot and is unloaded the moment it
|
|
// exits, so asking systemd "what is running" answers a different question than "what did this
|
|
// platform promise a user".
|
|
func (s *Store) ListLiveRuns(ctx context.Context) ([]LiveRun, error) {
|
|
return s.queryRuns(ctx, `
|
|
join run_attempts a on a.run_id = r.id and a.ended_at is null
|
|
left join reservations res on res.engine_run_id = r.id || '#' || a.attempt_no
|
|
and res.state = 'open'
|
|
where r.finished_at is null
|
|
order by r.started_at`)
|
|
}
|
|
|
|
// RunsToReconcile is ListLiveRuns minus the attempts that are serving a deferral.
|
|
//
|
|
// The two lists are deliberately different and the difference is the whole of the starvation fix.
|
|
// This one is what the sweep WORKS through, and an attempt the last pass could not finish is not in
|
|
// it until its deferral lapses. ListLiveRuns stays whole because its other caller is telemetry
|
|
// (`Lag`), and a run hidden from the number that says how far behind the projection is would be a
|
|
// run nobody can see is stuck.
|
|
//
|
|
// Ordering is unchanged — oldest first — and it is only safe BECAUSE of the filter: a list ordered
|
|
// the same way on every pass hands the head to whichever item is oldest, so an item that can never
|
|
// succeed held that head forever. Deferral is what takes it out of the head rather than a different
|
|
// ordering, because "oldest first" is right for everything that is merely slow.
|
|
//
|
|
// ⚠ A STOP OUTRANKS A DEFERRAL: the deferral is a statement about the past, and the sweep is the only
|
|
// reader of live runs, so a stopped run sat at `translating` with its hold reserved until the backoff
|
|
// lapsed — minutes on a run already deferred a few times, and against Stop's own promise to re-issue
|
|
// on the next pass.
|
|
func (s *Store) RunsToReconcile(ctx context.Context, now time.Time) ([]LiveRun, error) {
|
|
return s.queryRuns(ctx, `
|
|
join run_attempts a on a.run_id = r.id and a.ended_at is null
|
|
left join reservations res on res.engine_run_id = r.id || '#' || a.attempt_no
|
|
and res.state = 'open'
|
|
where r.finished_at is null
|
|
and (a.reconcile_after is null or a.reconcile_after <= $1
|
|
or r.stop_requested_at is not null)
|
|
order by r.started_at`, now)
|
|
}
|
|
|
|
// DeferRun records that a pass could not finish this attempt, and when the next one may try.
|
|
//
|
|
// It returns the number of CONSECUTIVE failures including this one, so the caller can decide the
|
|
// next deadline from it and can say out loud when an attempt has stopped being merely slow. The
|
|
// count lives in the database and not in the process because the process restarts — several times a
|
|
// week by design (research/25) — and a counter that restarts with it never reaches any threshold.
|
|
//
|
|
// The reason goes through `truncateReason` — repaired to valid UTF-8 and then cut on a rune
|
|
// boundary — and it is an operator's field, never a client's: it is a Go error string, which may
|
|
// name a path, a systemd unit or the engine's own stderr, while the wire's vocabulary of failure is
|
|
// a closed enum decided elsewhere (`failureReason`). See truncateReason for why the repair is not
|
|
// decoration: a write that Postgres refuses here is a failure that never gets counted, and the
|
|
// attempt then holds the head of the list forever.
|
|
// ⚠ NOT restricted to live attempts: a finished one whose MONEY is still open holds the head of the
|
|
// settlement list the same way, and the two phases select on opposite sides of `ended_at`.
|
|
func (s *Store) DeferRun(ctx context.Context, attemptID int64, after time.Time, reason string) (int, error) {
|
|
var failures int
|
|
err := s.pool.QueryRow(ctx, `
|
|
update run_attempts
|
|
set reconcile_failures = reconcile_failures + 1,
|
|
reconcile_after = $2,
|
|
reconcile_error = $3
|
|
where id = $1
|
|
returning reconcile_failures`, attemptID, after, truncateReason(reason)).Scan(&failures)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
// No such attempt. Nothing to defer, and nothing wrong.
|
|
return 0, nil
|
|
}
|
|
if err != nil {
|
|
return 0, fmt.Errorf("pgstore: defer the reconciliation of a run: %w", err)
|
|
}
|
|
return failures, nil
|
|
}
|
|
|
|
// AttemptEnded reports whether an attempt is over. It exists for one decision and says so: when the
|
|
// RECONCILIATION phase writes a deferral after a failure, the attempt it is deferring may already
|
|
// have ended during that same pass — `finish`, `finishStopped` and `restart` all close it and then
|
|
// settle — and an ended attempt belongs to the SETTLEMENT phase's list, whose deferral is capped
|
|
// shorter because it gates the user's own resume (runs.settlementBackoffCap).
|
|
func (s *Store) AttemptEnded(ctx context.Context, attemptID int64) (bool, error) {
|
|
var ended bool
|
|
if err := s.pool.QueryRow(ctx,
|
|
`select ended_at is not null from run_attempts where id = $1`, attemptID).Scan(&ended); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return false, nil
|
|
}
|
|
return false, fmt.Errorf("pgstore: read whether the attempt ended: %w", err)
|
|
}
|
|
return ended, nil
|
|
}
|
|
|
|
// ClearRunDeferral forgets the failures of an attempt that has just been reconciled successfully.
|
|
//
|
|
// Conditional on there being something to forget: the overwhelming majority of passes succeed, and
|
|
// an unconditional write would put a row update on every live run on every tick — the cost the
|
|
// deferral exists to avoid, paid back on the healthy path.
|
|
func (s *Store) ClearRunDeferral(ctx context.Context, attemptID int64) error {
|
|
if _, err := s.pool.Exec(ctx, `
|
|
update run_attempts set reconcile_failures = 0, reconcile_after = null, reconcile_error = null
|
|
where id = $1 and (reconcile_failures <> 0 or reconcile_after is not null)`, attemptID); err != nil {
|
|
return fmt.Errorf("pgstore: clear the deferral of a run: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// StalledRun is a run the reconciler keeps failing on, as an operator needs to see it.
|
|
type StalledRun struct {
|
|
RunID string
|
|
Title string
|
|
// AttemptID and Spawns are the subject and generation of the proof `run abandon` takes from this
|
|
// listing. See AbandonOrder.ProofAttemptID.
|
|
AttemptID int64
|
|
Spawns int
|
|
UnitName string
|
|
AttemptNo int
|
|
Status string
|
|
Failures int
|
|
NextTry *time.Time
|
|
LastError string
|
|
// HeldMicroUSD is what is still reserved against this run, and HeldSeconds how long. It is the
|
|
// operator's whole question: a stalled run is money that is not moving and a book that cannot
|
|
// start another run.
|
|
HeldMicroUSD int64
|
|
HeldSeconds float64
|
|
// SpentMicroUSD is what the engine has reported spending on THIS attempt, as the stream last said.
|
|
// It is here because of the decision this table exists to inform: `run abandon --release-hold`
|
|
// gives the hold back WHOLE, and without this number that choice is made blind to how much work is
|
|
// being written off.
|
|
//
|
|
// ⚠ IT IS A DIFFERENCE, not the column: `spend_micro_usd` holds the engine's figure verbatim, and
|
|
// that figure is the BOOK's lifetime total (ingest.Spend, "CUMULATIVE"), so printing it raw told an
|
|
// operator what the book had cost since upload and called it this attempt's. Same arithmetic as
|
|
// `attemptSpend`, clamp included.
|
|
//
|
|
// ⚠ NIL IS "NOT KNOWN", never zero: an attempt with no baseline is the one `settle` refuses to
|
|
// price at all, and a zero here is a figure an operator could write a hold off against.
|
|
SpentMicroUSD *int64
|
|
// QuarantineReason is why the projection of this attempt stopped being materialized, or empty.
|
|
// Here because the lift (`run unquarantine`) is a decision, and the reason is what it is made on.
|
|
QuarantineReason string
|
|
// ParkedAt is when this attempt's projection parked: the journal continued under a foreign stream
|
|
// id and the tailer stopped there (ingest.ErrForeignStreamAhead), so freshness comes from the
|
|
// repair channel alone.
|
|
//
|
|
// Separate from QuarantineReason because the remedies differ: a quarantine is lifted by a person
|
|
// (`run unquarantine`), a park clears itself on the pass whose verdict changes (PD-438).
|
|
ParkedAt *time.Time
|
|
// Settling says which half of the stall this row is: an attempt still LIVE that the reconciler
|
|
// cannot finish, or one that has ENDED whose money never closed. They are one list because they
|
|
// are one operator question — "what is stuck and what is it holding" — and one counter
|
|
// (reconcile_failures, through the shared deferItem); they are told apart because the remedy
|
|
// differs and because the second half was invisible until this column existed (PD-385).
|
|
Settling bool
|
|
}
|
|
|
|
// StalledRuns lists what the reconciler cannot finish, at `atLeast` consecutive failures or more.
|
|
// Zero lists every live run, which is what the operator's plain `runs` asks for.
|
|
//
|
|
// It exists because the metric that showed this was a counter with no names in it: an operator could
|
|
// see that a pass did not finish and could not see WHICH run, why, or what it was holding. A number
|
|
// without a handle is half a mechanism.
|
|
//
|
|
// ⚠ TWO POPULATIONS, and until PD-385 this listed only the first of them. The reconciler has two
|
|
// phases sharing one failure counter through `deferItem`, and this query was built for LIVE runs
|
|
// alone — `a.ended_at is null` joined with `r.finished_at is null`. So a run that ENDED and whose
|
|
// money never closed was invisible here, invisible to the gauge built on the same predicate, and
|
|
// refused by `run abandon`; the ERROR logged at the threshold sent an operator to this very command
|
|
// and it answered "no run is failing to reconcile" over a frozen hold. Measured on state grown
|
|
// through the ordinary paths: five failures, an open reservation of 90000 micro, and all three
|
|
// surfaces silent.
|
|
//
|
|
// The second half is keyed exactly as the settlement worklist is (UnsettledRuns): the ATTEMPT is
|
|
// over and its reservation is still open. What it is NOT keyed on is the run being unfinished, which
|
|
// is the whole bug.
|
|
//
|
|
// ⚠ The floor for the settling half is `greatest($1, 1)` and that is not a fudge: an ordinary
|
|
// settlement closes on its first pass and never records a failure, so at `atLeast` zero — the plain
|
|
// `runs` — every settlement in flight would appear for the seconds it lives, and a table that lists
|
|
// healthy work is one an operator stops reading. One failure is the smallest number that means
|
|
// something went wrong. The live half keeps its own floor of zero, because a live run IS the answer
|
|
// to plain `runs`. The literal `>= 1` written beside it is not redundant: it is the CONSTANT the
|
|
// planner needs to prove the partial index applies, which `greatest($1, 1)` cannot give it under a
|
|
// generic plan.
|
|
func (s *Store) StalledRuns(ctx context.Context, atLeast int) ([]StalledRun, error) {
|
|
// ⚠ A UNION OF TWO ARMS and not one WHERE with an OR, and the honest reason is PREDICTABILITY of
|
|
// the plan — NOT the buffer count, which an earlier version of this comment claimed and a
|
|
// re-measurement refused. Measured on 200 000 attempts, four cells, because there are two
|
|
// variables and the first draft named one:
|
|
//
|
|
// without 00028 with 00028
|
|
// one WHERE with an OR 3647 buffers 10 buffers (BitmapOr over both partial indexes)
|
|
// UNION of two arms 3653 buffers 18 buffers (each arm on its own index)
|
|
//
|
|
// So the catastrophe — a parallel sequential scan of every attempt ever, on a command the runbook
|
|
// recommends from a cron line and on a twin gauge the daemon runs every fifteen seconds forever —
|
|
// is removed by the INDEX (00028), which both shapes need equally. What the UNION buys is that
|
|
// each arm carries its access path STRUCTURALLY: `run_attempts_live_idx` (00009) for the live
|
|
// half, `run_attempts_settling_idx` (00028) for the settling one. The OR shape's BitmapOr is the
|
|
// planner's choice, taken on statistics — and the statistics of a HEALTHY deployment are an empty
|
|
// stalled population, which is the case least like the one it was measured on. Eight buffers is
|
|
// what that costs; the top-left cell is what getting it wrong costs.
|
|
rows, err := s.pool.Query(ctx, `
|
|
with stalled as (
|
|
select r.id as run_id, b.title, a.id as attempt_id, a.spawns, a.parked_at,
|
|
coalesce(a.unit_name, '') as unit, a.attempt_no, r.status,
|
|
a.reconcile_failures, a.reconcile_after, coalesce(a.reconcile_error, '') as last_error,
|
|
coalesce(a.quarantine_reason, '') as quarantine,
|
|
a.spend_micro_usd, a.spend_baseline_micro_usd, r.started_at, false as settling
|
|
from run_attempts a
|
|
join runs r on r.id = a.run_id and r.finished_at is null
|
|
join books b on b.id = r.book_id
|
|
where a.ended_at is null and a.reconcile_failures >= $1
|
|
union all
|
|
select r.id, b.title, a.id, a.spawns, a.parked_at,
|
|
coalesce(a.unit_name, ''), a.attempt_no, r.status,
|
|
a.reconcile_failures, a.reconcile_after, coalesce(a.reconcile_error, ''),
|
|
coalesce(a.quarantine_reason, ''),
|
|
a.spend_micro_usd, a.spend_baseline_micro_usd, r.started_at, true
|
|
from run_attempts a
|
|
join runs r on r.id = a.run_id
|
|
join books b on b.id = r.book_id
|
|
where a.ended_at is not null
|
|
and a.reconcile_failures >= 1
|
|
and a.reconcile_failures >= greatest($1, 1)
|
|
and exists (select 1 from reservations res
|
|
where res.engine_run_id = a.run_id || '#' || a.attempt_no
|
|
and res.state = 'open')
|
|
)
|
|
select s.run_id, s.title, s.attempt_id, s.spawns, s.parked_at, s.unit, s.attempt_no, s.status,
|
|
s.reconcile_failures, s.reconcile_after, s.last_error, s.quarantine,
|
|
coalesce(res.amount_micro_usd, 0),
|
|
coalesce(extract(epoch from (now() - res.opened_at)), 0),
|
|
case when s.spend_baseline_micro_usd is null then null
|
|
else greatest(coalesce(s.spend_micro_usd, 0) - s.spend_baseline_micro_usd, 0) end,
|
|
s.settling
|
|
from stalled s
|
|
left join reservations res on res.engine_run_id = s.run_id || '#' || s.attempt_no
|
|
and res.state = 'open'
|
|
order by s.reconcile_failures desc, s.started_at`, atLeast)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pgstore: list stalled runs: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
var out []StalledRun
|
|
for rows.Next() {
|
|
var v StalledRun
|
|
if err := rows.Scan(&v.RunID, &v.Title, &v.AttemptID, &v.Spawns, &v.ParkedAt, &v.UnitName, &v.AttemptNo,
|
|
&v.Status, &v.Failures, &v.NextTry, &v.LastError, &v.QuarantineReason, &v.HeldMicroUSD, &v.HeldSeconds,
|
|
&v.SpentMicroUSD, &v.Settling); err != nil {
|
|
return nil, fmt.Errorf("pgstore: scan stalled run: %w", err)
|
|
}
|
|
out = append(out, v)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// ErrRunMayHaveAProcess is an abandon asked for on a run whose attempt still carries evidence that a
|
|
// process exists. Refused rather than forced: closing a run over a live engine leaves that engine
|
|
// spending against a run nothing looks at any more, which is the one outcome worse than the stall.
|
|
//
|
|
// ⚠ There are TWO pieces of such evidence and only one of them is the unit's name. `ReleaseSpawnClaim`
|
|
// clears the name when the unit could not be CREATED — and "could not be created" is not "was not
|
|
// created": a `systemd-run` killed after it had already asked leaves an engine running, which is why
|
|
// the spend baseline is deliberately kept on such an attempt (RecordSpawn). That baseline is the
|
|
// tombstone, and the reconciler's own close of this case reads it rather than the name
|
|
// (runs.finishStopped). This guard reads both, because it is the same question.
|
|
var ErrRunMayHaveAProcess = errors.New("pgstore: the run's attempt still carries evidence of a process")
|
|
|
|
// AbandonOrder is one operator's terminal verdict together with the proof it stands on.
|
|
//
|
|
// A struct and not five positional arguments because two of the five are booleans that mean opposite
|
|
// kinds of thing — one is about WHEN the money moves, the other about whether a process exists — and
|
|
// a call site where those two sit next to each other unnamed is a call site nobody can read.
|
|
type AbandonOrder struct {
|
|
RunID string
|
|
// Reason is required by the caller and written to the attempt and the ledger: this row is the
|
|
// only account anyone will ever have of why a paid-for run was declared over.
|
|
Reason string
|
|
// ReleaseHold gives the hold back inside this transaction instead of on the next sweep. It is a
|
|
// convenience for an operator working with the daemon STOPPED and never a money decision — see
|
|
// the note below. Ignored by the two branches that have no sweep to hand the money to.
|
|
ReleaseHold bool
|
|
// ProcessGone is the caller's PROOF that the stuck attempt has no engine process, and it is the
|
|
// whole of what makes the orphan branch reachable.
|
|
//
|
|
// ⚠ IT IS A PROOF AND NOT A PREFERENCE, and the caller owes the asking. What this package can see
|
|
// is only the attempt's own columns — a unit name and a spend baseline, both of which say a
|
|
// process was CREATED and neither of which says whether it is still there. Whether it is running
|
|
// is systemd's to answer, so the answer has to come from whoever can ask (cmd/tmplatformctl,
|
|
// `run abandon`, which asks and refuses on anything but a clear "gone", including on systemd
|
|
// being unreachable).
|
|
//
|
|
// ⚠ WHAT IT COSTS IF THE PROOF IS WRONG, said plainly because it is a destructive operator
|
|
// surface over money: a live engine whose run is abandoned keeps spending against its own book
|
|
// ceiling, and the account's hold is closed here — so that spend is charged to nobody and the
|
|
// DEPLOYMENT pays the provider for it. It is bounded (by the engine's `--ceiling-usd`, which is
|
|
// this run's own) and it is not exploitable by an account: reaching this state takes a run the
|
|
// deployment's own reconciler could not finish.
|
|
//
|
|
// The proof is a SNAPSHOT taken before this transaction opens, and `RecordSpawn` takes no book
|
|
// lock, so the world can move under it. The two ways below are named, re-read under the lock and
|
|
// refused.
|
|
//
|
|
// ⚠ A THIRD IS NOT CAUGHT, and it is named rather than left to be discovered. `RecordSpawn`
|
|
// commits the claim — and the counter with it — BEFORE the unit is created (runs.spawnAttempt,
|
|
// where `Runner.Start` follows the record). A proof taken inside that window reads the counter
|
|
// already bumped and hears «gone» from a systemd the unit has not reached yet, so both
|
|
// comparisons agree and the write-off lands on an attempt whose unit is about to rise. Catching
|
|
// it needs the store to know that a spawn is IN FLIGHT, which is a state nothing records today —
|
|
// register row PD-465. The window is the milliseconds between two statements of one function, and
|
|
// it costs what every wrong proof costs (see the paragraph on ProcessGone below).
|
|
ProcessGone bool
|
|
// ProofAttemptID is which attempt the caller asked systemd about. The write below reads "the live
|
|
// attempt", so a restart in the window would otherwise spend a proof about attempt N on attempt
|
|
// N+1 — a fresh unit with an engine mid-call, and its hold closed underneath it.
|
|
ProofAttemptID int64
|
|
// ProofSpawns is `run_attempts.spawns` as the caller read it. `unit_name` cannot serve here:
|
|
// ReleaseSpawnClaim returns it to NULL, so a claim-then-release inside the window leaves the name
|
|
// exactly as the proof saw it while an engine may be running. The counter never goes back.
|
|
ProofSpawns int
|
|
Now time.Time
|
|
}
|
|
|
|
// ErrProofOvertaken is an abandon whose proof stopped being about the run in front of it. Refused
|
|
// rather than resolved here: whether a process exists is systemd's answer and the caller's to ask.
|
|
var ErrProofOvertaken = errors.New("pgstore: the attempt moved while its absence was being proved")
|
|
|
|
// AbandonRun is the OPERATOR's terminal verdict on a run the reconciler cannot finish.
|
|
//
|
|
// It is not a policy this platform applies by itself, and that boundary is deliberate. The
|
|
// reconciler's rule everywhere else is that "I could not ask" is never "the run is gone"; a run that
|
|
// has failed N times is exactly a run nobody could ask about, so deciding on its behalf would be
|
|
// that same mistake with a counter in front of it. What the counter buys is that an operator is TOLD
|
|
// (StalledRuns, the stalled gauge); what this buys is that being told is worth something.
|
|
//
|
|
// ⚠ THE HOLD COMES BACK WHOLE EITHER WAY, and the flag only decides WHEN. The guard below admits
|
|
// only an attempt that never reached the engine, so nothing was spent, so the ordinary settlement
|
|
// releases it whole on its next pass — `--release-hold` does it inside this transaction instead,
|
|
// which is what an operator working with the daemon stopped actually needs. What the flag never was
|
|
// is a money decision: the doc used to promise the hold "stays open while there is any chance the
|
|
// engine will answer", which the guard has already made impossible. Neither branch is the escrow
|
|
// design (unified backlog row 136, zone backlog П-18).
|
|
func (s *Store) AbandonRun(ctx context.Context, o AbandonOrder) (AbandonVerdict, error) {
|
|
runID, reason, giveTheHoldBack, now := o.RunID, o.Reason, o.ReleaseHold, o.Now
|
|
var verdict AbandonVerdict
|
|
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
|
// ⚠ THE BOOK'S ROW FIRST, before the run and its attempt. That is the package's global order
|
|
// (lockBook) and it is not a preference: this used to take the run and the attempt together
|
|
// through `for update of r, a` and the book afterwards, which is the inversion `RestartRun`
|
|
// names as the pair that deadlocked — measured here at 7 aborted transactions in 60 concurrent
|
|
// pairs against `FinishRun` and 3 against `PauseRun`, both of which are on the settlement path.
|
|
// Postgres breaks the cycle by aborting one side, so the price is a failed sweep or an operator
|
|
// command that answers with raw driver text.
|
|
//
|
|
// The book id is read WITHOUT a lock to find out which book to lock; everything the write
|
|
// depends on is then re-read under it.
|
|
var bookID string
|
|
switch err := tx.QueryRow(ctx,
|
|
`select book_id from runs where id = $1`, runID).Scan(&bookID); {
|
|
case errors.Is(err, pgx.ErrNoRows):
|
|
return ErrNoRun
|
|
case err != nil:
|
|
return fmt.Errorf("pgstore: read the run to abandon: %w", err)
|
|
}
|
|
if err := lockBook(ctx, tx, bookID); err != nil {
|
|
return err
|
|
}
|
|
// ⚠ WHICH OF THE TWO VERDICTS THIS IS, read UNDER THE BOOK LOCK and not before it. The
|
|
// unlocked read above is only for finding out which book to lock, and reading the branch from
|
|
// it was a real race with a money answer: `RestartRun` takes this same book lock, clears
|
|
// `finished_at` and opens a NEW attempt with a NEW hold, so an abandon that queued behind a
|
|
// resume would come through holding a snapshot that says "finished" and act on a run that is
|
|
// live again. Everything the write depends on is re-read here, which is what the paragraph
|
|
// above already promised and this line now keeps.
|
|
var finished *time.Time
|
|
if err := tx.QueryRow(ctx,
|
|
`select finished_at from runs where id = $1`, runID).Scan(&finished); err != nil {
|
|
return fmt.Errorf("pgstore: read the run to abandon: %w", err)
|
|
}
|
|
if finished != nil {
|
|
verdict = AbandonedSettlement
|
|
return abandonSettlement(ctx, tx, runID, reason, true, now)
|
|
}
|
|
// ⚠ THE RUN IS LIVE AND THAT DOES NOT MEAN ITS MONEY IS. Before anything is decided about the
|
|
// live attempt, the run is asked whether it carries an ORPHANED one — an attempt that ended
|
|
// with its reservation still open. That population is `PD-418`, and the reason it had no
|
|
// handle is exactly this branch: everything terminal used to ask `runs.finished_at`, which
|
|
// says nothing about whether some earlier attempt's hold is stuck. Branching on the PRESENCE
|
|
// of the orphan instead is the durable cure both rows name, and it is one query.
|
|
switch err := abandonSettlement(ctx, tx, runID, reason, false, now); {
|
|
case err == nil:
|
|
verdict = AbandonedSettlement
|
|
return nil
|
|
case !errors.Is(err, errNoOrphanedAttempt):
|
|
return err
|
|
}
|
|
verdict = AbandonedRun
|
|
var unit string
|
|
var attemptID int64
|
|
var attemptNo int
|
|
var baseline *int64
|
|
var reported int64
|
|
var failures int
|
|
var spawns int
|
|
err := tx.QueryRow(ctx, `
|
|
select a.id, a.attempt_no, coalesce(a.unit_name, ''), a.spend_baseline_micro_usd,
|
|
a.spend_micro_usd, a.reconcile_failures, a.spawns
|
|
from runs r join run_attempts a on a.run_id = r.id and a.ended_at is null
|
|
where r.id = $1 and r.finished_at is null
|
|
for update of r, a`, runID).
|
|
Scan(&attemptID, &attemptNo, &unit, &baseline, &reported, &failures, &spawns)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
// It FINISHED while this transaction was taking the book — the unlocked read above said
|
|
// live and the locked one does not. That is not "there is no such run", and answering it
|
|
// as one is the very confusion PD-385 is about: the operator is looking at the run. It is
|
|
// the settlement case, one pass late, and it is handled as the settlement case.
|
|
verdict = AbandonedSettlement
|
|
return abandonSettlement(ctx, tx, runID, reason, true, now)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("pgstore: read the attempt to abandon: %w", err)
|
|
}
|
|
var writeOff *money.MicroUSD
|
|
if unit != "" || baseline != nil {
|
|
// Whether a process is still RUNNING is systemd's to answer, and this refuses the case where
|
|
// nobody asked. The unit's name is derivable from the run and the attempt even when the
|
|
// column is empty, so the message names it: that is where the operator has to go.
|
|
if !o.ProcessGone {
|
|
return fmt.Errorf("%w: tm-run-%s-%d", ErrRunMayHaveAProcess, runID, attemptNo)
|
|
}
|
|
// Both halves are re-read above under the book lock. A second look from the CLI would be
|
|
// one more snapshot with one more window after it (PD-424).
|
|
switch {
|
|
case o.ProofAttemptID != attemptID:
|
|
return fmt.Errorf("%w: the proof was about attempt %d and this run's live attempt is now #%d",
|
|
ErrProofOvertaken, o.ProofAttemptID, attemptNo)
|
|
case o.ProofSpawns != spawns:
|
|
return fmt.Errorf("%w: attempt #%d was claimed for a spawn (%d claims at the proof, %d now)",
|
|
ErrProofOvertaken, attemptNo, o.ProofSpawns, spawns)
|
|
}
|
|
// The caller brought the proof, and the floor is the same one the settlement branch waits
|
|
// for: SEEING a run go wrong and being allowed to end it are different permissions. Below
|
|
// the threshold the reconciler is still retrying, and a run it would have closed correctly
|
|
// on its own must not be written off a tick early.
|
|
if failures < abandonAfter {
|
|
return fmt.Errorf("%w: %d of %d", ErrRunNotStalledYet, failures, abandonAfter)
|
|
}
|
|
verdict = AbandonedOrphan
|
|
// ⚠ WHAT IT COSTS, and the two cases are not the same money. `spend_micro_usd` is the BOOK's
|
|
// lifetime total as the stream last reported it, so the attempt's own work is the DIFFERENCE
|
|
// against its baseline — the same arithmetic `StalledRuns` prints and `attemptSpend`
|
|
// settles by, clamp included. A baseline plus that figure is the last thing the engine SAID
|
|
// about this attempt, and charging it is
|
|
// strictly more honest than handing the whole hold back — the run demonstrably did that
|
|
// much work. Without a baseline there is no figure this platform could justify, and the
|
|
// hold goes back whole, which is the settlement branch's own reasoning arriving here.
|
|
if baseline != nil {
|
|
spent := money.MicroUSD(max(reported-*baseline, 0))
|
|
writeOff = &spent
|
|
}
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
update runs set status = 'failed', failure_reason = 'service_error', finished_at = $2,
|
|
paused_reason = null,
|
|
revision = (select revision + 1 from books where id = runs.book_id)
|
|
where id = $1`, runID, now); err != nil {
|
|
return fmt.Errorf("pgstore: abandon run: %w", err)
|
|
}
|
|
// ⚠ THE DEFERRAL GOES WITH THE VERDICT, and leaving it was PD-391. `UnsettledRuns` filters on
|
|
// `reconcile_after`, and a run that reached an operator is by construction one that has been
|
|
// deferred — `deferItem` sets now + backoff, and backoff at five failures is pinned at its
|
|
// thirty-minute cap. So for the WHOLE population this command exists for, the CLI's "its hold
|
|
// comes back whole on the next sweep" and the runbook's copy of it were false: the credit
|
|
// stayed debited, `tm_platform_oldest_open_hold_seconds` kept rising AFTER the operator acted,
|
|
// and what they read was "I did it and nothing happened". Measured: forty-five seconds and
|
|
// three sweeps with the hold open; clearing the column closes it on the next one.
|
|
if _, err := tx.Exec(ctx,
|
|
`update run_attempts set ended_at = $2, reconcile_error = $3, reconcile_after = null
|
|
where id = $1`,
|
|
attemptID, now, truncateReason("abandoned by an operator: "+reason)); err != nil {
|
|
return fmt.Errorf("pgstore: end the abandoned attempt: %w", err)
|
|
}
|
|
// The book owes a reading surface like it does after every other ending: whatever the run did
|
|
// translate before it stalled is bought and paid for, and this is the one write that puts it
|
|
// back in front of a reader.
|
|
if _, err := tx.Exec(ctx, `
|
|
update books set status = 'failed', `+owesAReadingSurface+`
|
|
revision = `+nextRevisionOfThisBooksLibrary+` where id = $1`, bookID); err != nil {
|
|
return fmt.Errorf("pgstore: mark the abandoned run's book: %w", err)
|
|
}
|
|
if err := emitStatus(ctx, tx, bookID); err != nil {
|
|
return err
|
|
}
|
|
if !giveTheHoldBack && verdict != AbandonedOrphan {
|
|
return nil // the settlement phase closes it on its next pass
|
|
}
|
|
// ⚠ THE ORPHAN BRANCH CLOSES ITS MONEY INLINE WHATEVER THE FLAG SAYS, for the same reason the
|
|
// settlement branch does: the attempt this ends is one no sweep will ever settle — that is
|
|
// what made it an orphan — so leaving the hold for a pass that is not coming is the defect
|
|
// this handle exists to end, not a policy.
|
|
key := ReservationKey(runID, attemptNo)
|
|
state := "released"
|
|
if writeOff != nil {
|
|
state = "settled"
|
|
}
|
|
userID, held, err := closeReservation(ctx, tx, key, state, now)
|
|
if errors.Is(err, ErrNoReservation) {
|
|
return nil // already settled or released; the abandon still stands
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := releaseHold(ctx, tx, userID, key, held, now); err != nil {
|
|
return err
|
|
}
|
|
if writeOff != nil {
|
|
// The last figure the engine gave for THIS attempt, charged rather than forgiven. Capped at
|
|
// the hold by the same rule the ordinary settlement uses: the hold is what the account
|
|
// agreed to, and a meter above it is the engine's business to explain, never this
|
|
// account's to pay.
|
|
spent := *writeOff
|
|
// The basis is HALTED without asking anything: an abandon is by definition an attempt cut
|
|
// off mid-work, and its figure is the last one the engine managed to report. See
|
|
// SettlementBasis for why every settlement says which floor it is (PD-441).
|
|
note := BasisHalted.note() + "; abandoned by an operator over an orphaned attempt"
|
|
if spent > held {
|
|
note += "; capped at the hold"
|
|
spent = held
|
|
}
|
|
applied, err := appendLedger(ctx, tx, userID, "settlement", -spent, "run_settle", key, note, now)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !applied {
|
|
// Reported rather than applied, exactly as the ordinary settlement reports it: a spent
|
|
// key here would return the hold and charge nothing.
|
|
return fmt.Errorf("%w: %s", ErrSettlementKeySpent, key)
|
|
}
|
|
}
|
|
// The run's money is RESOLVED, and saying so is this branch's own job: it takes the run out of
|
|
// the settlement list before that list ever sees it, so nothing else would ever stamp the
|
|
// column and the run stayed "unsettled" for good.
|
|
_, err = tx.Exec(ctx, `update runs set settled_at = $2 where id = $1 and settled_at is null`, runID, now)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return verdict, nil
|
|
}
|
|
|
|
// AbandonVerdict is which of the two terminal verdicts AbandonRun applied. Returned rather than left
|
|
// for the caller to re-derive: the two mean different things to whoever typed the command — one
|
|
// ended a run, the other gave up on money — and a command that printed the same sentence for both
|
|
// would be the sentence that hid the second one (PD-385).
|
|
type AbandonVerdict string
|
|
|
|
const (
|
|
// AbandonedRun is the live case: a run the reconciler could not finish is declared over.
|
|
AbandonedRun AbandonVerdict = "run"
|
|
// AbandonedSettlement is the other half: the run had already ended and its MONEY was stuck.
|
|
//
|
|
// ⚠ Since the orphan handle it is ALSO the verdict of a LIVE run that carries an ended attempt
|
|
// whose hold never closed — the population of `PD-418`. The run itself is untouched there: what
|
|
// was stuck was one attempt's money, and ending a run over it would take a translation away from
|
|
// a user who did not ask.
|
|
AbandonedSettlement AbandonVerdict = "settlement"
|
|
// AbandonedOrphan is the third and the newest: a run that is LIVE, whose attempt still names an
|
|
// engine process, and whose process the caller has PROVEN is gone. It is the half of `PD-424`
|
|
// that stood open through two packs — the run was visible, counted and un-actionable, and the
|
|
// only exit was the user pressing Stop without being told so.
|
|
AbandonedOrphan AbandonVerdict = "orphan"
|
|
)
|
|
|
|
// ErrRunNotStalledYet is an orphan abandon asked for on a live run the reconciler has not given up
|
|
// on. The floor is the same `abandonAfter` the settlement branch waits for, and for the same reason
|
|
// written out there: seeing a run go wrong and being allowed to end it are different permissions.
|
|
var ErrRunNotStalledYet = errors.New("pgstore: the run has not failed to reconcile enough times to be written off")
|
|
|
|
// errNoOrphanedAttempt says a run carries no ended attempt with an open reservation. Unexported: it
|
|
// is a fall-through between the branches of one function and never an answer to a caller — for a
|
|
// FINISHED run the absence is `ErrMoneyAlreadyClosed` or `ErrSettlementNotStuck`, and for a live one
|
|
// it simply means the live attempt is the thing to look at.
|
|
var errNoOrphanedAttempt = errors.New("pgstore: the run carries no orphaned attempt")
|
|
|
|
// ErrMoneyAlreadyClosed is an abandon asked for on a run that has finished and whose money is
|
|
// already resolved. Told apart from ErrNoRun deliberately: "there is nothing to do" and "there is no
|
|
// such run" send an operator to different places, and answering the first with the second is what
|
|
// made the settlement stall unactionable (PD-385).
|
|
var ErrMoneyAlreadyClosed = errors.New("pgstore: the run has finished and its money is already closed")
|
|
|
|
// ErrSettlementNotStuck is an abandon asked for on a finished run whose money is open but whose
|
|
// settlement has not failed even once — that is, one the next sweep is about to close correctly.
|
|
// Refused rather than obeyed: the hold this command returns is returned WHOLE, so obeying would be
|
|
// giving away whatever the run actually spent, and the operator has no way to know that from the
|
|
// outside. It is also the state the operator's own table deliberately does not show.
|
|
var ErrSettlementNotStuck = errors.New("pgstore: the run's settlement has not failed; it is still being retried")
|
|
|
|
// abandonSettlement is the operator's terminal verdict on the OTHER half of the stall: a run that
|
|
// ended and whose settlement will never complete.
|
|
//
|
|
// It exists because there was no handle at all for this population, by construction: every terminal
|
|
// path read `finished_at is null` and answered ErrNoRun, so a hold behind a settlement that could
|
|
// not be computed — the engine binary gone at a rollout, an attempt from before the baseline column —
|
|
// was frozen until somebody wrote SQL in production. (A project REPLACED under the platform is
|
|
// deliberately NOT in that list, though the first draft of this paragraph had it: the reconciler
|
|
// catches that one by the meter reading below the attempt's own baseline and settles it at nothing,
|
|
// so it never reaches this handle.)
|
|
//
|
|
// ⚠ ONLY A SETTLEMENT THAT IS STALLED, and the floor here is the THRESHOLD and not the one failure
|
|
// the operator's table shows from. The two numbers are deliberately different, and the difference is
|
|
// see-early / act-late:
|
|
//
|
|
// - `StalledRuns` lists the settling half from ONE failure, because an operator wants to see a
|
|
// settlement going wrong while it is only going wrong;
|
|
// - this command REFUNDS THE WHOLE HOLD, so it waits for `abandonAfter` — the same
|
|
// `runs.StalledAfter` the threshold and the gauge use.
|
|
//
|
|
// The first draft of this branch admitted from one failure, matching the list, and acceptance
|
|
// measured what that costs: an engine unavailable for a single tick puts a row in the plain table
|
|
// whose CLI comment points at this command, and running it then writes off the run's real spend —
|
|
// measured at $1.234567 spent, ordinary sweep charging $0.300000 and this command a tick earlier
|
|
// charging $0.000000. Seeing a thing and being allowed to destroy it are different permissions.
|
|
//
|
|
// ⚠ THE HOLD COMES BACK WHOLE, and that is a decision rather than an omission. What is missing is
|
|
// the engine's committed figure — that is the definition of this state — so there is no number this
|
|
// platform could justify charging, and the two alternatives are worse in both directions: charging
|
|
// the ceiling bills for work nobody can show, and leaving it open is today's defect. It is not
|
|
// exploitable by the account either: reaching the admitted set takes a settlement the deployment's
|
|
// own engine could not answer, never something a user can ask for.
|
|
//
|
|
// ⚠ It does NOT touch the run's status. The run already ended with an outcome of its own, and
|
|
// overwriting it with `failed` would erase what actually happened to a run that may well have
|
|
// succeeded. Nor does it bump a revision or emit a frame: the ORDINARY settlement does neither
|
|
// (MarkSettled), and this is that settlement reaching its end by another road.
|
|
//
|
|
// The evidence-of-a-process guard the live branch applies is satisfied here by the ending itself and
|
|
// not skipped: `ended_at` is written only where the platform established the process is over —
|
|
// `finish` off a marker, or `finishStopped`, which asks systemd before closing an attempt that
|
|
// carries a spend baseline (runs.finishStopped).
|
|
// abandonAfter is how many consecutive settlement failures make a run's money an operator's to write
|
|
// off. It is `runs.StalledAfter` written here rather than imported: `internal/runs` depends on this
|
|
// package, so the constant cannot travel the other way, and a second literal is worse than a named
|
|
// one. The two are pinned equal by TestTheAbandonFloorIsTheStalledThreshold.
|
|
const abandonAfter = 5
|
|
|
|
func abandonSettlement(ctx context.Context, tx pgx.Tx, runID, reason string, runFinished bool, now time.Time) error {
|
|
// ⚠ EVERY orphaned attempt of this run, not the oldest one. A run holds at most one open
|
|
// reservation on the ordinary path — `reopen` refuses to start attempt N+1 while attempt N's is
|
|
// open — but a run needing this command is by definition one whose ordinary path came apart, and
|
|
// the state this package already worries about out loud is exactly "an attempt that was
|
|
// interrupted and replaced leaves its reservation open while its run goes on" (UnsettledRuns).
|
|
// Taking one row and then stamping the RUN settled would leave the second hold debited under a
|
|
// message that says the money came back whole.
|
|
// ⚠ NO `r.finished_at is not null`, and its removal is the durable cure of `PD-418`. An orphaned
|
|
// attempt is orphaned by the PRESENCE of its own open reservation over its own `ended_at`, and
|
|
// whether the RUN has since finished says nothing about that — a live run whose previous attempt
|
|
// never settled is precisely the population the runbook promised this command served and the code
|
|
// answered about the process instead. The caller says which shape it is in `runFinished`, and
|
|
// that decides only what an EMPTY result means.
|
|
rows, err := tx.Query(ctx, `
|
|
select a.id, a.attempt_no
|
|
from runs r
|
|
join run_attempts a on a.run_id = r.id and a.ended_at is not null
|
|
and a.reconcile_failures >= $2
|
|
join reservations res on res.engine_run_id = r.id || '#' || a.attempt_no
|
|
and res.state = 'open'
|
|
where r.id = $1
|
|
order by a.attempt_no
|
|
for update of r, a`, runID, abandonAfter)
|
|
if err != nil {
|
|
return fmt.Errorf("pgstore: read the settlement to abandon: %w", err)
|
|
}
|
|
type orphan struct {
|
|
id int64
|
|
attemptNo int
|
|
}
|
|
var orphans []orphan
|
|
for rows.Next() {
|
|
var o orphan
|
|
if err := rows.Scan(&o.id, &o.attemptNo); err != nil {
|
|
rows.Close()
|
|
return fmt.Errorf("pgstore: scan the settlement to abandon: %w", err)
|
|
}
|
|
orphans = append(orphans, o)
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return fmt.Errorf("pgstore: read the settlement to abandon: %w", err)
|
|
}
|
|
if len(orphans) == 0 {
|
|
if !runFinished {
|
|
// A LIVE run with no orphan is not a state to answer about: its own attempt is what the
|
|
// caller came for, and this is a fall-through rather than a verdict.
|
|
return errNoOrphanedAttempt
|
|
}
|
|
// Two states, told apart, because the operator's next move differs: money already resolved
|
|
// (nothing to do) versus money still open on a settlement that has not failed yet (wait — the
|
|
// next sweep closes it, and giving it back now would be a gift of whatever the run spent).
|
|
var pending bool
|
|
if err := tx.QueryRow(ctx, `
|
|
select exists (
|
|
select 1 from run_attempts a
|
|
join reservations res on res.engine_run_id = a.run_id || '#' || a.attempt_no
|
|
and res.state = 'open'
|
|
where a.run_id = $1 and a.ended_at is not null)`, runID).Scan(&pending); err != nil {
|
|
return fmt.Errorf("pgstore: read the settlement to abandon: %w", err)
|
|
}
|
|
if pending {
|
|
return ErrSettlementNotStuck
|
|
}
|
|
return ErrMoneyAlreadyClosed
|
|
}
|
|
for _, o := range orphans {
|
|
key := ReservationKey(runID, o.attemptNo)
|
|
userID, held, err := closeReservation(ctx, tx, key, "released", now)
|
|
switch {
|
|
case errors.Is(err, ErrNoReservation):
|
|
// The settlement sweep closed it between this transaction's read and this write. Tolerated
|
|
// rather than reported, exactly as the live branch tolerates it: the money is resolved,
|
|
// which is what the operator asked for, and a raw "no open reservation" would read as a
|
|
// failure of the command.
|
|
continue
|
|
case err != nil:
|
|
return err
|
|
}
|
|
if err := releaseHold(ctx, tx, userID, key, held, now); err != nil {
|
|
return err
|
|
}
|
|
// The reason on the record, and the deferral cleared for the same reason the live branch
|
|
// clears it (PD-391): a row that stays deferred is one the sweep keeps stepping over.
|
|
if _, err := tx.Exec(ctx, `
|
|
update run_attempts set reconcile_error = $2, reconcile_after = null where id = $1`,
|
|
o.id, truncateReason("settlement abandoned by an operator: "+reason)); err != nil {
|
|
return fmt.Errorf("pgstore: record the abandoned settlement: %w", err)
|
|
}
|
|
}
|
|
if !runFinished {
|
|
// ⚠ A LIVE run is NOT stamped settled: `settled_at` is a statement about the WHOLE run's
|
|
// money, and the live attempt's hold is still open and still the reconciler's to close.
|
|
//
|
|
// ⚠ AND THE COLUMN HAS NO READER TODAY, which an adversarial pass established and an earlier
|
|
// edition of this comment did not know: `UnsettledRuns` keys on an OPEN RESERVATION
|
|
// (`sink.go`), not on this column, and the ordinary sweep already stamps it on live runs
|
|
// through `MarkSettled`. So what this branch protects is the column's MEANING and not a
|
|
// worklist — worth keeping (a field that lies is a field the next reader believes) and not
|
|
// worth claiming more for.
|
|
return nil
|
|
}
|
|
// Said here and nowhere else: this takes the run out of the settlement worklist before that list
|
|
// sees it again, so nothing else would ever stamp the column and the run would read "unsettled"
|
|
// for good — the same trap the `--release-hold` branch above names. Stamped only once every
|
|
// orphan is closed, so the column cannot claim a resolution one of them contradicts.
|
|
if _, err := tx.Exec(ctx,
|
|
`update runs set settled_at = $2 where id = $1 and settled_at is null`, runID, now); err != nil {
|
|
return fmt.Errorf("pgstore: mark the abandoned settlement settled: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// runColumns is the reconciler's view of a run. Written once because the two queries that use it
|
|
// differ only in which runs they select, and a scan list copied twice is a scan list that drifts.
|
|
// SpawnOrder is the book's order RESOLVED against the tree as it stands, for one spawn.
|
|
//
|
|
// ⚠ THREE STATES AND THEY ARE NOT INTERCHANGEABLE. A whole-book order has no boundary and needs none
|
|
// (`Order.Whole()`); a partial order that RESOLVES gives a unit count; a partial order that does NOT
|
|
// resolve is a book cut again under a purchase, and the honest answer there is neither a count nor
|
|
// «the whole book» — it is a refusal to spawn, because either number would be a volume nobody
|
|
// bought.
|
|
//
|
|
// Resolved at every spawn rather than frozen at admission, for the same reason the boundary is
|
|
// stored as an identity: what «through chapter 12» is worth in units is a property of the CURRENT
|
|
// cut, and a restart of a run that already delivered half its order must not hand the engine the
|
|
// whole of it again.
|
|
type SpawnOrder struct {
|
|
Order BookOrder
|
|
// Units is the order in output units; Resolved says whether its boundary could be found at all.
|
|
Units int
|
|
Resolved bool
|
|
// Delivered is how many of the units the order runs THROUGH are already handed over — bounded by
|
|
// the same boundary the order is. Counted book-wide, work delivered BEYOND the boundary would
|
|
// cancel work still owed inside it (see deliveredWithinOrder).
|
|
Delivered int
|
|
// UnitsLeft is what is undelivered book-WIDE: the allowance a WHOLE-BOOK order is given on a
|
|
// continuation. Giving one at all is not belt-and-braces — with no ceiling in force the engine
|
|
// builds no volume scope, and the scope is what carries its protective order of work (new book
|
|
// before re-made book, backend/internal/pipeline/volume.go). Without it a continuation carrying
|
|
// `--resnapshot` re-pays the beginning of the book before editing what was just bought.
|
|
UnitsLeft int
|
|
// HasPriorRun is whether this book has EVER had another run — the condition for `--resnapshot`
|
|
// beside the correction door's own flag (PD-422): the auto-bank grows by MINING during an
|
|
// ordinary run, mining moves the ENRICHED memory version, and that version is folded into the
|
|
// EDIT wave's snapshot alone (backend/internal/pipeline/snapshot.go, snapshotIDForWave). So the
|
|
// second purchase of a mining book meets its own already-pinned edit jobs under a moved snapshot
|
|
// and the drift guard stops the engine — after the hold was taken, with nothing translated.
|
|
HasPriorRun bool
|
|
}
|
|
|
|
// ReadOrderForSpawn resolves a run's order against the tree as it stands now.
|
|
//
|
|
// ⛔ ITS OWN QUERY, ON THE SPAWN PATH, and that placement is the whole of it. Resolving an order
|
|
// means walking the book's units three ways, and these figures were once columns of `runColumns` —
|
|
// the reconciler's list of live runs, read for every live run on every sweep, for numbers only a
|
|
// spawn reads. A spawn happens once per attempt; a sweep happens every few seconds.
|
|
func (s *Store) ReadOrderForSpawn(ctx context.Context, runID string) (SpawnOrder, error) {
|
|
var out SpawnOrder
|
|
err := s.pool.QueryRow(ctx, `
|
|
select coalesce(b.ordered_through_chapter_id, ''), coalesce(b.ordered_through_unit_id, ''),
|
|
coalesce(b.ordered_through_chapter_number, 0),
|
|
`+orderedUnits+`, `+orderResolved+`,
|
|
`+deliveredWithinOrder+`, `+bookUnitsLeft+`,
|
|
exists (select 1 from runs pr where pr.book_id = b.id and pr.id <> r.id)
|
|
from runs r join books b on b.id = r.book_id where r.id = $1`, runID).
|
|
Scan(&out.Order.ThroughChapterID, &out.Order.ThroughUnitID, &out.Order.ThroughChapterNumber,
|
|
&out.Units, &out.Resolved, &out.Delivered, &out.UnitsLeft, &out.HasPriorRun)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return SpawnOrder{}, ErrNoRun
|
|
}
|
|
if err != nil {
|
|
return SpawnOrder{}, fmt.Errorf("pgstore: read the run's order: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
const runColumns = `
|
|
select r.id, r.book_id, b.owner_id, b.workdir, r.verify_bank, r.resnapshot,
|
|
r.accept_rebill_micro, r.ceiling_chapters,
|
|
coalesce(r.paused_reason, ''), r.started_at, r.status, r.stop_requested_at,
|
|
a.id, a.attempt_no, coalesce(a.unit_name, ''), coalesce(a.engine_run_id, ''),
|
|
a.last_offset, a.last_seq, a.last_line_sha256, a.quarantine_reason is not null,
|
|
a.parked_at is not null,
|
|
coalesce(res.amount_micro_usd, 0), a.engine_binary, a.spend_baseline_micro_usd,
|
|
a.ceiling_arg_micro_usd, a.started_at, a.reconcile_failures
|
|
from runs r
|
|
join books b on b.id = r.book_id`
|
|
|
|
func (s *Store) queryRuns(ctx context.Context, tail string, args ...any) ([]LiveRun, error) {
|
|
rows, err := s.pool.Query(ctx, runColumns+tail, args...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pgstore: list runs: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
var out []LiveRun
|
|
for rows.Next() {
|
|
var l LiveRun
|
|
var ceiling, ceilingArg, acceptRebill int64
|
|
var baseline *int64
|
|
if err := rows.Scan(&l.RunID, &l.BookID, &l.UserID, &l.Workdir, &l.VerifyBank, &l.Resnapshot,
|
|
&acceptRebill, &l.OrderedChapters,
|
|
&l.PausedReason, &l.StartedAt, &l.Status, &l.StopRequestedAt,
|
|
&l.AttemptID, &l.AttemptNo, &l.UnitName, &l.EngineRunID,
|
|
&l.Position.Offset, &l.Position.LastSeq, &l.Position.LastHash, &l.Quarantined, &l.Parked, &ceiling,
|
|
&l.EngineBinary, &baseline, &ceilingArg, &l.AttemptStartedAt, &l.ReconcileFailures); err != nil {
|
|
return nil, fmt.Errorf("pgstore: scan run: %w", err)
|
|
}
|
|
l.Ceiling = money.MicroUSD(ceiling)
|
|
l.CeilingArg = money.MicroUSD(ceilingArg)
|
|
l.AcceptRebill = money.MicroUSD(acceptRebill)
|
|
if baseline != nil {
|
|
v := money.MicroUSD(*baseline)
|
|
l.SpendBaseline = &v
|
|
}
|
|
out = append(out, l)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// RunSpent is what a run has actually been charged so far, across all its attempts. Settlements are
|
|
// ledger rows keyed by attempt, so the sum is over the run's own key space and nothing else's.
|
|
func (s *Store) RunSpent(ctx context.Context, runID string) (money.MicroUSD, error) {
|
|
var v int64
|
|
err := s.pool.QueryRow(ctx, `
|
|
select coalesce(-sum(amount_micro_usd), 0) from credit_ledger
|
|
where source = 'run_settle' and split_part(source_id, '#', 1) = $1`, runID).Scan(&v)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("pgstore: read run spend: %w", err)
|
|
}
|
|
return money.MicroUSD(v), nil
|
|
}
|
|
|
|
// ErrNoFirstHold is a run whose first attempt's reservation cannot be found. Every admission opens
|
|
// that row in the same transaction as the run (StartRun → holdTx), and closing a reservation keeps
|
|
// the row and changes its state — so this is a broken invariant rather than a state, and the caller
|
|
// is told by name instead of being handed a figure the rate would give.
|
|
var ErrNoFirstHold = errors.New("pgstore: the run's first hold is missing, so what it was sold for cannot be read")
|
|
|
|
// RunBudget is what the run was SOLD for: the hold its FIRST attempt took, read back from the
|
|
// reservation rather than quoted again.
|
|
//
|
|
// The two are one number on the day of the admission and different afterwards: an order's quote is
|
|
// made of the engine's projection (re-derived on every re-cut of the book) and of a cushion the
|
|
// deployment sets, and both move, while the hold is the figure the buyer agreed to. Quoting it again
|
|
// would re-price a paid run in both directions — up, and the continuation holds more than the buyer
|
|
// was ever shown; down, and the remainder goes negative and a run with chapters left pauses as
|
|
// exhausted (PD-168: with the setting doubled, $5.50 held for a $2.50 remainder).
|
|
//
|
|
// `amount_micro_usd` and not `ceiling_micro_usd`: the two carry the same figure at open
|
|
// (OpenReservation writes one argument into both), but the amount is what the ledger debited and what
|
|
// settlement gives back, i.e. the money fact, while the ceiling column's own comment describes a
|
|
// meaning the engine's flag does not have (D39.122, PD-377's class).
|
|
func (s *Store) RunBudget(ctx context.Context, runID string) (money.MicroUSD, error) {
|
|
var v int64
|
|
err := s.pool.QueryRow(ctx,
|
|
`select amount_micro_usd from reservations where engine_run_id = $1`, engineRunKey(runID, 1)).Scan(&v)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return 0, fmt.Errorf("%w: run %s", ErrNoFirstHold, runID)
|
|
}
|
|
if err != nil {
|
|
return 0, fmt.Errorf("pgstore: read run budget: %w", err)
|
|
}
|
|
return money.MicroUSD(v), nil
|
|
}
|
|
|
|
// SpendBound is the UPPER bound on what an attempt can have cost: the smallest meter reading any
|
|
// LATER attempt of the same book recorded before it started.
|
|
//
|
|
// It exists because settlement reads a lifetime counter of the BOOK at the moment it retries, and
|
|
// that retry can happen after another run of the same book has already moved it. A settlement is
|
|
// allowed to defer — the engine has to be asked and asking can fail — and a deferred one is not
|
|
// blocked from being overtaken: the earlier run is finished, so nothing stops the account starting
|
|
// another. Measured: a run that cost $0.10 was charged $2.10, the difference being what its
|
|
// successor had spent by then, and the successor then paid that same amount again.
|
|
//
|
|
// A later attempt's baseline is exactly the right bound, because it was read BEFORE that attempt
|
|
// added anything and AFTER this one had stopped. Nil means no later attempt has been spawned, and
|
|
// then the counter has not been moved by anyone else.
|
|
func (s *Store) SpendBound(ctx context.Context, bookID string, attemptID int64) (*money.MicroUSD, error) {
|
|
var v *int64
|
|
err := s.pool.QueryRow(ctx, `
|
|
select min(a.spend_baseline_micro_usd)
|
|
from run_attempts a join runs r on r.id = a.run_id
|
|
where r.book_id = $1 and a.id > $2 and a.spend_baseline_micro_usd is not null`,
|
|
bookID, attemptID).Scan(&v)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pgstore: read spend bound: %w", err)
|
|
}
|
|
if v == nil {
|
|
return nil, nil
|
|
}
|
|
bound := money.MicroUSD(*v)
|
|
return &bound, nil
|
|
}
|
|
|
|
// ErrRunNotLive is a run that cannot be stopped because it is already over. Distinct from ErrNoRun,
|
|
// which is a run this account cannot see at all: the first is the contract's 409 and the second its
|
|
// 404, and answering the wrong one either tells a stranger that a run exists or tells an owner that
|
|
// theirs does not.
|
|
var ErrRunNotLive = errors.New("pgstore: the run is not live")
|
|
|
|
// ErrStopRequested is a restart refused because the run it would restart has been asked to stop. Not
|
|
// an error of the caller: the reconciler ends the run instead.
|
|
var ErrStopRequested = errors.New("pgstore: the run has been asked to stop")
|
|
|
|
// RequestStop records that THIS platform asked a run to stop, and hands back the unit to ask.
|
|
//
|
|
// The order is the whole design (migration 00014): the intent is COMMITTED before systemd is
|
|
// touched, so a platform that dies between the two still knows on its next sweep that the run it
|
|
// finds ended was stopped on purpose — and, just as important, that it must not restart it.
|
|
//
|
|
// Idempotent by coalesce: pressing stop twice is one intent with the FIRST timestamp, because the
|
|
// timestamp is evidence about which of the two events came first and a later one would erase that.
|
|
//
|
|
// The revision it answers with is the BOOK's — see ReadRun for why every run-carrying response uses
|
|
// one counter.
|
|
func (s *Store) RequestStop(ctx context.Context, userID, runID string, now time.Time) (Run, string, error) {
|
|
// The bar comes from the shared fragment and is not spelled out again here: a second copy is how
|
|
// a projection drifts, and this one already needed the segment predicate twice.
|
|
const q = `
|
|
update runs r set stop_requested_at = coalesce(r.stop_requested_at, $3)
|
|
from books b
|
|
where r.id = $1 and b.id = r.book_id and b.owner_id = $2 and r.finished_at is null
|
|
returning ` + runRow + `,
|
|
coalesce((select a.unit_name from run_attempts a
|
|
where a.run_id = r.id and a.ended_at is null
|
|
order by a.attempt_no desc limit 1), '')`
|
|
var out Run
|
|
var unit string
|
|
err := scanRun(s.pool.QueryRow(ctx, q, runID, userID, now), &out, &unit)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
// Nothing matched, and the two reasons need different answers. Asked separately and only on
|
|
// this path, so the ordinary stop stays one round trip.
|
|
return Run{}, "", s.whyNotLive(ctx, userID, runID)
|
|
}
|
|
if err != nil {
|
|
return Run{}, "", fmt.Errorf("pgstore: request stop: %w", err)
|
|
}
|
|
return out, unit, nil
|
|
}
|
|
|
|
func (s *Store) whyNotLive(ctx context.Context, userID, runID string) error {
|
|
var visible bool
|
|
if err := s.pool.QueryRow(ctx, `
|
|
select exists (select 1 from runs r join books b on b.id = r.book_id
|
|
where r.id = $1 and b.owner_id = $2)`, runID, userID).Scan(&visible); err != nil {
|
|
return fmt.Errorf("pgstore: read run: %w", err)
|
|
}
|
|
if visible {
|
|
return ErrRunNotLive
|
|
}
|
|
return ErrNoRun
|
|
}
|
|
|
|
// ReadRunForResume loads a run and its LAST attempt, whether or not either is still live.
|
|
//
|
|
// The reconciler's own list is deliberately not reusable here: it selects the attempt that has not
|
|
// ended, and every run this call is about has ended. What resume needs is the state the run stopped
|
|
// in and the attempt whose money and journal position it stopped at.
|
|
// LatestRun answers which run of the book every book-scoped read resolves — a LIVE run first and
|
|
// then the newest-started, the `newestRun` order the read model's `lastRun` splices — and whether
|
|
// that row is still live. The resume gate compares against it: re-opening any OTHER run leaves the
|
|
// card and every progress frame quoting the wrong row, and the two reasons carry different words —
|
|
// a LIVE newer run is «the book is being translated», a finished one is «resume the latest».
|
|
//
|
|
// ⚠ The order is SPLICED and not written out again, and that is the half of PD-402 the read fix
|
|
// could not do alone: this query used to hand-write `order by started_at desc` while the read model
|
|
// spliced a constant, and the two are the same question. Answered differently they invert the
|
|
// defect instead of closing it — every surface would follow the live run while this gate refused to
|
|
// resume it, calling it "not the latest".
|
|
func (s *Store) LatestRun(ctx context.Context, bookID string) (id string, live bool, err error) {
|
|
err = s.pool.QueryRow(ctx, `
|
|
select id, finished_at is null from runs
|
|
where book_id = $1`+newestRun,
|
|
bookID).Scan(&id, &live)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return "", false, ErrNoRun
|
|
}
|
|
if err != nil {
|
|
return "", false, fmt.Errorf("pgstore: latest run: %w", err)
|
|
}
|
|
return id, live, nil
|
|
}
|
|
|
|
func (s *Store) ReadRunForResume(ctx context.Context, userID, runID string) (LiveRun, error) {
|
|
rows, err := s.queryRuns(ctx, `
|
|
join run_attempts a on a.run_id = r.id
|
|
and a.attempt_no = (select max(attempt_no) from run_attempts
|
|
where run_id = r.id)
|
|
left join reservations res on res.engine_run_id = r.id || '#' || a.attempt_no
|
|
and res.state = 'open'
|
|
where r.id = $1 and b.owner_id = $2`, runID, userID)
|
|
if err != nil {
|
|
return LiveRun{}, err
|
|
}
|
|
if len(rows) == 0 {
|
|
return LiveRun{}, ErrNoRun
|
|
}
|
|
return rows[0], nil
|
|
}
|
|
|
|
// ReadRun is the run row as the contract projects it, read under the caller's ownership.
|
|
//
|
|
// ⚠ The revision is the BOOK's, not the `runs.revision` column, and that is the contract rather than
|
|
// a shortcut: "the counter is PER BOOK — every book-scoped read and the id of every stream frame of
|
|
// that book's run carry the same number" (§Revision), and a client MUST DROP a read whose revision is
|
|
// below one it has applied. The book card already answers this way (httpapi getBook); a handle that
|
|
// answered from the run's own column would hand the client a number below the card's and the client,
|
|
// obeying the contract, would drop the answer to the button it just pressed.
|
|
func (s *Store) ReadRun(ctx context.Context, userID, runID string) (Run, error) {
|
|
// The BAR travels with every receipt and not only with the book card: a client that reads
|
|
// `progress.total` against `ceiling_chapters` (canon §Run) would otherwise see 0/0 on the answer
|
|
// to its own stop or resume, stamped with a fresh revision — which passes its staleness guard and
|
|
// rolls the bar it was watching back to zero.
|
|
const q = `
|
|
select ` + runRow + `
|
|
from books b join runs r on r.book_id = b.id
|
|
where r.id = $1 and b.owner_id = $2`
|
|
var out Run
|
|
err := scanRun(s.pool.QueryRow(ctx, q, runID, userID), &out)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return Run{}, ErrNoRun
|
|
}
|
|
if err != nil {
|
|
return Run{}, fmt.Errorf("pgstore: read run: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// RestartInput is one interrupted attempt being replaced.
|
|
type RestartInput struct {
|
|
RunID string
|
|
AttemptID int64
|
|
UserID string
|
|
BookID string
|
|
// Ceiling is what is LEFT of the run's budget. A restart that reserved the full ceiling again
|
|
// would let one run spend it twice.
|
|
Ceiling money.MicroUSD
|
|
Offset int64
|
|
// EngineBinary overrides the pinned path. Nil means INHERIT — which is the default, because a
|
|
// resume is the same run continuing and row 139 lets another version in only on purpose.
|
|
EngineBinary *string
|
|
// Resnapshot/AcceptRebill GRANT the re-pass consents on re-open (a resume of a run the bank
|
|
// moved under — P10 §3.1). Widening only: the row keeps a consent it already carries, and the
|
|
// sum only grows (a smaller fresh projection is covered by the larger consent already given).
|
|
// The reconciler's restarts pass zero values and change nothing.
|
|
Resnapshot bool
|
|
AcceptRebill money.MicroUSD
|
|
// OnlyIfLive refuses to re-open a run that has already FINISHED. The reconciler sets it and a
|
|
// resume does not, and the difference is money.
|
|
//
|
|
// A sweep decides from a snapshot and writes seconds later, and another generation of the sweep
|
|
// can close the run inside that window (register row PD-181's class). Without this, the second
|
|
// pass would clear `finished_at`, take a fresh hold and spawn an engine for a run whose owner has
|
|
// already been told it ended — and the run the user sees finished starts spending again. A resume
|
|
// does exactly that on purpose, which is why the guard is the caller's to ask for.
|
|
OnlyIfLive bool
|
|
Now time.Time
|
|
}
|
|
|
|
// ErrRunFinished is a restart refused because another pass has already ended the run. Not an error of
|
|
// the caller: the pass that holds the stale snapshot simply has nothing left to do.
|
|
var ErrRunFinished = errors.New("pgstore: the run has already finished")
|
|
|
|
// RestartRun closes an interrupted attempt and opens the next one, with its own hold, in ONE
|
|
// transaction — the same reason StartRun is one transaction.
|
|
//
|
|
// The old attempt's reservation is NOT carried over: it was taken for a process that no longer
|
|
// exists, and it is closed by the settlement that runs before this.
|
|
func (s *Store) RestartRun(ctx context.Context, in RestartInput) (LiveRun, error) {
|
|
var out LiveRun
|
|
err := s.inTx(ctx, func(tx Tx) error {
|
|
// Book first, as in every other transaction that touches both (lockBook). This one took the
|
|
// attempt first and the materializer takes the book first, which is the pair that deadlocked.
|
|
if err := lockBook(ctx, tx, in.BookID); err != nil {
|
|
return err
|
|
}
|
|
// The RUN row second — the order lockBook writes down, which this transaction used to take
|
|
// last, after the money. Taking it here also does the work below: what is read from the row is
|
|
// read under the lock that will do the writing.
|
|
var stopRequested, finished *time.Time
|
|
if err := tx.QueryRow(ctx,
|
|
`select stop_requested_at, finished_at from runs where id = $1 for update`, in.RunID).
|
|
Scan(&stopRequested, &finished); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrNoRun
|
|
}
|
|
return fmt.Errorf("pgstore: lock run: %w", err)
|
|
}
|
|
// A stop asked for on a run that is STILL LIVE outranks a restart, and the check belongs here
|
|
// rather than in the caller's snapshot: the reconciler decides to restart, spends seconds
|
|
// settling (a `tmctl status` call), and the user presses stop inside that window. Restarting
|
|
// then cleared the fresh intent, opened a second attempt and took a new hold — the user had a
|
|
// 202 for a stop that never happened and paid for the work they had just cancelled.
|
|
//
|
|
// On a FINISHED run the same column is history: a resume re-opens a run that was stopped, and
|
|
// the intent belongs to the life that ended — it is cleared below with the rest of what said
|
|
// the run was over.
|
|
//
|
|
// A stop arriving DURING this transaction is not lost either: it waits on this row lock and
|
|
// lands on the re-opened run, which the next sweep then stops.
|
|
if stopRequested != nil && finished == nil {
|
|
return ErrStopRequested
|
|
}
|
|
// The reconciler asked about a run it believed was still going. If it is not, this pass is
|
|
// holding a snapshot another generation has already acted on, and re-opening would resurrect a
|
|
// finished run with a fresh hold — see OnlyIfLive.
|
|
if in.OnlyIfLive && finished != nil {
|
|
return ErrRunFinished
|
|
}
|
|
// The previous attempt is closed if it is still open, and left exactly as it is if it is not.
|
|
// Both callers arrive here: the reconciler replaces an attempt that was INTERRUPTED and is
|
|
// still open, while a resume continues a run whose attempt already ended with a verdict of its
|
|
// own — and overwriting that verdict would erase how the run the user stopped actually ended.
|
|
//
|
|
// What serializes two callers is no longer this row but the next one: `unique (run_id,
|
|
// attempt_no)` lets exactly one of them insert attempt N+1, and the loser gets the same
|
|
// ErrNoRun it always got.
|
|
var attemptNo int
|
|
if err := tx.QueryRow(ctx, `
|
|
update run_attempts set ended_at = coalesce(ended_at, $2),
|
|
exit_result = coalesce(exit_result, 'interrupted')
|
|
where id = $1 returning attempt_no`, in.AttemptID, in.Now).Scan(&attemptNo); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrNoRun // the attempt is gone
|
|
}
|
|
return fmt.Errorf("pgstore: close interrupted attempt: %w", err)
|
|
}
|
|
next := attemptNo + 1
|
|
var id int64
|
|
var pinned string
|
|
// The new attempt INHERITS the binary the run was pinned to (unified backlog row 139): the
|
|
// engine ships more often than a run finishes, so resuming on whatever was deployed since is a
|
|
// different program continuing someone else's work. Changing it is allowed, but only by saying
|
|
// so — see runs.Config.AllowEngineVersionChange.
|
|
if err := tx.QueryRow(ctx, `
|
|
insert into run_attempts (run_id, attempt_no, started_at, last_offset, engine_binary, engine_run_id)
|
|
select $1, $2, $3, $4, coalesce($5, prev.engine_binary, ''), $7
|
|
from run_attempts prev where prev.id = $6
|
|
returning id, engine_binary`, in.RunID, next, in.Now, in.Offset, in.EngineBinary, in.AttemptID,
|
|
EngineStreamID(in.RunID, next)).
|
|
Scan(&id, &pinned); err != nil {
|
|
if isUnique(err, "run_attempts_run_id_attempt_no_key") {
|
|
return ErrNoRun // another caller opened this attempt first
|
|
}
|
|
return fmt.Errorf("pgstore: open next attempt: %w", err)
|
|
}
|
|
if err := holdTx(ctx, tx, in.UserID, in.BookID, engineRunKey(in.RunID, next), in.Ceiling, in.Now); err != nil {
|
|
return err
|
|
}
|
|
// finished_at, settled_at and stop_requested_at are CLEARED, and each for its own reason. The
|
|
// reconciler's restart works on a run where all three are already null, so there it changes
|
|
// nothing; a RESUME re-opens a run that ended, and leaving them would mean a live run the
|
|
// one-live-per-book index does not see, money marked resolved that this attempt has not spent
|
|
// yet, and a stop request from the previous life that would classify this attempt's ending as
|
|
// a stop nobody asked for.
|
|
// ⚠ The bar's baselines are NOT re-taken here, and that is the through-bar's one rule (row
|
|
// 200): the bar counts both waves against the baselines of the run's START, so lifting the
|
|
// stop moves nothing — the draft half stands at what the first segment did and the last-pass
|
|
// half continues from where the run began. The re-basing that used to live here is what made
|
|
// the bar restart from zero at the signing stop.
|
|
if _, err := tx.Exec(ctx, `
|
|
update runs r set status = 'translating', paused_reason = null, finished_at = null,
|
|
settled_at = null, stop_requested_at = null,
|
|
resnapshot = r.resnapshot or $2,
|
|
accept_rebill_micro = greatest(r.accept_rebill_micro, $3),
|
|
revision = b.revision + 1
|
|
from books b
|
|
where r.id = $1 and b.id = r.book_id`, in.RunID,
|
|
in.Resnapshot, int64(in.AcceptRebill)); err != nil {
|
|
// Clearing finished_at puts the run back under the one-live-run-per-book index, and the
|
|
// book may already have a NEWER live run — nothing stops an account starting one after it
|
|
// stopped this one. That is a conflict and not a failure: the whole transaction rolls back,
|
|
// so the hold taken three lines above is undone with it, and the caller gets the same error
|
|
// a second admission would have got.
|
|
if isUnique(err, "runs_one_live_per_book") {
|
|
return ErrRunInFlight
|
|
}
|
|
return fmt.Errorf("pgstore: reopen run: %w", err)
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
update books set status = 'translating',
|
|
revision = `+nextRevisionOfThisBooksLibrary+` where id = $1`,
|
|
in.BookID); err != nil {
|
|
return fmt.Errorf("pgstore: mark book translating: %w", err)
|
|
}
|
|
// A restart moves the book back to `translating` — a status change like any other, and one a
|
|
// watching client has no other way to learn: without the frame the stream stays silent and a
|
|
// reconnect can even be answered 204 ("stop reconnecting") on a run that is going again.
|
|
if err := emitStatus(ctx, tx, in.BookID); err != nil {
|
|
return err
|
|
}
|
|
out = LiveRun{RunID: in.RunID, BookID: in.BookID, UserID: in.UserID, AttemptID: id,
|
|
AttemptNo: next, Ceiling: in.Ceiling, StartedAt: in.Now, AttemptStartedAt: in.Now,
|
|
EngineBinary: pinned, Position: Position{Offset: in.Offset}}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return LiveRun{}, err
|
|
}
|
|
// The fields the caller needs to spawn but this transaction did not read.
|
|
//
|
|
// ⚠ THE ORDER IS DELIBERATELY NOT AMONG THEM. It is resolved on the spawn path, by
|
|
// ReadOrderForSpawn, once per spawn — see runColumns for why it is not carried on every LiveRun.
|
|
// What THIS list must never lose is a field the spawn reads FROM the LiveRun: a field left out
|
|
// here reaches the spawn as its zero value, which for the order used to mean «no boundary» and
|
|
// therefore «the whole book» (found by TestTheVolumeAllowanceIsWhatIsLeftOfTheOrderAtEverySpawn,
|
|
// before the order moved off this struct).
|
|
const q = `select b.workdir, r.verify_bank, r.resnapshot, r.accept_rebill_micro,
|
|
r.ceiling_chapters
|
|
from runs r join books b on b.id = r.book_id where r.id = $1`
|
|
var acceptRebill int64
|
|
if err := s.pool.QueryRow(ctx, q, in.RunID).Scan(&out.Workdir, &out.VerifyBank,
|
|
&out.Resnapshot, &acceptRebill, &out.OrderedChapters); err != nil {
|
|
return LiveRun{}, fmt.Errorf("pgstore: read restarted run: %w", err)
|
|
}
|
|
out.AcceptRebill = money.MicroUSD(acceptRebill)
|
|
return out, nil
|
|
}
|
|
|
|
// ReleaseSpawnClaim gives an attempt back after a unit could NOT be created, so the next sweep
|
|
// retries this attempt instead of reading a recorded-but-absent unit as an interrupted run.
|
|
func (s *Store) ReleaseSpawnClaim(ctx context.Context, attemptID int64) error {
|
|
_, err := s.pool.Exec(ctx,
|
|
`update run_attempts set unit_name = null where id = $1`, attemptID)
|
|
if err != nil {
|
|
return fmt.Errorf("pgstore: release spawn claim: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AttemptReservationOpen reports whether an attempt's money is still reserved. The restart path asks
|
|
// before it opens a SECOND reservation: settling can legitimately fail (the engine's figure could not
|
|
// be read), and proceeding then holds the ceiling twice and strands the first hold where no sweep
|
|
// looks for it again.
|
|
func (s *Store) AttemptReservationOpen(ctx context.Context, runID string, attempt int) (bool, error) {
|
|
var open bool
|
|
err := s.pool.QueryRow(ctx, `
|
|
select exists (select 1 from reservations where engine_run_id = $1 and state = 'open')`,
|
|
engineRunKey(runID, attempt)).Scan(&open)
|
|
if err != nil {
|
|
return false, fmt.Errorf("pgstore: read reservation state: %w", err)
|
|
}
|
|
return open, nil
|
|
}
|
|
|
|
// RunPausedReason reads what the run's projection currently says: its STATUS and its ceiling reason.
|
|
//
|
|
// It exists because the reconciler's snapshot is taken BEFORE the journal is drained, and the events
|
|
// that decide how an ending is classified can arrive in that very drain — the ceiling event, and
|
|
// (since the bank-stop protection of unified-backlog row 240) the bank-stop event, which writes
|
|
// `awaiting_bank` on the row and nothing the snapshot carries. Judging the ending from the snapshot answered `failed` for a run whose own stream had just
|
|
// said `ceiling` — one sweep of staleness, on the one branch where being wrong is contractually
|
|
// visible (acceptance of D39.131, п.3).
|
|
//
|
|
// An empty string is "nothing said so", not an error: most runs never pause.
|
|
func (s *Store) RunPausedReason(ctx context.Context, runID string) (status, reason string, err error) {
|
|
err = s.pool.QueryRow(ctx,
|
|
`select status, coalesce(paused_reason, '') from runs where id = $1`, runID).Scan(&status, &reason)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return "", "", nil // the run is gone; the caller's next write refuses on its own
|
|
}
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("pgstore: read run state: %w", err)
|
|
}
|
|
return status, reason, nil
|
|
}
|
|
|
|
// PauseRun records a run that cannot go on, without pretending it failed.
|
|
//
|
|
// paused is false when the write did not apply — the run was already over, or the attempt it was
|
|
// asked to close is no longer the run's live one. It is a RESULT and not a silent nil because the
|
|
// caller logs "the run was paused" from it, and reporting a pause that did not happen is how a run
|
|
// that someone else finished got announced twice (register row PD-141).
|
|
func (s *Store) PauseRun(ctx context.Context, runID string, attemptID int64, reason string, now time.Time) (paused bool, err error) {
|
|
if !validPauseReason(reason) {
|
|
return false, fmt.Errorf("pgstore: %q is not a pause reason", reason)
|
|
}
|
|
err = s.inTx(ctx, func(tx Tx) error {
|
|
// Book first, as everywhere else that touches both (see FinishRun).
|
|
var bookID string
|
|
if err := tx.QueryRow(ctx,
|
|
`select id from books where id = (select book_id from runs where id = $1) for update`,
|
|
runID).Scan(&bookID); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("pgstore: lock book: %w", err)
|
|
}
|
|
// A stop the user asked for outranks a pause, and the check belongs inside this TRANSACTION,
|
|
// under the same locks: the reconciler decides to pause after settling — seconds of a
|
|
// `tmctl status` call — and a stop landing inside that window would otherwise be answered with
|
|
// `paused/credit_exhausted`, which
|
|
// says the money ran out when what happened is that its owner stopped it.
|
|
var stopRequested *time.Time
|
|
if err := tx.QueryRow(ctx,
|
|
`select stop_requested_at from runs where id = $1 for update`, runID).Scan(&stopRequested); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("pgstore: lock run: %w", err)
|
|
}
|
|
if stopRequested != nil {
|
|
return ErrStopRequested
|
|
}
|
|
// The attempt being closed must still be a LIVE attempt OF THIS RUN — the third path to carry
|
|
// the guard `FinishRun` (PD-181) and `FinishUnspawnedStop` (FP5-2) already have, and it was
|
|
// missing here. Without it a pass holding an old snapshot pauses a run whose attempt has since
|
|
// been restarted: the run reads `finished`, the SECOND attempt's hold stays open and falls out
|
|
// of both `ListLiveRuns` (the run is finished) and `UnsettledRuns` (the attempt is not), and an
|
|
// engine keeps spending under a run its owner is told ran out of money.
|
|
if err := tx.QueryRow(ctx, `
|
|
update runs set status = 'paused', paused_reason = $2, finished_at = $3,
|
|
revision = (select revision + 1 from books where id = runs.book_id)
|
|
where id = $1 and finished_at is null
|
|
and exists (select 1 from run_attempts a
|
|
where a.id = $4 and a.run_id = runs.id and a.ended_at is null)
|
|
returning book_id`, runID, reason, now, attemptID).Scan(&bookID); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil // already over, or this is not the run's live attempt any more
|
|
}
|
|
return fmt.Errorf("pgstore: pause run: %w", err)
|
|
}
|
|
paused = true
|
|
if _, err := tx.Exec(ctx, `
|
|
update run_attempts set ended_at = coalesce(ended_at, $3)
|
|
where id = $1 and run_id = $2`, attemptID, runID, now); err != nil {
|
|
return fmt.Errorf("pgstore: end attempt: %w", err)
|
|
}
|
|
// Owing a materialization like every other ending: a run that halted at a ceiling has translated
|
|
// everything up to it, and that text is bought and paid for.
|
|
if _, err := tx.Exec(ctx, `
|
|
update books set status = 'paused', `+owesAReadingSurface+`
|
|
revision = `+nextRevisionOfThisBooksLibrary+` where id = $1`, bookID); err != nil {
|
|
return fmt.Errorf("pgstore: pause book: %w", err)
|
|
}
|
|
// This is the reconciler's pause — the real credit-exhausted one — and it sets `finished_at`,
|
|
// which is what makes the book read as AT REST. Without a frame the stream ends and the next
|
|
// reconnect is answered 204 while the client still holds `translating`.
|
|
return emitStatus(ctx, tx, bookID)
|
|
})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return paused, nil
|
|
}
|
|
|
|
// RunForSpawn is what the worker needs to spawn an attempt.
|
|
type RunForSpawn struct {
|
|
LiveRun
|
|
// AlreadySpawned is a unit name that was recorded for this attempt. The worker must NOT spawn a
|
|
// second engine for it: a queue job can be retried after a platform restart, and the run it
|
|
// refers to may well still be running.
|
|
AlreadySpawned bool
|
|
}
|
|
|
|
// ReadRunForSpawn loads the live attempt of a run.
|
|
func (s *Store) ReadRunForSpawn(ctx context.Context, runID string) (RunForSpawn, error) {
|
|
live, err := s.ListLiveRuns(ctx)
|
|
if err != nil {
|
|
return RunForSpawn{}, err
|
|
}
|
|
for _, l := range live {
|
|
if l.RunID == runID {
|
|
return RunForSpawn{LiveRun: l, AlreadySpawned: l.UnitName != ""}, nil
|
|
}
|
|
}
|
|
return RunForSpawn{}, ErrNoRun
|
|
}
|
|
|
|
// SpawnRecord is what is written down about an attempt just before its unit is created.
|
|
type SpawnRecord struct {
|
|
AttemptID int64
|
|
Unit string
|
|
// Binary is the versioned engine path this attempt is pinned to (unified backlog row 139).
|
|
Binary string
|
|
// EngineRunID is the stream identity the platform gives this attempt's engine, in the same write
|
|
// that claims the right to start it. Recorded BEFORE the handshake rather than learned from it:
|
|
// the journal is per book, so a reader that adopts the first hello it meets adopts whatever
|
|
// happens to be written at its offset (runs.engineStreamID).
|
|
EngineRunID string
|
|
// Ceiling is the attempt's own budget: the increment the user bought and the amount held.
|
|
Ceiling money.MicroUSD
|
|
// CeilingArg is what the engine is actually told, which is the book's cumulative cap and
|
|
// therefore a different number from the second run of a book onwards (D39.122).
|
|
CeilingArg money.MicroUSD
|
|
// Baseline is where the book's lifetime meter stood before this attempt added to it.
|
|
Baseline money.MicroUSD
|
|
}
|
|
|
|
// RecordSpawn claims the right to start this attempt, and writes down what is about to run: the
|
|
// unit, the binary version it is pinned to, the budget it carries and the ceiling it is given.
|
|
// Written BEFORE the unit is created, so a crash between the two leaves a record to reconcile rather
|
|
// than an unattributable process.
|
|
//
|
|
// claimed is false when the attempt already has a unit name, when the attempt has ENDED, or when its
|
|
// run is over. It is a COMPARE-AND-SET rather than a plain update because two callers legitimately
|
|
// reach here at once — the queue worker that was handed the run and the reconciler that found it
|
|
// unspawned — and the loser must not start a second engine. systemd would refuse the duplicate NAME,
|
|
// so that accident was survivable; surviving by someone else's uniqueness rule is not the same as
|
|
// being correct, and the day a resume changes the naming it stops holding.
|
|
//
|
|
// ⚠ The three conditions BESIDES the unit name are money, and they were not here until a run could be
|
|
// stopped before it ever spawned. The window: a worker sits inside `bookMeter` for the seconds a
|
|
// `tmctl status` takes, the user stops the run, the sweep ends it and gives the whole hold back
|
|
// (nothing was spawned) — and the worker then wakes up and creates a unit for a run that is finished
|
|
// and settled. That engine would spend against its own book cap with NO open reservation, and
|
|
// nothing would ever look at it: the reconciler lists runs by `finished_at is null` and settlements
|
|
// by an open reservation, so it is in neither list. The stop INTENT is asked about one step earlier
|
|
// for the same reason: it lands while the worker is inside that same `bookMeter`, before any sweep
|
|
// could have finished the run, and a claim granted then starts an engine for work its owner
|
|
// cancelled before it began. All three are asked in the statement that claims, because a check made
|
|
// before it is a check with a window after it.
|
|
func (s *Store) RecordSpawn(ctx context.Context, r SpawnRecord) (claimed bool, err error) {
|
|
// What a PREVIOUS claim of this attempt decided is kept. The claim is given back when the unit
|
|
// could not be created (ReleaseSpawnClaim), and "could not be created" is not the same as "was not
|
|
// created": a systemd-run that was killed after it had already asked for the unit reports a failure
|
|
// and leaves an engine running. The next claim would then overwrite the baseline with a meter that
|
|
// engine has been moving, and the attempt would be billed for the difference from a figure that
|
|
// already includes its own work — an underpayment nothing later looks for. Where the unit really
|
|
// was not created the two values are identical, so keeping the first costs nothing.
|
|
// engine_run_id is ASSIGNED and not coalesced. It is derived from (run, attempt), so a retry of the
|
|
// same claim computes the same string and the "keep what the first claim decided" rule buys
|
|
// nothing — what a coalesce could keep is a FOREIGN id an early drain adopted before the row was
|
|
// named (see EngineStreamID), and keeping that would blind the run for its whole life. Attempts
|
|
// created since EngineStreamID moved into their INSERT already carry it; this is the backfill for
|
|
// the ones that do not.
|
|
tag, err := s.pool.Exec(ctx, `
|
|
update run_attempts
|
|
set unit_name = $2, engine_binary = $3, ceiling_micro_usd = $4,
|
|
-- Incremented inside the claim itself: a count written anywhere else would have a
|
|
-- window beside it. Read by AbandonOrder.ProofSpawns (PD-424).
|
|
spawns = spawns + 1,
|
|
ceiling_arg_micro_usd = case when spend_baseline_micro_usd is null
|
|
then $5 else ceiling_arg_micro_usd end,
|
|
spend_baseline_micro_usd = coalesce(spend_baseline_micro_usd, $6),
|
|
engine_run_id = coalesce(nullif($7, ''), engine_run_id)
|
|
where id = $1 and unit_name is null and ended_at is null
|
|
and exists (select 1 from runs r where r.id = run_attempts.run_id
|
|
and r.finished_at is null and r.stop_requested_at is null)`,
|
|
r.AttemptID, r.Unit, r.Binary, int64(r.Ceiling), int64(r.CeilingArg), int64(r.Baseline),
|
|
r.EngineRunID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("pgstore: record spawn: %w", err)
|
|
}
|
|
return tag.RowsAffected() == 1, nil
|
|
}
|
|
|
|
// MarkParked records that this attempt's projection has parked, or that it no longer has.
|
|
//
|
|
// ⚠ SET ONCE AND CLEARED ONCE, which is what makes the timestamp mean «since when» rather than «as of
|
|
// the last sweep». The park is re-derived from (cursor, journal) on every pass, so a plain write
|
|
// would move the stamp forward forever and the one question an operator asks — how long has this run
|
|
// been quiet — would have no answer anywhere.
|
|
//
|
|
// The caller only reaches here when its verdict DIFFERS from the snapshot's, so the steady state
|
|
// costs no statement at all; the guards below are the second half of that, for the two sweeps that
|
|
// disagree at once.
|
|
func (s *Store) MarkParked(ctx context.Context, attemptID int64, parked bool, now time.Time) error {
|
|
// Two whole statements rather than one built from pieces: the battery plans every statement in
|
|
// this package against the migrated schema, and it can only do that for a CONSTANT
|
|
// (sqlgate_test.go). A query assembled at run time is a query nothing checks.
|
|
var err error
|
|
if parked {
|
|
_, err = s.pool.Exec(ctx,
|
|
`update run_attempts set parked_at = $2 where id = $1 and parked_at is null`, attemptID, now)
|
|
} else {
|
|
_, err = s.pool.Exec(ctx,
|
|
`update run_attempts set parked_at = null where id = $1 and parked_at is not null`, attemptID)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("pgstore: mark the attempt parked: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SaveCursor persists the tailer's position when nothing was materialized — a re-read of lines the
|
|
// cursor already covers still moves the byte hint, and losing that means re-reading them forever.
|
|
func (s *Store) SaveCursor(ctx context.Context, attemptID int64, p Position) error {
|
|
_, err := s.pool.Exec(ctx, `
|
|
update run_attempts set last_offset = $2 where id = $1 and last_offset < $2`, attemptID, p.Offset)
|
|
if err != nil {
|
|
return fmt.Errorf("pgstore: save cursor: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Quarantine stops materializing an attempt without touching the run.
|
|
//
|
|
// The engine is NOT stopped: it is spending money the account has already reserved, and our
|
|
// inability to read its journal is not a reason to throw that away. What stops is the projection —
|
|
// after this the run's state is only as fresh as the resync channel makes it, and the reason says
|
|
// so out loud rather than leaving a screen that quietly stopped moving.
|
|
func (s *Store) Quarantine(ctx context.Context, attemptID int64, reason string) error {
|
|
_, err := s.pool.Exec(ctx,
|
|
`update run_attempts set quarantine_reason = $2 where id = $1 and quarantine_reason is null`,
|
|
attemptID, reason)
|
|
if err != nil {
|
|
return fmt.Errorf("pgstore: quarantine attempt: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var (
|
|
// ErrNotQuarantined is a lift asked for on a live attempt that is not quarantined. Its own word,
|
|
// apart from "no such run": the operator was right about the run and wrong about its state, and
|
|
// the answer is "nothing to do" rather than "look elsewhere".
|
|
ErrNotQuarantined = errors.New("pgstore: the run's live attempt is not quarantined")
|
|
// ErrNoLiveAttempt is a lift asked for on a run with no attempt still open. A finished run's
|
|
// journal is not materialized by anyone, so there is nothing a lift could resume.
|
|
ErrNoLiveAttempt = errors.New("pgstore: the run has no live attempt")
|
|
)
|
|
|
|
// LiftedQuarantine is what Unquarantine found and cleared: the attempt, the reason it carried, and
|
|
// the cursor the next sweep reads on from.
|
|
type LiftedQuarantine struct {
|
|
AttemptNo int
|
|
Reason string
|
|
Position Position
|
|
}
|
|
|
|
// Unquarantine clears the quarantine of a run's LIVE attempt, so the next sweep materializes its
|
|
// journal again from the cursor the projection stopped at.
|
|
//
|
|
// It is the operator's half of Quarantine (PD-426): the state it stands for — a journal this build
|
|
// could not read from here on — is not always permanent (an operator's stray tmctl stops writing; a
|
|
// build is replaced; a release relaxes a reader's rule), and a column nothing clears would make it
|
|
// so. What is NOT touched is the cursor: it is the record of what was applied, and the tailer
|
|
// resumes from it. If the same bytes are still unreadable the next sweep quarantines again with the
|
|
// same reason, which is the honest answer and is visible in the runs listing. No money moves and no
|
|
// process is touched.
|
|
func (s *Store) Unquarantine(ctx context.Context, runID string) (LiftedQuarantine, error) {
|
|
var out LiftedQuarantine
|
|
err := s.inTx(ctx, func(tx pgx.Tx) error {
|
|
// The RUN row first, in the order every transaction on a run takes (lockBook: book, then run,
|
|
// then its attempts). RestartRun closes the live attempt and opens the next one under this
|
|
// same row lock, so a lift that queued behind it reads the attempt that replaced the old one
|
|
// rather than a snapshot in which the run has no live attempt at all.
|
|
var one int
|
|
if err := tx.QueryRow(ctx, `select 1 from runs where id = $1 for update`, runID).Scan(&one); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrNoRun
|
|
}
|
|
return fmt.Errorf("pgstore: lock run: %w", err)
|
|
}
|
|
var id int64
|
|
var reason *string
|
|
err := tx.QueryRow(ctx, `
|
|
select id, attempt_no, quarantine_reason, last_offset, last_seq from run_attempts
|
|
where run_id = $1 and ended_at is null
|
|
order by attempt_no desc limit 1 for update`, runID).
|
|
Scan(&id, &out.AttemptNo, &reason, &out.Position.Offset, &out.Position.LastSeq)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrNoLiveAttempt
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("pgstore: read live attempt: %w", err)
|
|
}
|
|
if reason == nil {
|
|
return ErrNotQuarantined
|
|
}
|
|
out.Reason = *reason
|
|
if _, err := tx.Exec(ctx, `update run_attempts set quarantine_reason = null where id = $1`, id); err != nil {
|
|
return fmt.Errorf("pgstore: lift quarantine: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return LiftedQuarantine{}, err
|
|
}
|
|
return out, nil
|
|
}
|