textmachine/platform/internal/runs/runs.go

// Package runs owns the life of a translation run: admitting one, spawning its transient systemd
// unit, tailing what it writes, and deciding what happened to it afterwards.
//
// The shape follows from D39.106: the engine is NOT a child of this service, so no component here
// waits on a process. Admission (the queue) only hands out permission to start; everything after
// that is a RECONCILER that reads the world — Postgres, the book's journal, the exit marker — and
// moves the read model to match. That is what makes a run survive a deploy: the process that
// started it and the process that finishes it are usually not the same process.
package runs

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"sync"
	"time"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/money"
	"textmachine/platform/internal/pgstore"
	"textmachine/platform/internal/pricing"
	"textmachine/platform/internal/runner"
)

// UnitRunner is the transient-unit half of the seam. An interface so the reconciler's decisions can
// be pinned without systemd, and so a test that DOES want systemd is obviously a different test.
type UnitRunner interface {
	Start(ctx context.Context, s runner.Spec) error
	Stop(ctx context.Context, unit string) error
	Alive(ctx context.Context, unit string) (bool, error)
}

// EngineStatus is the ratified repair channel: `tmctl status --json` against a book.
type EngineStatus interface {
	Status(ctx context.Context, binary, workdir string) (ingest.StatusReport, error)
}

// BankApplier is the correction channel: `tmctl bank-apply` against a book (D39.156 pack 2в). An
// interface for the same reason EngineStatus is one: the door's decisions are pinned without an
// engine binary, and a test that wants the real verb is visibly a different test.
type BankApplier interface {
	BankApply(ctx context.Context, binary, workdir, decisionsPath string, preview bool) (runner.BankApplyOutcome, error)
}

// Config is everything the runner needs that an operator chooses.
type Config struct {
	// StateDir is where exit markers live. Platform state, deliberately NOT the book's directory:
	// the engine owns that one and the platform does not write into it (D39.110).
	StateDir string
	// EngineBinary is the VERSIONED path of tmctl (unified backlog row 139). An attempt is pinned to
	// the path it was started with, so a deploy of the engine mid-run cannot change what a resumed
	// attempt executes.
	EngineBinary string
	// MarkerArgv is the command systemd runs as ExecStopPost to write the exit marker.
	MarkerArgv []string
	// Ceiling is how the run's ceiling reaches the engine, as the argv template of the landed flag
	// (row 145). Empty means the configured override rendered nothing — whitespace, say, since an
	// unset variable takes the default — and then starting a run is refused rather than started under
	// the book's own limit.
	Ceiling runner.CeilingTemplate
	// KeysFile is the deployment's provider-key file, passed to `translate` as `--keys-file`
	// (row 211). The path is an engine argument: keys are never put into the unit's environment and
	// never pass through this process. Empty passes no flag.
	KeysFile string
	// MemoryMax and TasksMax bound ONE run's cgroup — the answer to PD-13.
	MemoryMax string
	TasksMax  int
	// AllowEngineVersionChange lets a RESUME run a different engine build than the one its run
	// started with. Off by default and it has to be turned on deliberately (unified backlog row 139):
	// the engine is deployed more often than a translation finishes, so the quiet behaviour would be
	// for a resumed run to continue under a program nobody chose for it.
	AllowEngineVersionChange bool
	// RunBudget is what ONE run may cost a pass of the sweep; zero takes the default. It bounds the
	// starvation the pass itself cannot: the list is ordered the same way every time, so a handful of
	// runs whose engine hangs used to mean the tail was never reached (register row PD-169).
	RunBudget time.Duration
	// ResyncEvery is how often a live run is reconciled from `tmctl status --json`. It is a SLOW
	// poll on purpose: every call re-ingests and re-chunks the source (~1.4-1.5 s of CPU on a 23 MB
	// book, unified backlog row 100). Since the emitter landed (D39.131) it is the REPAIR channel and
	// not the source of progress — see maybeResync for which runs still reach it.
	ResyncEvery time.Duration
}

// Service is the run lifecycle.
type Service struct {
	Store   *pgstore.Store
	Runner  UnitRunner
	Engine  EngineStatus
	Bank    BankApplier
	Pricing pricing.Model
	Queue   Enqueuer
	Cfg     Config
	Log     *slog.Logger
	// Now is injectable so the reconciler's clocks are testable without sleeping.
	Now func() time.Time

	// resynced remembers when each run was last reconciled from status. In memory on purpose: it is
	// a rate limit, not a fact — losing it on restart costs one extra status call.
	resynced map[string]time.Time
	// books serializes one book's admissions, resumes and corrections against each other — the
	// per-book rule of the correction door (see bank.go, lockBook). In memory on purpose: the
	// sections it guards live inside one process's calls.
	booksMu sync.Mutex
	books   map[string]*bookLock
}

// Enqueuer hands a run to the queue.
//
// Two methods because the two callers need different atomicity, not different queues: an ADMISSION
// writes the run, its hold and its queue entry in one transaction (a job for a run that does not
// exist, or a hold with no job, are both ways to lose money or work), while a RESUME re-opens a run
// that already exists and only then asks for a worker. A lost entry costs a sweep interval either
// way — the reconciler is the backstop for both.
type Enqueuer interface {
	EnqueueRun(ctx context.Context, tx pgstore.Tx, runID string) error
	EnqueueRunNow(ctx context.Context, runID string) error
}

func (s *Service) now() time.Time {
	if s.Now != nil {
		return s.Now()
	}
	return time.Now()
}

func (s *Service) log() *slog.Logger {
	if s.Log == nil {
		return slog.New(slog.DiscardHandler)
	}
	return s.Log
}

// ErrCeilingOutOfBounds is a requested ceiling the account or the book cannot carry. It is the
// contract's 409 on a run start: the bounds are read by GET run-options and may move between that
// read and this call, because a hold taken for another book lowers what is left.
var ErrCeilingOutOfBounds = errors.New("runs: the requested ceiling is outside the bounds")

// ErrRePassUnavailable — a re-pass was asked for on a book that has nothing to re-pass: the bank
// has not moved since the last run, or its move touched no already-paid unit. The remedy is the
// options read, which is what says whether a re-pass exists to buy (P10 §3.2).
var ErrRePassUnavailable = errors.New("runs: the book has nothing to re-pass")

// ErrRunnerIncomplete is a DEPLOYMENT that cannot start runs — the exit-marker command is missing,
// so a unit that ended would have no way to say so and its hold would stay reserved forever.
//
// It is a refusal of the run and NOT of the service: an instance whose runner is misconfigured still
// serves every read, and taking the library down with it would turn a partial outage into a total
// one during exactly the operation that causes it, a binary upgrade.
var ErrRunnerIncomplete = errors.New("runs: this deployment cannot record how a run ends")

// ErrBookNotReady is a run asked for on a book whose intake has not finished — one still arriving,
// still being cut, or rejected. The contract's 409: the book exists and the client can see it, and
// what cannot happen yet is a translation of it.
var ErrBookNotReady = errors.New("runs: the book is not ready to be translated")

// ErrNotStoppable is a stop asked for on a run that is already over.
var ErrNotStoppable = errors.New("runs: the run is not live")

// ErrNotResumable is a resume that would not move the run: a run that is already going, or one that
// finished. A bank stop is NOT among them — signing is one act over the whole bank and `resume`
// lifts that stop with the decisions as they stand (D39.144).
var ErrNotResumable = errors.New("runs: the run cannot be continued")

// ErrCeilingReached is a resume of a run that stopped at a LIMIT — the chapters it bought, or the
// credit behind them. It is ErrNotResumable with the one cause the contract enumerates, because the
// remedy differs and is the thing a client has to be told: a limit travels with the START of a run,
// so a NEW run with a larger one continues the work and this call never will.
var ErrCeilingReached = fmt.Errorf("%w: it stopped at a limit", ErrNotResumable)

// ErrCreditUnavailable is a resume of a run that has room LEFT in its limit, over an account that
// cannot cover the rest of it. Told apart from ErrCeilingReached because the remedies are opposite:
// this one clears when the account is topped up and the SAME run then continues, while that one is
// finished with and needs a new run. Answering either in place of the other sends the user to do the
// wrong thing (canon §resumeRun).
var ErrCreditUnavailable = fmt.Errorf("%w: the account cannot cover the rest of it", ErrNotResumable)

// CreditHeldError is ErrCeilingOutOfBounds with the book that caused it: another book of the account
// has a run holding the credit, and naming it is what lets a client offer the user somewhere to go.
type CreditHeldError struct{ BookID string }

func (e *CreditHeldError) Error() string {
	return "runs: the account's credit is held by another book"
}

func (e *CreditHeldError) Is(target error) bool { return target == ErrCeilingOutOfBounds }

// Options is what a run may be started with: the scale, and why it is smaller than the account could
// otherwise afford.
type Options struct {
	Ceiling pricing.Bounds
	// BlockedBy is the book whose hold is holding the scale down, or "".
	BlockedBy string
}

// Bounds computes the run-ceiling scale for a book, and what is holding it down.
func (s *Service) Bounds(ctx context.Context, userID, bookID string) (Options, error) {
	book, err := s.Store.ReadBookForRun(ctx, userID, bookID)
	if err != nil {
		return Options{}, err
	}
	acct, err := s.Store.ReadAccount(ctx, userID)
	if err != nil {
		return Options{}, err
	}
	held, heldAmount, err := s.Store.CreditHeldBy(ctx, userID, bookID)
	if err != nil {
		return Options{}, err
	}
	// Balance AS IT IS. A hold is a debit when it is taken, so the balance already excludes the holds
	// open against it and subtracting Reserved a second time would halve the scale (D39.115 §2a).
	scale := s.Pricing.Scale(acct.Balance, book.ChaptersLeft)
	if !shortensTheScale(s.Pricing, acct.Balance, heldAmount, book.ChaptersLeft) {
		// `blocked` answers "why is the scale SHORTER than this account could afford" (canon
		// §RunOptions), so a hold that costs this book nothing — the book is simply this short — must
		// not name another book the user would then go and stop for nothing.
		held = ""
	}
	return Options{Ceiling: scale, BlockedBy: held}, nil
}

// shortensTheScale answers whether giving the held credit back would make the scale longer.
func shortensTheScale(p pricing.Model, balance, held money.MicroUSD, chaptersLeft int) bool {
	return p.Scale(balance+held, chaptersLeft).Max > p.Scale(balance, chaptersLeft).Max
}

// StartRequest is one accepted call of POST /books/{id}/runs.
type StartRequest struct {
	UserID          string
	BookID          string
	VerifyBank      bool
	CeilingChapters int
	// RePass buys the re-pass instead of chapters (P10, D39.165 §3): legal only when the options
	// announced work (the bank moved and touched paid units), it admits a zero-chapter run whose
	// hold is the materialized projection and whose bar walks the book (the re-pass form). The
	// wire's carrier is the contract half's `re_pass` member; CeilingChapters is ignored with it.
	RePass bool
}

// Start admits a run.
//
// The ceiling is re-judged here and not trusted from the client: run-options is a read, the bounds
// move, and the number that decides how much money is reserved cannot be one the caller chose
// unilaterally.
func (s *Service) Start(ctx context.Context, in StartRequest) (pgstore.Run, error) {
	// Refused BEFORE the money moves: a run the engine cannot be given a ceiling for would spend
	// under the book's own limit instead of the account's, and that is not a difference to discover
	// after the hold (see runner.CeilingTemplate).
	if err := s.runnable(); err != nil {
		return pgstore.Run{}, err
	}
	// The book's own serialization (bank.go): a run admitted while the correction door is mid-verb
	// on this book would spawn a `translate` straight into the verb's flock — a wasted attempt. The
	// door's own budget bounds each queued call, and the caller's context bounds THIS wait.
	unlock, err := s.lockBook(ctx, in.BookID)
	if err != nil {
		return pgstore.Run{}, err
	}
	defer unlock()
	book, err := s.Store.ReadBookForRun(ctx, in.UserID, in.BookID)
	if err != nil {
		return pgstore.Run{}, err
	}
	if !readyToTranslate(book.Status) {
		// A book whose intake has not finished has no chapter tree to translate and, while it is
		// `uploading`, half a file on disk. Refused here rather than discovered by the engine inside a
		// transient unit, where the only trace would be a marker reading "exit-code 1" — and after the
		// account's money had already been held for it.
		return pgstore.Run{}, fmt.Errorf("%w: it is %s", ErrBookNotReady, book.Status)
	}
	if book.HasLiveRun {
		return pgstore.Run{}, pgstore.ErrRunInFlight
	}
	acct, err := s.Store.ReadAccount(ctx, in.UserID)
	if err != nil {
		return pgstore.Run{}, err
	}
	// The re-pass consents, decided HERE — under the book lock, once — and stored on the run row
	// so every spawn of every attempt renders the same argv (P10 §3.1). A run admitted over a moved
	// bank needs both engine flags: without --resnapshot the snapshot guard stops it loudly AFTER
	// the hold, and without a capped --accept-rebill the consent gate does the same on any
	// correction worth more than the ~half-cent threshold. The cap is FUNDED: the run's own hold —
	// «re-pay no more than this run may spend at all» — because a projection to cap against does
	// not exist at admission time (the engine's status is blind to a correction until the next
	// translate folds the bank in; errata 28.08-к) and an unfunded cap let a run burn its whole
	// budget on re-billing (adversarial K7). Never the bare blanket form.
	resnapshot, consent := false, money.MicroUSD(0)
	if book.BankMoved {
		resnapshot = true
		consent = s.Pricing.Ceiling(in.CeilingChapters)
	}
	if in.RePass {
		// The re-pass purchase (P10 §3.2, the deferred-projection form): «a re-pass costs up to
		// your hold». What it would actually cost is unknowable at admission (errata 28.08-к), so
		// the hold is the honest ceiling of the work bought — the whole book's chapter price, of
		// which untouched units come back at $0 and the difference is released on settlement. The
		// consent cap equals the hold: funded by construction, strictly positive because a
		// translatable book has chapters.
		if !book.BankMoved {
			return pgstore.Run{}, ErrRePassUnavailable
		}
		offset, err := journalSize(book.Workdir)
		if err != nil {
			return pgstore.Run{}, err
		}
		hold := s.Pricing.Ceiling(max(book.ChapterCount, 1))
		started, err := s.Store.StartRun(ctx, pgstore.StartRunInput{
			UserID:          in.UserID,
			BookID:          in.BookID,
			CeilingChapters: 0,
			Ceiling:         hold,
			Now:             s.now(),
			Resnapshot:      true,
			AcceptRebill:    hold,
		}, offset, s.enqueue)
		if err != nil {
			return pgstore.Run{}, err
		}
		s.log().InfoContext(ctx, "re-pass admitted", "run", started.ID)
		return started.Run, nil
	}
	bounds := s.Pricing.Scale(acct.Balance, book.ChaptersLeft)
	if in.CeilingChapters < bounds.Min || in.CeilingChapters > bounds.Max {
		// WHY the scale does not fit decides what the client can offer next: bounds that moved are
		// waited out, while another book's hold is somewhere the user can go and act.
		// …and the hold is the cause only if the request WOULD have fit without it.
		if held, amount, err := s.Store.CreditHeldBy(ctx, in.UserID, in.BookID); err == nil && held != "" &&
			in.CeilingChapters >= bounds.Min &&
			in.CeilingChapters <= s.Pricing.Scale(acct.Balance+amount, book.ChaptersLeft).Max {
			return pgstore.Run{}, &CreditHeldError{BookID: held}
		}
		return pgstore.Run{}, fmt.Errorf("%w: %d is not within %d..%d", ErrCeilingOutOfBounds,
			in.CeilingChapters, bounds.Min, bounds.Max)
	}
	offset, err := journalSize(book.Workdir)
	if err != nil {
		return pgstore.Run{}, err
	}
	started, err := s.Store.StartRun(ctx, pgstore.StartRunInput{
		UserID:          in.UserID,
		BookID:          in.BookID,
		VerifyBank:      in.VerifyBank,
		CeilingChapters: in.CeilingChapters,
		Ceiling:         s.Pricing.Ceiling(in.CeilingChapters),
		Now:             s.now(),
		Resnapshot:      resnapshot,
		AcceptRebill:    consent,
	}, offset, s.enqueue)
	if err != nil {
		return pgstore.Run{}, err
	}
	// Neither the book id nor the money: an INFO line names the run and nothing that identifies a
	// user's library or the size of their wallet (D39.84, PD-99).
	s.log().InfoContext(ctx, "run admitted", "run", started.ID, "ceiling_chapters", in.CeilingChapters)
	return started.Run, nil
}

// readyToTranslate is the half of the book vocabulary a run may start from: everything a finished
// intake can leave behind, plus every state a previous run can leave a parsed book in.
//
// Written as an allowlist and not as "not uploading, not parsing, not rejected": a status added
// later is one this function has never thought about, and refusing it is the safe direction — the
// user gets a 409 they can report, rather than an engine started against a book in a state nobody
// designed for.
func readyToTranslate(status string) bool {
	switch status {
	case "not_started", "translating", "awaiting_bank", "ready", "paused", "stopped", "failed":
		return true
	}
	return false
}

func (s *Service) enqueue(ctx context.Context, tx pgstore.Tx, runID string) error {
	if s.Queue == nil {
		return nil // no queue configured: the reconciler still picks the run up on its next sweep
	}
	return s.Queue.EnqueueRun(ctx, tx, runID)
}

// runnable is what this DEPLOYMENT cannot do, as opposed to what this run cannot do. Both refusals
// are of the run and not of the service: an instance whose runner is misconfigured still serves
// every read, and taking the library down with it would turn a partial outage into a total one
// during exactly the operation that causes it, a binary upgrade.
func (s *Service) runnable() error {
	if len(s.Cfg.MarkerArgv) == 0 {
		return ErrRunnerIncomplete
	}
	if len(s.Cfg.Ceiling) == 0 {
		return runner.ErrCeilingNotWired
	}
	return nil
}

// ceilingFor renders the engine's ceiling argument for an amount.
func (s *Service) ceilingFor(amount money.MicroUSD) ([]string, error) {
	return s.Cfg.Ceiling.Args(amount)
}