textmachine/platform/internal/runs/runs.go

// Package runs owns the life of a translation run: admitting one, spawning its transient systemd
// unit, tailing what it writes, and deciding what happened to it afterwards.
//
// The shape follows from D39.106: the engine is NOT a child of this service, so no component here
// waits on a process. Admission (the queue) only hands out permission to start; everything after
// that is a RECONCILER that reads the world — Postgres, the book's journal, the exit marker — and
// moves the read model to match. That is what makes a run survive a deploy: the process that
// started it and the process that finishes it are usually not the same process.
package runs

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"time"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/money"
	"textmachine/platform/internal/pgstore"
	"textmachine/platform/internal/pricing"
	"textmachine/platform/internal/runner"
)

// UnitRunner is the transient-unit half of the seam. An interface so the reconciler's decisions can
// be pinned without systemd, and so a test that DOES want systemd is obviously a different test.
type UnitRunner interface {
	Start(ctx context.Context, s runner.Spec) error
	Stop(ctx context.Context, unit string) error
	Alive(ctx context.Context, unit string) (bool, error)
}

// EngineStatus is the ratified repair channel: `tmctl status --json` against a book.
type EngineStatus interface {
	Status(ctx context.Context, binary, workdir string) (ingest.StatusReport, error)
}

// Config is everything the runner needs that an operator chooses.
type Config struct {
	// StateDir is where exit markers live. Platform state, deliberately NOT the book's directory:
	// the engine owns that one and the platform does not write into it (D39.110).
	StateDir string
	// EngineBinary is the VERSIONED path of tmctl (unified backlog row 139). An attempt is pinned to
	// the path it was started with, so a deploy of the engine mid-run cannot change what a resumed
	// attempt executes.
	EngineBinary string
	// MarkerArgv is the command systemd runs as ExecStopPost to write the exit marker.
	MarkerArgv []string
	// Ceiling is how the run's ceiling reaches the engine, as the argv template of the landed flag
	// (row 145). Empty means the configured override rendered nothing — whitespace, say, since an
	// unset variable takes the default — and then starting a run is refused rather than started under
	// the book's own limit.
	Ceiling runner.CeilingTemplate
	// MemoryMax and TasksMax bound ONE run's cgroup — the answer to PD-13.
	MemoryMax string
	TasksMax  int
	// AllowEngineVersionChange lets a RESUME run a different engine build than the one its run
	// started with. Off by default and it has to be turned on deliberately (unified backlog row 139):
	// the engine is deployed more often than a translation finishes, so the quiet behaviour would be
	// for a resumed run to continue under a program nobody chose for it.
	AllowEngineVersionChange bool
	// RunBudget is what ONE run may cost a pass of the sweep; zero takes the default. It bounds the
	// starvation the pass itself cannot: the list is ordered the same way every time, so a handful of
	// runs whose engine hangs used to mean the tail was never reached (register row PD-169).
	RunBudget time.Duration
	// ResyncEvery is how often a live run is reconciled from `tmctl status --json`. It is a SLOW
	// poll on purpose: every call re-ingests and re-chunks the source (~1.4-1.5 s of CPU on a 23 MB
	// book, unified backlog row 100). Until the event emitter exists (row 103) it is the only thing
	// that moves a run's progress at all.
	ResyncEvery time.Duration
}

// Service is the run lifecycle.
type Service struct {
	Store   *pgstore.Store
	Runner  UnitRunner
	Engine  EngineStatus
	Pricing pricing.Model
	Queue   Enqueuer
	Cfg     Config
	Log     *slog.Logger
	// Now is injectable so the reconciler's clocks are testable without sleeping.
	Now func() time.Time

	// resynced remembers when each run was last reconciled from status. In memory on purpose: it is
	// a rate limit, not a fact — losing it on restart costs one extra status call.
	resynced map[string]time.Time
}

// Enqueuer hands a run to the queue.
//
// Two methods because the two callers need different atomicity, not different queues: an ADMISSION
// writes the run, its hold and its queue entry in one transaction (a job for a run that does not
// exist, or a hold with no job, are both ways to lose money or work), while a RESUME re-opens a run
// that already exists and only then asks for a worker. A lost entry costs a sweep interval either
// way — the reconciler is the backstop for both.
type Enqueuer interface {
	EnqueueRun(ctx context.Context, tx pgstore.Tx, runID string) error
	EnqueueRunNow(ctx context.Context, runID string) error
}

func (s *Service) now() time.Time {
	if s.Now != nil {
		return s.Now()
	}
	return time.Now()
}

func (s *Service) log() *slog.Logger {
	if s.Log == nil {
		return slog.New(slog.DiscardHandler)
	}
	return s.Log
}

// ErrCeilingOutOfBounds is a requested ceiling the account or the book cannot carry. It is the
// contract's 409 on a run start: the bounds are read by GET run-options and may move between that
// read and this call, because a hold taken for another book lowers what is left.
var ErrCeilingOutOfBounds = errors.New("runs: the requested ceiling is outside the bounds")

// ErrRunnerIncomplete is a DEPLOYMENT that cannot start runs — the exit-marker command is missing,
// so a unit that ended would have no way to say so and its hold would stay reserved forever.
//
// It is a refusal of the run and NOT of the service: an instance whose runner is misconfigured still
// serves every read, and taking the library down with it would turn a partial outage into a total
// one during exactly the operation that causes it, a binary upgrade.
var ErrRunnerIncomplete = errors.New("runs: this deployment cannot record how a run ends")

// ErrBookNotReady is a run asked for on a book whose intake has not finished — one still arriving,
// still being cut, or rejected. The contract's 409: the book exists and the client can see it, and
// what cannot happen yet is a translation of it.
var ErrBookNotReady = errors.New("runs: the book is not ready to be translated")

// ErrNotStoppable is a stop asked for on a run that is already over.
var ErrNotStoppable = errors.New("runs: the run is not live")

// ErrNotResumable is a resume that would not move the run: a run that is already going, one that
// finished, or a bank stop whose set of decisions is not complete (contract §resumeRun, 409).
var ErrNotResumable = errors.New("runs: the run cannot be continued")

// Bounds computes the run-ceiling scale for a book.
func (s *Service) Bounds(ctx context.Context, userID, bookID string) (pricing.Bounds, error) {
	book, err := s.Store.ReadBookForRun(ctx, userID, bookID)
	if err != nil {
		return pricing.Bounds{}, err
	}
	acct, err := s.Store.ReadAccount(ctx, userID)
	if err != nil {
		return pricing.Bounds{}, err
	}
	// Balance AS IT IS. A hold is a debit when it is taken, so the balance already excludes the holds
	// open against it and subtracting Reserved a second time would halve the scale (D39.115 §2a).
	return s.Pricing.Scale(acct.Balance, book.ChaptersLeft), nil
}

// StartRequest is one accepted call of POST /books/{id}/runs.
type StartRequest struct {
	UserID          string
	BookID          string
	VerifyBank      bool
	CeilingChapters int
}

// Start admits a run.
//
// The ceiling is re-judged here and not trusted from the client: run-options is a read, the bounds
// move, and the number that decides how much money is reserved cannot be one the caller chose
// unilaterally.
func (s *Service) Start(ctx context.Context, in StartRequest) (pgstore.Run, error) {
	// Refused BEFORE the money moves: a run the engine cannot be given a ceiling for would spend
	// under the book's own limit instead of the account's, and that is not a difference to discover
	// after the hold (see runner.CeilingTemplate).
	if err := s.runnable(); err != nil {
		return pgstore.Run{}, err
	}
	book, err := s.Store.ReadBookForRun(ctx, in.UserID, in.BookID)
	if err != nil {
		return pgstore.Run{}, err
	}
	if !readyToTranslate(book.Status) {
		// A book whose intake has not finished has no chapter tree to translate and, while it is
		// `uploading`, half a file on disk. Refused here rather than discovered by the engine inside a
		// transient unit, where the only trace would be a marker reading "exit-code 1" — and after the
		// account's money had already been held for it.
		return pgstore.Run{}, fmt.Errorf("%w: it is %s", ErrBookNotReady, book.Status)
	}
	if book.HasLiveRun {
		return pgstore.Run{}, pgstore.ErrRunInFlight
	}
	acct, err := s.Store.ReadAccount(ctx, in.UserID)
	if err != nil {
		return pgstore.Run{}, err
	}
	bounds := s.Pricing.Scale(acct.Balance, book.ChaptersLeft)
	if in.CeilingChapters < bounds.Min || in.CeilingChapters > bounds.Max {
		return pgstore.Run{}, fmt.Errorf("%w: %d is not within %d..%d", ErrCeilingOutOfBounds,
			in.CeilingChapters, bounds.Min, bounds.Max)
	}
	offset, err := journalSize(book.Workdir)
	if err != nil {
		return pgstore.Run{}, err
	}
	started, err := s.Store.StartRun(ctx, pgstore.StartRunInput{
		UserID:          in.UserID,
		BookID:          in.BookID,
		VerifyBank:      in.VerifyBank,
		CeilingChapters: in.CeilingChapters,
		Ceiling:         s.Pricing.Ceiling(in.CeilingChapters),
		Now:             s.now(),
	}, offset, s.enqueue)
	if err != nil {
		return pgstore.Run{}, err
	}
	// Neither the book id nor the money: an INFO line names the run and nothing that identifies a
	// user's library or the size of their wallet (D39.84, PD-99).
	s.log().InfoContext(ctx, "run admitted", "run", started.ID, "ceiling_chapters", in.CeilingChapters)
	return started.Run, nil
}

// readyToTranslate is the half of the book vocabulary a run may start from: everything a finished
// intake can leave behind, plus every state a previous run can leave a parsed book in.
//
// Written as an allowlist and not as "not uploading, not parsing, not rejected": a status added
// later is one this function has never thought about, and refusing it is the safe direction — the
// user gets a 409 they can report, rather than an engine started against a book in a state nobody
// designed for.
func readyToTranslate(status string) bool {
	switch status {
	case "not_started", "translating", "awaiting_bank", "finalizing", "ready", "paused", "stopped", "failed":
		return true
	}
	return false
}

func (s *Service) enqueue(ctx context.Context, tx pgstore.Tx, runID string) error {
	if s.Queue == nil {
		return nil // no queue configured: the reconciler still picks the run up on its next sweep
	}
	return s.Queue.EnqueueRun(ctx, tx, runID)
}

// runnable is what this DEPLOYMENT cannot do, as opposed to what this run cannot do. Both refusals
// are of the run and not of the service: an instance whose runner is misconfigured still serves
// every read, and taking the library down with it would turn a partial outage into a total one
// during exactly the operation that causes it, a binary upgrade.
func (s *Service) runnable() error {
	if len(s.Cfg.MarkerArgv) == 0 {
		return ErrRunnerIncomplete
	}
	if len(s.Cfg.Ceiling) == 0 {
		return runner.ErrCeilingNotWired
	}
	return nil
}

// ceilingFor renders the engine's ceiling argument for an amount.
func (s *Service) ceilingFor(amount money.MicroUSD) ([]string, error) {
	return s.Cfg.Ceiling.Args(amount)
}