// Package runs owns the life of a translation run: admitting one, spawning its transient systemd // unit, tailing what it writes, and deciding what happened to it afterwards. // // The shape follows from D39.106: the engine is NOT a child of this service, so no component here // waits on a process. Admission (the queue) only hands out permission to start; everything after // that is a RECONCILER that reads the world — Postgres, the book's journal, the exit marker — and // moves the read model to match. That is what makes a run survive a deploy: the process that // started it and the process that finishes it are usually not the same process. package runs import ( "context" "errors" "fmt" "log/slog" "time" "textmachine/platform/internal/ingest" "textmachine/platform/internal/money" "textmachine/platform/internal/pgstore" "textmachine/platform/internal/pricing" "textmachine/platform/internal/runner" ) // UnitRunner is the transient-unit half of the seam. An interface so the reconciler's decisions can // be pinned without systemd, and so a test that DOES want systemd is obviously a different test. type UnitRunner interface { Start(ctx context.Context, s runner.Spec) error Stop(ctx context.Context, unit string) error Alive(ctx context.Context, unit string) (bool, error) } // EngineStatus is the ratified repair channel: `tmctl status --json` against a book. type EngineStatus interface { Status(ctx context.Context, binary, workdir string) (ingest.StatusReport, error) } // Config is everything the runner needs that an operator chooses. type Config struct { // StateDir is where exit markers live. Platform state, deliberately NOT the book's directory: // the engine owns that one and the platform does not write into it (D39.110). StateDir string // EngineBinary is the VERSIONED path of tmctl (unified backlog row 139). An attempt is pinned to // the path it was started with, so a deploy of the engine mid-run cannot change what a resumed // attempt executes. EngineBinary string // MarkerArgv is the command systemd runs as ExecStopPost to write the exit marker. MarkerArgv []string // Ceiling is how the run's ceiling reaches the engine, as the argv template of the landed flag // (row 145). Empty means the configured override rendered nothing — whitespace, say, since an // unset variable takes the default — and then starting a run is refused rather than started under // the book's own limit. Ceiling runner.CeilingTemplate // MemoryMax and TasksMax bound ONE run's cgroup — the answer to PD-13. MemoryMax string TasksMax int // AllowEngineVersionChange lets a RESUME run a different engine build than the one its run // started with. Off by default and it has to be turned on deliberately (unified backlog row 139): // the engine is deployed more often than a translation finishes, so the quiet behaviour would be // for a resumed run to continue under a program nobody chose for it. AllowEngineVersionChange bool // RunBudget is what ONE run may cost a pass of the sweep; zero takes the default. It bounds the // starvation the pass itself cannot: the list is ordered the same way every time, so a handful of // runs whose engine hangs used to mean the tail was never reached (register row PD-169). RunBudget time.Duration // ResyncEvery is how often a live run is reconciled from `tmctl status --json`. It is a SLOW // poll on purpose: every call re-ingests and re-chunks the source (~1.4-1.5 s of CPU on a 23 MB // book, unified backlog row 100). Until the event emitter exists (row 103) it is the only thing // that moves a run's progress at all. ResyncEvery time.Duration } // Service is the run lifecycle. type Service struct { Store *pgstore.Store Runner UnitRunner Engine EngineStatus Pricing pricing.Model Queue Enqueuer Cfg Config Log *slog.Logger // Now is injectable so the reconciler's clocks are testable without sleeping. Now func() time.Time // resynced remembers when each run was last reconciled from status. In memory on purpose: it is // a rate limit, not a fact — losing it on restart costs one extra status call. resynced map[string]time.Time } // Enqueuer hands a run to the queue. // // Two methods because the two callers need different atomicity, not different queues: an ADMISSION // writes the run, its hold and its queue entry in one transaction (a job for a run that does not // exist, or a hold with no job, are both ways to lose money or work), while a RESUME re-opens a run // that already exists and only then asks for a worker. A lost entry costs a sweep interval either // way — the reconciler is the backstop for both. type Enqueuer interface { EnqueueRun(ctx context.Context, tx pgstore.Tx, runID string) error EnqueueRunNow(ctx context.Context, runID string) error } func (s *Service) now() time.Time { if s.Now != nil { return s.Now() } return time.Now() } func (s *Service) log() *slog.Logger { if s.Log == nil { return slog.New(slog.DiscardHandler) } return s.Log } // ErrCeilingOutOfBounds is a requested ceiling the account or the book cannot carry. It is the // contract's 409 on a run start: the bounds are read by GET run-options and may move between that // read and this call, because a hold taken for another book lowers what is left. var ErrCeilingOutOfBounds = errors.New("runs: the requested ceiling is outside the bounds") // ErrRunnerIncomplete is a DEPLOYMENT that cannot start runs — the exit-marker command is missing, // so a unit that ended would have no way to say so and its hold would stay reserved forever. // // It is a refusal of the run and NOT of the service: an instance whose runner is misconfigured still // serves every read, and taking the library down with it would turn a partial outage into a total // one during exactly the operation that causes it, a binary upgrade. var ErrRunnerIncomplete = errors.New("runs: this deployment cannot record how a run ends") // ErrBookNotReady is a run asked for on a book whose intake has not finished — one still arriving, // still being cut, or rejected. The contract's 409: the book exists and the client can see it, and // what cannot happen yet is a translation of it. var ErrBookNotReady = errors.New("runs: the book is not ready to be translated") // ErrNotStoppable is a stop asked for on a run that is already over. var ErrNotStoppable = errors.New("runs: the run is not live") // ErrNotResumable is a resume that would not move the run: a run that is already going, one that // finished, or a bank stop whose set of decisions is not complete (contract §resumeRun, 409). var ErrNotResumable = errors.New("runs: the run cannot be continued") // Bounds computes the run-ceiling scale for a book. func (s *Service) Bounds(ctx context.Context, userID, bookID string) (pricing.Bounds, error) { book, err := s.Store.ReadBookForRun(ctx, userID, bookID) if err != nil { return pricing.Bounds{}, err } acct, err := s.Store.ReadAccount(ctx, userID) if err != nil { return pricing.Bounds{}, err } // Balance AS IT IS. A hold is a debit when it is taken, so the balance already excludes the holds // open against it and subtracting Reserved a second time would halve the scale (D39.115 §2a). return s.Pricing.Scale(acct.Balance, book.ChaptersLeft), nil } // StartRequest is one accepted call of POST /books/{id}/runs. type StartRequest struct { UserID string BookID string VerifyBank bool CeilingChapters int } // Start admits a run. // // The ceiling is re-judged here and not trusted from the client: run-options is a read, the bounds // move, and the number that decides how much money is reserved cannot be one the caller chose // unilaterally. func (s *Service) Start(ctx context.Context, in StartRequest) (pgstore.Run, error) { // Refused BEFORE the money moves: a run the engine cannot be given a ceiling for would spend // under the book's own limit instead of the account's, and that is not a difference to discover // after the hold (see runner.CeilingTemplate). if err := s.runnable(); err != nil { return pgstore.Run{}, err } book, err := s.Store.ReadBookForRun(ctx, in.UserID, in.BookID) if err != nil { return pgstore.Run{}, err } if !readyToTranslate(book.Status) { // A book whose intake has not finished has no chapter tree to translate and, while it is // `uploading`, half a file on disk. Refused here rather than discovered by the engine inside a // transient unit, where the only trace would be a marker reading "exit-code 1" — and after the // account's money had already been held for it. return pgstore.Run{}, fmt.Errorf("%w: it is %s", ErrBookNotReady, book.Status) } if book.HasLiveRun { return pgstore.Run{}, pgstore.ErrRunInFlight } acct, err := s.Store.ReadAccount(ctx, in.UserID) if err != nil { return pgstore.Run{}, err } bounds := s.Pricing.Scale(acct.Balance, book.ChaptersLeft) if in.CeilingChapters < bounds.Min || in.CeilingChapters > bounds.Max { return pgstore.Run{}, fmt.Errorf("%w: %d is not within %d..%d", ErrCeilingOutOfBounds, in.CeilingChapters, bounds.Min, bounds.Max) } offset, err := journalSize(book.Workdir) if err != nil { return pgstore.Run{}, err } started, err := s.Store.StartRun(ctx, pgstore.StartRunInput{ UserID: in.UserID, BookID: in.BookID, VerifyBank: in.VerifyBank, CeilingChapters: in.CeilingChapters, Ceiling: s.Pricing.Ceiling(in.CeilingChapters), Now: s.now(), }, offset, s.enqueue) if err != nil { return pgstore.Run{}, err } // Neither the book id nor the money: an INFO line names the run and nothing that identifies a // user's library or the size of their wallet (D39.84, PD-99). s.log().InfoContext(ctx, "run admitted", "run", started.ID, "ceiling_chapters", in.CeilingChapters) return started.Run, nil } // readyToTranslate is the half of the book vocabulary a run may start from: everything a finished // intake can leave behind, plus every state a previous run can leave a parsed book in. // // Written as an allowlist and not as "not uploading, not parsing, not rejected": a status added // later is one this function has never thought about, and refusing it is the safe direction — the // user gets a 409 they can report, rather than an engine started against a book in a state nobody // designed for. func readyToTranslate(status string) bool { switch status { case "not_started", "translating", "awaiting_bank", "finalizing", "ready", "paused", "stopped", "failed": return true } return false } func (s *Service) enqueue(ctx context.Context, tx pgstore.Tx, runID string) error { if s.Queue == nil { return nil // no queue configured: the reconciler still picks the run up on its next sweep } return s.Queue.EnqueueRun(ctx, tx, runID) } // runnable is what this DEPLOYMENT cannot do, as opposed to what this run cannot do. Both refusals // are of the run and not of the service: an instance whose runner is misconfigured still serves // every read, and taking the library down with it would turn a partial outage into a total one // during exactly the operation that causes it, a binary upgrade. func (s *Service) runnable() error { if len(s.Cfg.MarkerArgv) == 0 { return ErrRunnerIncomplete } if len(s.Cfg.Ceiling) == 0 { return runner.ErrCeilingNotWired } return nil } // ceilingFor renders the engine's ceiling argument for an amount. func (s *Service) ceilingFor(amount money.MicroUSD) ([]string, error) { return s.Cfg.Ceiling.Args(amount) }