// Package runs owns the life of a translation run: admitting one, spawning its transient systemd // unit, tailing what it writes, and deciding what happened to it afterwards. // // The shape follows from D39.106: the engine is NOT a child of this service, so no component here // waits on a process. Admission (the queue) only hands out permission to start; everything after // that is a RECONCILER that reads the world — Postgres, the book's journal, the exit marker — and // moves the read model to match. That is what makes a run survive a deploy: the process that // started it and the process that finishes it are usually not the same process. package runs import ( "context" "errors" "fmt" "log/slog" "sync" "time" "textmachine/platform/internal/ingest" "textmachine/platform/internal/money" "textmachine/platform/internal/pgstore" "textmachine/platform/internal/pricing" "textmachine/platform/internal/runner" ) // UnitRunner is the transient-unit half of the seam. An interface so the reconciler's decisions can // be pinned without systemd, and so a test that DOES want systemd is obviously a different test. type UnitRunner interface { Start(ctx context.Context, s runner.Spec) error Stop(ctx context.Context, unit string) error Alive(ctx context.Context, unit string) (bool, error) } // EngineStatus is the ratified repair channel: `tmctl status --json` against a book. type EngineStatus interface { Status(ctx context.Context, binary, workdir string) (ingest.StatusReport, error) } // BankApplier is the correction channel: `tmctl bank-apply` against a book (D39.156 pack 2в). An // interface for the same reason EngineStatus is one: the door's decisions are pinned without an // engine binary, and a test that wants the real verb is visibly a different test. type BankApplier interface { BankApply(ctx context.Context, binary, workdir, decisionsPath string, preview bool) (runner.BankApplyOutcome, error) } // Config is everything the runner needs that an operator chooses. type Config struct { // StateDir is where exit markers live. Platform state, deliberately NOT the book's directory: // the engine owns that one and the platform does not write into it (D39.110). StateDir string // EngineBinary is the VERSIONED path of tmctl (unified backlog row 139). An attempt is pinned to // the path it was started with, so a deploy of the engine mid-run cannot change what a resumed // attempt executes. EngineBinary string // MarkerArgv is the command systemd runs as ExecStopPost to write the exit marker. MarkerArgv []string // Ceiling is how the run's ceiling reaches the engine, as the argv template of the landed flag // (row 145). Empty means the configured override rendered nothing — whitespace, say, since an // unset variable takes the default — and then starting a run is refused rather than started under // the book's own limit. Ceiling runner.CeilingTemplate // KeysFile is the deployment's provider-key file, passed to `translate` as `--keys-file` // (row 211). The path is an engine argument: keys are never put into the unit's environment and // never pass through this process. Empty passes no flag. KeysFile string // MemoryMax and TasksMax bound ONE run's cgroup — the answer to PD-13. MemoryMax string TasksMax int // AllowEngineVersionChange lets a RESUME run a different engine build than the one its run // started with. Off by default and it has to be turned on deliberately (unified backlog row 139): // the engine is deployed more often than a translation finishes, so the quiet behaviour would be // for a resumed run to continue under a program nobody chose for it. AllowEngineVersionChange bool // RunBudget is what ONE run may cost a pass of the sweep; zero takes the default. It bounds the // starvation the pass itself cannot: the list is ordered the same way every time, so a handful of // runs whose engine hangs used to mean the tail was never reached (register row PD-169). RunBudget time.Duration // ResyncEvery is how often a live run is reconciled from `tmctl status --json`. It is a SLOW // poll on purpose: every call re-ingests and re-chunks the source (~1.4-1.5 s of CPU on a 23 MB // book, unified backlog row 100). Since the emitter landed (D39.131) it is the REPAIR channel and // not the source of progress — see maybeResync for which runs still reach it. ResyncEvery time.Duration } // Service is the run lifecycle. type Service struct { Store *pgstore.Store Runner UnitRunner Engine EngineStatus Bank BankApplier Pricing pricing.Model Queue Enqueuer Cfg Config Log *slog.Logger // Now is injectable so the reconciler's clocks are testable without sleeping. Now func() time.Time // resynced remembers when each run was last reconciled from status. In memory on purpose: it is // a rate limit, not a fact — losing it on restart costs one extra status call. resynced map[string]time.Time // books serializes one book's admissions, resumes and corrections against each other — the // per-book rule of the correction door (see bank.go, lockBook). In memory on purpose: the // sections it guards live inside one process's calls. booksMu sync.Mutex books map[string]*bookLock } // Enqueuer hands a run to the queue. // // Two methods because the two callers need different atomicity, not different queues: an ADMISSION // writes the run, its hold and its queue entry in one transaction (a job for a run that does not // exist, or a hold with no job, are both ways to lose money or work), while a RESUME re-opens a run // that already exists and only then asks for a worker. A lost entry costs a sweep interval either // way — the reconciler is the backstop for both. type Enqueuer interface { EnqueueRun(ctx context.Context, tx pgstore.Tx, runID string) error EnqueueRunNow(ctx context.Context, runID string) error } func (s *Service) now() time.Time { if s.Now != nil { return s.Now() } return time.Now() } func (s *Service) log() *slog.Logger { if s.Log == nil { return slog.New(slog.DiscardHandler) } return s.Log } // ErrCeilingOutOfBounds is a requested ceiling the account or the book cannot carry. It is the // contract's 409 on a run start: the bounds are read by GET run-options and may move between that // read and this call, because a hold taken for another book lowers what is left. var ErrCeilingOutOfBounds = errors.New("runs: the requested ceiling is outside the bounds") // ErrRePassUnavailable — a re-pass was asked for on a book that has nothing to re-pass: the bank // has not moved since the last run, or its move touched no already-paid unit. The remedy is the // options read, which is what says whether a re-pass exists to buy (P10 §3.2). var ErrRePassUnavailable = errors.New("runs: the book has nothing to re-pass") // ErrRunnerIncomplete is a DEPLOYMENT that cannot start runs — the exit-marker command is missing, // so a unit that ended would have no way to say so and its hold would stay reserved forever. // // It is a refusal of the run and NOT of the service: an instance whose runner is misconfigured still // serves every read, and taking the library down with it would turn a partial outage into a total // one during exactly the operation that causes it, a binary upgrade. var ErrRunnerIncomplete = errors.New("runs: this deployment cannot record how a run ends") // ErrBookNotReady is a run asked for on a book whose intake has not finished — one still arriving, // still being cut, or rejected. The contract's 409: the book exists and the client can see it, and // what cannot happen yet is a translation of it. var ErrBookNotReady = errors.New("runs: the book is not ready to be translated") // ErrNotStoppable is a stop asked for on a run that is already over. var ErrNotStoppable = errors.New("runs: the run is not live") // ErrNotResumable is a resume that would not move the run: a run that is already going, or one that // finished. A bank stop is NOT among them — signing is one act over the whole bank and `resume` // lifts that stop with the decisions as they stand (D39.144). var ErrNotResumable = errors.New("runs: the run cannot be continued") // ErrCeilingReached is a resume of a run that stopped at a LIMIT — the chapters it bought, or the // credit behind them. It is ErrNotResumable with the one cause the contract enumerates, because the // remedy differs and is the thing a client has to be told: a limit travels with the START of a run, // so a NEW run with a larger one continues the work and this call never will. var ErrCeilingReached = fmt.Errorf("%w: it stopped at a limit", ErrNotResumable) // ErrCreditUnavailable is a resume of a run that has room LEFT in its limit, over an account that // cannot cover the rest of it. Told apart from ErrCeilingReached because the remedies are opposite: // this one clears when the account is topped up and the SAME run then continues, while that one is // finished with and needs a new run. Answering either in place of the other sends the user to do the // wrong thing (canon §resumeRun). var ErrCreditUnavailable = fmt.Errorf("%w: the account cannot cover the rest of it", ErrNotResumable) // CreditHeldError is ErrCeilingOutOfBounds with the book that caused it: another book of the account // has a run holding the credit, and naming it is what lets a client offer the user somewhere to go. type CreditHeldError struct{ BookID string } func (e *CreditHeldError) Error() string { return "runs: the account's credit is held by another book" } func (e *CreditHeldError) Is(target error) bool { return target == ErrCeilingOutOfBounds } // Options is what a run may be started with: the scale, and why it is smaller than the account could // otherwise afford. type Options struct { Ceiling pricing.Bounds // BlockedBy is the book whose hold is holding the scale down, or "". BlockedBy string } // Bounds computes the run-ceiling scale for a book, and what is holding it down. func (s *Service) Bounds(ctx context.Context, userID, bookID string) (Options, error) { book, err := s.Store.ReadBookForRun(ctx, userID, bookID) if err != nil { return Options{}, err } acct, err := s.Store.ReadAccount(ctx, userID) if err != nil { return Options{}, err } held, heldAmount, err := s.Store.CreditHeldBy(ctx, userID, bookID) if err != nil { return Options{}, err } // Balance AS IT IS. A hold is a debit when it is taken, so the balance already excludes the holds // open against it and subtracting Reserved a second time would halve the scale (D39.115 §2a). scale := s.Pricing.Scale(acct.Balance, book.ChaptersLeft) if !shortensTheScale(s.Pricing, acct.Balance, heldAmount, book.ChaptersLeft) { // `blocked` answers "why is the scale SHORTER than this account could afford" (canon // §RunOptions), so a hold that costs this book nothing — the book is simply this short — must // not name another book the user would then go and stop for nothing. held = "" } return Options{Ceiling: scale, BlockedBy: held}, nil } // shortensTheScale answers whether giving the held credit back would make the scale longer. func shortensTheScale(p pricing.Model, balance, held money.MicroUSD, chaptersLeft int) bool { return p.Scale(balance+held, chaptersLeft).Max > p.Scale(balance, chaptersLeft).Max } // StartRequest is one accepted call of POST /books/{id}/runs. type StartRequest struct { UserID string BookID string VerifyBank bool CeilingChapters int // RePass buys the re-pass instead of chapters (P10, D39.165 §3): legal only when the options // announced work (the bank moved and touched paid units), it admits a zero-chapter run whose // hold is the materialized projection and whose bar walks the book (the re-pass form). The // wire's carrier is the contract half's `re_pass` member; CeilingChapters is ignored with it. RePass bool } // Start admits a run. // // The ceiling is re-judged here and not trusted from the client: run-options is a read, the bounds // move, and the number that decides how much money is reserved cannot be one the caller chose // unilaterally. func (s *Service) Start(ctx context.Context, in StartRequest) (pgstore.Run, error) { // Refused BEFORE the money moves: a run the engine cannot be given a ceiling for would spend // under the book's own limit instead of the account's, and that is not a difference to discover // after the hold (see runner.CeilingTemplate). if err := s.runnable(); err != nil { return pgstore.Run{}, err } // The book's own serialization (bank.go): a run admitted while the correction door is mid-verb // on this book would spawn a `translate` straight into the verb's flock — a wasted attempt. The // door's own budget bounds each queued call, and the caller's context bounds THIS wait. unlock, err := s.lockBook(ctx, in.BookID) if err != nil { return pgstore.Run{}, err } defer unlock() book, err := s.Store.ReadBookForRun(ctx, in.UserID, in.BookID) if err != nil { return pgstore.Run{}, err } if !readyToTranslate(book.Status) { // A book whose intake has not finished has no chapter tree to translate and, while it is // `uploading`, half a file on disk. Refused here rather than discovered by the engine inside a // transient unit, where the only trace would be a marker reading "exit-code 1" — and after the // account's money had already been held for it. return pgstore.Run{}, fmt.Errorf("%w: it is %s", ErrBookNotReady, book.Status) } if book.HasLiveRun { return pgstore.Run{}, pgstore.ErrRunInFlight } acct, err := s.Store.ReadAccount(ctx, in.UserID) if err != nil { return pgstore.Run{}, err } // The re-pass consents, decided HERE — under the book lock, once — and stored on the run row // so every spawn of every attempt renders the same argv (P10 §3.1). A run admitted over a moved // bank needs both engine flags: without --resnapshot the snapshot guard stops it loudly AFTER // the hold, and without a capped --accept-rebill the consent gate does the same on any // correction worth more than the ~half-cent threshold. The cap is FUNDED: the run's own hold — // «re-pay no more than this run may spend at all» — because a projection to cap against does // not exist at admission time (the engine's status is blind to a correction until the next // translate folds the bank in; errata 28.08-к) and an unfunded cap let a run burn its whole // budget on re-billing (adversarial K7). Never the bare blanket form. resnapshot, consent := false, money.MicroUSD(0) if book.BankMoved { resnapshot = true consent = s.Pricing.Ceiling(in.CeilingChapters) } if in.RePass { // The re-pass purchase (P10 §3.2, the deferred-projection form): «a re-pass costs up to // your hold». What it would actually cost is unknowable at admission (errata 28.08-к), so // the hold is the honest ceiling of the work bought — the whole book's chapter price, of // which untouched units come back at $0 and the difference is released on settlement. The // consent cap equals the hold: funded by construction, strictly positive because a // translatable book has chapters. if !book.BankMoved { return pgstore.Run{}, ErrRePassUnavailable } offset, err := journalSize(book.Workdir) if err != nil { return pgstore.Run{}, err } hold := s.Pricing.Ceiling(max(book.ChapterCount, 1)) started, err := s.Store.StartRun(ctx, pgstore.StartRunInput{ UserID: in.UserID, BookID: in.BookID, CeilingChapters: 0, Ceiling: hold, Now: s.now(), Resnapshot: true, AcceptRebill: hold, }, offset, s.enqueue) if err != nil { return pgstore.Run{}, err } s.log().InfoContext(ctx, "re-pass admitted", "run", started.ID) return started.Run, nil } bounds := s.Pricing.Scale(acct.Balance, book.ChaptersLeft) if in.CeilingChapters < bounds.Min || in.CeilingChapters > bounds.Max { // WHY the scale does not fit decides what the client can offer next: bounds that moved are // waited out, while another book's hold is somewhere the user can go and act. // …and the hold is the cause only if the request WOULD have fit without it. if held, amount, err := s.Store.CreditHeldBy(ctx, in.UserID, in.BookID); err == nil && held != "" && in.CeilingChapters >= bounds.Min && in.CeilingChapters <= s.Pricing.Scale(acct.Balance+amount, book.ChaptersLeft).Max { return pgstore.Run{}, &CreditHeldError{BookID: held} } return pgstore.Run{}, fmt.Errorf("%w: %d is not within %d..%d", ErrCeilingOutOfBounds, in.CeilingChapters, bounds.Min, bounds.Max) } offset, err := journalSize(book.Workdir) if err != nil { return pgstore.Run{}, err } started, err := s.Store.StartRun(ctx, pgstore.StartRunInput{ UserID: in.UserID, BookID: in.BookID, VerifyBank: in.VerifyBank, CeilingChapters: in.CeilingChapters, Ceiling: s.Pricing.Ceiling(in.CeilingChapters), Now: s.now(), Resnapshot: resnapshot, AcceptRebill: consent, }, offset, s.enqueue) if err != nil { return pgstore.Run{}, err } // Neither the book id nor the money: an INFO line names the run and nothing that identifies a // user's library or the size of their wallet (D39.84, PD-99). s.log().InfoContext(ctx, "run admitted", "run", started.ID, "ceiling_chapters", in.CeilingChapters) return started.Run, nil } // readyToTranslate is the half of the book vocabulary a run may start from: everything a finished // intake can leave behind, plus every state a previous run can leave a parsed book in. // // Written as an allowlist and not as "not uploading, not parsing, not rejected": a status added // later is one this function has never thought about, and refusing it is the safe direction — the // user gets a 409 they can report, rather than an engine started against a book in a state nobody // designed for. func readyToTranslate(status string) bool { switch status { case "not_started", "translating", "awaiting_bank", "ready", "paused", "stopped", "failed": return true } return false } func (s *Service) enqueue(ctx context.Context, tx pgstore.Tx, runID string) error { if s.Queue == nil { return nil // no queue configured: the reconciler still picks the run up on its next sweep } return s.Queue.EnqueueRun(ctx, tx, runID) } // runnable is what this DEPLOYMENT cannot do, as opposed to what this run cannot do. Both refusals // are of the run and not of the service: an instance whose runner is misconfigured still serves // every read, and taking the library down with it would turn a partial outage into a total one // during exactly the operation that causes it, a binary upgrade. func (s *Service) runnable() error { if len(s.Cfg.MarkerArgv) == 0 { return ErrRunnerIncomplete } if len(s.Cfg.Ceiling) == 0 { return runner.ErrCeilingNotWired } return nil } // ceilingFor renders the engine's ceiling argument for an amount. func (s *Service) ceilingFor(amount money.MicroUSD) ([]string, error) { return s.Cfg.Ceiling.Args(amount) }