// Package runs owns the life of a translation run: admitting one, spawning its transient systemd // unit, tailing what it writes, and deciding what happened to it afterwards. // // The shape follows from D39.106: the engine is NOT a child of this service, so no component here // waits on a process. Admission (the queue) only hands out permission to start; everything after // that is a RECONCILER that reads the world — Postgres, the book's journal, the exit marker — and // moves the read model to match. That is what makes a run survive a deploy: the process that // started it and the process that finishes it are usually not the same process. package runs import ( "context" "errors" "fmt" "io/fs" "log/slog" "os" "sync" "time" "textmachine/platform/internal/books" "textmachine/platform/internal/ingest" "textmachine/platform/internal/money" "textmachine/platform/internal/pgstore" "textmachine/platform/internal/pricing" "textmachine/platform/internal/runner" ) // UnitRunner is the transient-unit half of the seam. An interface so the reconciler's decisions can // be pinned without systemd, and so a test that DOES want systemd is obviously a different test. type UnitRunner interface { Start(ctx context.Context, s runner.Spec) error Stop(ctx context.Context, unit string) error Alive(ctx context.Context, unit string) (bool, error) } // EngineStatus is the ratified repair channel: `tmctl status --json` against a book. type EngineStatus interface { Status(ctx context.Context, binary, workdir string) (ingest.StatusReport, error) } // BankApplier is the correction channel: `tmctl bank-apply` against a book (D39.156 pack 2в). An // interface for the same reason EngineStatus is one: the door's decisions are pinned without an // engine binary, and a test that wants the real verb is visibly a different test. type BankApplier interface { BankApply(ctx context.Context, binary, workdir, decisionsPath string, preview bool) (runner.BankApplyOutcome, error) } // Config is everything the runner needs that an operator chooses. type Config struct { // StateDir is where exit markers live. Platform state, deliberately NOT the book's directory: // the engine owns that one and the platform does not write into it (D39.110). StateDir string // BooksDir is the root the intake creates book directories under — the same path that package // holds, and READ ONLY: this package never writes under it. It is here for one question, asked // at the door when a book's directory is missing (sourceThere): is this book's own directory // gone, or the whole volume? Empty answers "the storage is not there", which refuses the start // as a deployment fault — the safe direction on an instance that was never configured to hold // books, since it cannot tell the two apart and must not blame a book for it. BooksDir string // EngineBinary is the VERSIONED path of tmctl (unified backlog row 139). An attempt is pinned to // the path it was started with, so a deploy of the engine mid-run cannot change what a resumed // attempt executes. EngineBinary string // MarkerArgv is the command systemd runs as ExecStopPost to write the exit marker. MarkerArgv []string // Ceiling is how the run's ceiling reaches the engine, as the argv template of the landed flag // (row 145). Empty means the configured override rendered nothing — whitespace, say, since an // unset variable takes the default — and then starting a run is refused rather than started under // the book's own limit. Ceiling runner.CeilingTemplate // KeysFile is the deployment's provider-key file, passed to `translate` as `--keys-file` // (row 211). The path is an engine argument: keys are never put into the unit's environment and // never pass through this process. Empty passes no flag. KeysFile string // MemoryMax and TasksMax bound ONE run's cgroup — the answer to PD-13. MemoryMax string TasksMax int // AllowEngineVersionChange lets a RESUME run a different engine build than the one its run // started with. Off by default and it has to be turned on deliberately (unified backlog row 139): // the engine is deployed more often than a translation finishes, so the quiet behaviour would be // for a resumed run to continue under a program nobody chose for it. AllowEngineVersionChange bool // RunBudget is what ONE run may cost a pass of the sweep; zero takes the default. It bounds the // starvation the pass itself cannot: the list is ordered the same way every time, so a handful of // runs whose engine hangs used to mean the tail was never reached (register row PD-169). RunBudget time.Duration // MaxAttempts is how many attempts one run may have before this platform stops restarting it; // zero takes the default. It is the ONLY bound on the restart loop, and until it existed there // was none: an attempt that ends with no marker is restarted by the reconciler, so on a // deployment whose marker machinery is broken every restart ends the same way, and the loop is // money-NEUTRAL per turn — the old hold comes back whole and an equal one is taken — which is // exactly why no balance ever stops it (unified backlog row 398). Each turn still writes an // attempt row and three ledger rows, which is the rubbish D39.240 forbids leaving behind. // // ⚠ IT IS COUNTED IN `run_attempts.attempt_no`, NEVER IN A COUNTER INSIDE ONE ATTEMPT, and that // is the whole of why it works. `reconcile_failures` lives on the attempt and a restart opens a // NEW attempt row, so a cap keyed on it starts from zero every turn: the defect repeated, wearing // the look of a cure. // // ⚠ THE PRICE OF COUNTING attempt_no, named because it is what makes this a choice: a user's own // resume increments it too, so a run resumed more often than the cap allows loses its AUTOMATIC // restart — their resume itself still works, the cap binds the reconciler alone. At the shipped // default that population is empty, because a healthy run's count is a handful (the admission, a // stop and its resume, a bank stop and its resume) and the one pause that recurs daily is not // resumable at all (ErrCeilingReached). ⛔ THE CONDITION UNDER WHICH THAT STOPS HOLDING is // therefore exactly this: if resumes ever become frequent enough for a WORKING run to approach // the cap, the count has to move to automatic restarts only — which needs a column this // deliberately does not add. MaxAttempts int // ResyncEvery is how often a live run is reconciled from `tmctl status --json`. It is a SLOW // poll on purpose: every call re-ingests and re-chunks the source (~1.4-1.5 s of CPU on a 23 MB // book, unified backlog row 100). Since the emitter landed (D39.131) it is the REPAIR channel and // not the source of progress — see maybeResync for which runs still reach it. ResyncEvery time.Duration } // Service is the run lifecycle. type Service struct { Store *pgstore.Store Runner UnitRunner Engine EngineStatus Bank BankApplier Pricing pricing.Model Queue Enqueuer Cfg Config Log *slog.Logger // Now is injectable so the reconciler's clocks are testable without sleeping. Now func() time.Time // resynced remembers when each run was last reconciled from status. In memory on purpose: it is // a rate limit, not a fact — losing it on restart costs one extra status call. resynced map[string]time.Time // parkedSaid rate-limits the one line a PARKED attempt produces, keyed by attempt. The state it // reports can last a run's whole life and is not written down anywhere, so neither extreme is // right: said every pass it is the line an operator filters (this file's own rule, deferItem), // said once it is invisible to anyone who starts watching afterwards — and unlike a deferral, // nothing else carries the fact. Said at the crossing and then no more often than the repair // channel speaks. parkedSaid map[int64]time.Time // books serializes one book's admissions, resumes and corrections against each other — the // per-book rule of the correction door (see bank.go, lockBook). In memory on purpose: the // sections it guards live inside one process's calls. booksMu sync.Mutex books map[string]*bookLock } // Enqueuer hands a run to the queue. // // Two methods because the two callers need different atomicity, not different queues: an ADMISSION // writes the run, its hold and its queue entry in one transaction (a job for a run that does not // exist, or a hold with no job, are both ways to lose money or work), while a RESUME re-opens a run // that already exists and only then asks for a worker. A lost entry costs a sweep interval either // way — the reconciler is the backstop for both. type Enqueuer interface { EnqueueRun(ctx context.Context, tx pgstore.Tx, runID string) error EnqueueRunNow(ctx context.Context, runID string) error } func (s *Service) now() time.Time { if s.Now != nil { return s.Now() } return time.Now() } func (s *Service) log() *slog.Logger { if s.Log == nil { return slog.New(slog.DiscardHandler) } return s.Log } // ErrCeilingOutOfBounds is a requested ceiling the account or the book cannot carry. It is the // contract's 409 on a run start: the bounds are read by GET run-options and may move between that // read and this call, because a hold taken for another book lowers what is left. var ErrCeilingOutOfBounds = errors.New("runs: the requested ceiling is outside the bounds") // ErrRePassUnavailable — a re-pass was asked for on a book that has nothing to re-pass: the bank // has not moved since the last run, or its move touched no already-paid unit. The remedy is the // options read, which is what says whether a re-pass exists to buy (P10 §3.2). var ErrRePassUnavailable = errors.New("runs: the book has nothing to re-pass") // ErrRunnerIncomplete is a DEPLOYMENT that cannot start runs — the exit-marker command is missing, // so a unit that ended would have no way to say so and its hold would stay reserved forever. // // It is a refusal of the run and NOT of the service: an instance whose runner is misconfigured still // serves every read, and taking the library down with it would turn a partial outage into a total // one during exactly the operation that causes it, a binary upgrade. var ErrRunnerIncomplete = errors.New("runs: this deployment cannot record how a run ends") // ErrBookNotReady is a run asked for on a book whose intake has not finished — one still arriving, // still being cut, or rejected. The contract's 409: the book exists and the client can see it, and // what cannot happen yet is a translation of it. var ErrBookNotReady = errors.New("runs: the book is not ready to be translated") // ErrSourceGone is a book whose project directory is not on this deployment any more. // // ⚠ IT IS A KIND OF ErrBookNotReady rather than a code of its own, and the wrapping is what a client // reads: the contract's `book_not_ready` already means "the book is there and a translation of it // cannot happen", which is exactly this, and the second-level cause is where the difference belongs — // waiting clears the ordinary one and never clears this one. What it must not become is silence // after the money moved: a run over a directory that is not there spawns, fails, respawns and holds // the account's credit until a person comes (PD-162). var ErrSourceGone = fmt.Errorf("%w: its source directory is not on this deployment", ErrBookNotReady) // ErrStorageUnavailable is the book STORAGE ROOT missing under a book whose directory is missing // with it — an unmounted volume, or an instance pointed at a path that is not there yet. // // ⛔ TOLD APART FROM ErrSourceGone BECAUSE THE FAULT IS THE HOST'S, and reading the host's fault as // the book's is a defect this repository has already paid for once: in the intake it rejected every // book on the volume with a reason that blames the user's file and has no way back (PD-192). Here the // price would be smaller and the same in kind — a user told their book is unusable while the truth is // that an operator has a volume to mount — so it travels as the deployment's own 503, beside the // refusals of a runner that was never wired. var ErrStorageUnavailable = errors.New("runs: the book storage is not mounted on this deployment") // ErrNotStoppable is a stop asked for on a run that is already over. var ErrNotStoppable = errors.New("runs: the run is not live") // ErrNotResumable is a resume that would not move the run: a run that is already going, or one that // finished. A bank stop is NOT among them — signing is one act over the whole bank and `resume` // lifts that stop with the decisions as they stand (D39.144). var ErrNotResumable = errors.New("runs: the run cannot be continued") // ErrStopRequested is a resume of a run that is GOING and has already been asked to stop. // // It is the cut in the idempotent answer, and the cut is what keeps D39.240's hard stop hard: every // other live run answers this call with the run, because the run is continuing and that is what the // caller asked for — but a run under a stop is not continuing, and a 202 saying it is would read as // the stop having been undone. The remedy is the same call once the stop has landed. // // Its own sentinel rather than a bare ErrNotResumable because the client is told WHY: waiting works // here, which is the opposite of what `ceiling_reached` means. var ErrStopRequested = fmt.Errorf("%w: it has been asked to stop", ErrNotResumable) // ErrCeilingReached is a resume of a run that stopped at a LIMIT — the chapters it bought, or the // credit behind them. It is ErrNotResumable with the one cause the contract enumerates, because the // remedy differs and is the thing a client has to be told: a limit travels with the START of a run, // so a NEW run with a larger one continues the work and this call never will. var ErrCeilingReached = fmt.Errorf("%w: it stopped at a limit", ErrNotResumable) // ErrCreditUnavailable is a resume of a run that has room LEFT in its limit, over an account that // cannot cover the rest of it. Told apart from ErrCeilingReached because the remedies are opposite: // this one clears when the account is topped up and the SAME run then continues, while that one is // finished with and needs a new run. Answering either in place of the other sends the user to do the // wrong thing (canon §resumeRun). var ErrCreditUnavailable = fmt.Errorf("%w: the account cannot cover the rest of it", ErrNotResumable) // CreditHeldError is ErrCeilingOutOfBounds with the book that caused it: another book of the account // has a run holding the credit, and naming it is what lets a client offer the user somewhere to go. type CreditHeldError struct{ BookID string } func (e *CreditHeldError) Error() string { return "runs: the account's credit is held by another book" } func (e *CreditHeldError) Is(target error) bool { return target == ErrCeilingOutOfBounds } // ErrNotPriced is a book the engine has not priced, so no order over it can be honest. // // ⛔ A REFUSAL AND NOT A FALLBACK, and this is the line the per-chapter constant used to stand on. // $0.03 a chapter was measured 4.47× low (D39.179 §1) and it made the last chapters of every book // unbuyable at any balance (PD-440); replacing it with «guess again, more carefully» would keep the // shape of the defect. There is now one source for what a book costs — the engine's own projection, // published in `manifest --json` — and when that source has not spoken this platform says so. // // It clears by itself: the book's reading surface is owed the moment a manifest can be read, and the // materializer's drain writes the projection with the tree. An operator can force it with // `tmplatformctl book refresh --book `. var ErrNotPriced = errors.New("runs: this book has no price projection, so an order over it cannot be quoted") // Options is the whole order form for one book: what is left, what the balance covers, and the quote // for the default order — the whole book (D39.196 §1). type Options struct { pricing.Options // BlockedBy is the book whose hold is holding this one's order down, or "". BlockedBy string // Refusal is what the door would answer a start on this book right now, from the book's own // facts — nil when nothing about the book stands in the way. ADVISORY by construction: it is read // outside the book lock, so a run can begin or end between this read and the click, and the start // remains the only authority on whether a start may happen (canon §RunOptions). // // ⚠ What it is FOR is the systematic case rather than the one-second race: a run that has been // going for minutes was invisible to this form, so the form said `covers_all` over a book the // door would refuse and the user learned it by clicking (PD-455). Racing the door is not what // this closes and not what a client should expect of it. Refusal error // Structure is where the chapter cut came from, verbatim from the engine. ChapterOrders says // whether an order may be phrased in CHAPTERS against it at all — see ingest.ChapterOrdersOffered, // which trusts `detected` alone. Structure string ChapterOrders bool // SourceChars is what is left to translate, in characters — the unit an order takes when chapter // orders are not offered. SourceChars int64 } // Order computes the order form for a book: what it costs, what the balance covers, and what the // buyer is told before the click. func (s *Service) Order(ctx context.Context, userID, bookID string) (Options, error) { // ⚠ OWNERSHIP IS THIS QUERY'S OWN, not a separate check before it. `ReadBookForOrder` selects // `where b.id = $1 and b.owner_id = $2` and answers ErrNoBook for a stranger's book exactly as // for a missing one — telling them apart is what would let anyone enumerate other people's // libraries. A guard call ahead of it read the same row through a lateral join to say the same // thing, on the path a client polls. priced, err := s.Store.ReadBookForOrder(ctx, userID, bookID) if err != nil { return Options{}, err } acct, err := s.Store.ReadAccount(ctx, userID) if err != nil { return Options{}, err } if !priced.Priced { return Options{}, ErrNotPriced } // The door's own read of the book, and a SECOND read beside the one above rather than more // columns on it. The two answer differently shaped questions — one row of facts about the book // against one row per chapter — which is the reason PricedBook gives for being separate, and it // holds in this direction too: the form asks the door's question with the door's own query, so // there is one definition of "may a run start here" and not a copy of it phrased for the form. facts, err := s.Store.ReadBookForRun(ctx, userID, bookID) if err != nil { return Options{}, err } held, heldAmount, err := s.Store.CreditHeldBy(ctx, userID, bookID) if err != nil { return Options{}, err } book := priceBook(priced) // Balance AS IT IS. A hold is a debit when it is taken, so the balance already excludes the holds // open against it and subtracting Reserved a second time would halve the scale (D39.115 §2a). out := Options{ Options: s.Pricing.Order(book, acct.Balance), Structure: priced.Structure, ChapterOrders: ingest.Manifest{Structure: priced.Structure}.ChapterOrdersOffered(), Refusal: startable(facts), } for _, c := range book.Remaining { out.SourceChars += c.SourceChars } // `blocked` answers "why is the order SMALLER than this account could afford" (canon §RunOptions), // so a hold that costs this book nothing must not name another book the user would then go and // stop for nothing. if held != "" && s.Pricing.Affordable(book, acct.Balance+heldAmount) > out.AffordableChapters { out.BlockedBy = held } return out, nil } // priceBook maps the store's rows onto the arithmetic's own shape. It is a mapping and not a second // derivation: every figure in it was written by the materializer from one manifest. func priceBook(p pgstore.PricedBook) pricing.Book { book := pricing.Book{StepMax: p.StepMax, BookOnce: p.BookOnce, Remaining: make([]pricing.Chapter, 0, len(p.Remaining))} for _, c := range p.Remaining { book.Remaining = append(book.Remaining, pricing.Chapter{ ID: c.ID, Number: c.Number, Units: c.Units, Expected: c.Expected, SourceChars: c.SourceChars, }) } return book } // ErrBalanceCannotCarry is an order the account's money does not reach. // // ⛔ ITS OWN ERROR BECAUSE ITS REMEDY IS ITS OWN, and folding it into ErrCeilingOutOfBounds sent the // buyer to do the wrong thing. That error means «the options moved between the read and the call», // and the wire word for it, `bounds_moved`, promises a client that re-reading and retrying will // work. Here nothing moved and retrying is futile: what is missing is money, and the remedy is to // top up. This is the ONE refusal the whole order form exists to make honest — it is the answer to // «хватает или нет» arriving after the click — and it was travelling under somebody else's word. var ErrBalanceCannotCarry = errors.New("runs: the account's balance does not carry this order") // ErrChapterOrdersUnavailable is an order phrased in CHAPTERS over a book whose chapter cut is not // trustworthy enough to sell against — see ingest.ChapterOrdersOffered, which trusts `detected` // alone. The remedy is in the options read: such a book takes an order in CHARACTERS, or the whole // book, and the client is told the structure word so it can say why. var ErrChapterOrdersUnavailable = errors.New("runs: this book's chapter boundaries cannot be ordered against") // StartRequest is one accepted call of POST /books/{id}/runs. // // THE THREE KINDS OF ORDER (D39.196 §1, unified backlog row 279), and nothing chooses between them // by guessing: the whole book (both members nil, the default — «не тронута ни одна ручка ⇒ заказ = // вся книга»), a number of CHAPTERS, or a number of CHARACTERS. A re-pass is a fourth thing and buys // no volume at all. type StartRequest struct { UserID string BookID string VerifyBank bool // Chapters is how many of the REMAINING chapters to buy; nil is the whole book. It is refused on a // book whose cut is not trusted, because there «chapter 12» does not name what a buyer thinks. Chapters *int // Characters is how much SOURCE TEXT to buy, in runes. It resolves to the shortest prefix of units // that reaches it — units being the granularity the engine ships and stops at — and it is the only // partial order a book with one chapter can carry. Characters *int64 // RePass buys the re-pass instead of volume (P10, D39.165 §3): legal only when the options // announced work (the bank moved and touched paid units), it admits a run whose hold is the // materialized projection and whose bar walks the book. Chapters and Characters are ignored. RePass bool } // Start admits a run. // // The order is re-judged here and not trusted from the client: the options read is a READ, the // balance moves under it — another book's hold lowers what is left — and the number that decides how // much money is reserved cannot be one the caller chose unilaterally. func (s *Service) Start(ctx context.Context, in StartRequest) (pgstore.Run, error) { // Refused BEFORE the money moves: a run the engine cannot be given a ceiling for would spend // under the book's own limit instead of the account's, and that is not a difference to discover // after the hold (see runner.CeilingTemplate). if err := s.runnable(); err != nil { return pgstore.Run{}, err } // The book's own serialization (bank.go): a run admitted while the correction door is mid-verb // on this book would spawn a `translate` straight into the verb's flock — a wasted attempt. The // door's own budget bounds each queued call, and the caller's context bounds THIS wait. unlock, err := s.lockBook(ctx, in.BookID) if err != nil { return pgstore.Run{}, err } defer unlock() book, err := s.Store.ReadBookForRun(ctx, in.UserID, in.BookID) if err != nil { return pgstore.Run{}, err } // The book's own half of admission, AUTHORITATIVE here because it is read under the lock. The // order form asks the same predicate before the click, where the answer is advisory (Order). if err := startable(book); err != nil { return pgstore.Run{}, err } // And the half only the disk can answer, asked BEFORE the money: a run over a book whose // directory is not there can never move, and until it was asked here the platform took the hold // first and discovered the absence afterwards, in a place with no way back (PD-162). if err := s.sourceThere(book.Workdir); err != nil { return pgstore.Run{}, err } acct, err := s.Store.ReadAccount(ctx, in.UserID) if err != nil { return pgstore.Run{}, err } priced, err := s.Store.ReadBookForOrder(ctx, in.UserID, in.BookID) if err != nil { return pgstore.Run{}, err } if !priced.Priced { // No projection, no sale. See ErrNotPriced: the alternative is the constant that made the last // chapters of every book unbuyable, and a quieter version of it would be worse, not better. return pgstore.Run{}, ErrNotPriced } pb := priceBook(priced) // The re-pass consents, decided HERE — under the book lock, once — and stored on the run row so // every spawn of every attempt renders the same argv (P10 §3.1). // // ⛔ `--resnapshot` NOW RIDES EVERY CONTINUATION, not only a run over an edited bank, and that is // PD-422 closed rather than a widening. The auto-bank grows by MINING during an ordinary run; // mining moves the ENRICHED memory version; the enriched version is folded into the EDIT wave's // snapshot alone (backend/internal/pipeline/snapshot.go, snapshotIDForWave). So the SECOND // purchase of a mining book meets its own already-pinned edit jobs under a moved snapshot and the // drift guard stops the engine — `exit 1`, which this platform can only report as `failed`, after // the hold was taken and with nothing translated. The condition used to be the correction door's // flag alone, which no amount of mining ever sets. // // ⚠ Passing it where nothing moved costs NOTHING: the engine re-pins only where a snapshot // actually differs (stagerun.go compares per job), and the guard is per JOB, so it never concerns // a chapter this run does not touch. And the volume ceiling admits NEW book before re-made book // (volume.go, the unitFresh-then-unitRework pass), so a continuation spends its grant on // undelivered chapters rather than on re-making the beginning of the book. resnapshot := book.BankMoved || book.HasPriorRun if in.RePass { // The re-pass purchase (P10 §3.2): «a re-pass costs up to your hold». What it would actually // cost is unknowable at admission (errata 28.08-к) — the engine's status is blind to a // correction until the next translate folds the bank in — so the hold is the honest ceiling of // the work bought: the WHOLE book's projection, of which untouched units come back at $0 and // the difference is released on settlement. The consent cap equals the hold: funded by // construction, and never the bare blanket form. if !book.BankMoved { return pgstore.Run{}, ErrRePassUnavailable } hold, bondFunded := s.Pricing.Hold(priced.Expected-priced.BookOnce, priced.StepMax, priced.BookOnce, acct.Balance) if hold > acct.Balance { return pgstore.Run{}, fmt.Errorf("%w: a re-pass of this book reserves %s", ErrBalanceCannotCarry, hold.USD()) } offset, err := journalSize(book.Workdir) if err != nil { return pgstore.Run{}, err } started, err := s.Store.StartRun(ctx, pgstore.StartRunInput{ UserID: in.UserID, BookID: in.BookID, OrderedChapters: 0, Ceiling: hold, Now: s.now(), Resnapshot: true, AcceptRebill: hold, // ⚠ RECORDED HERE TOO, and its absence was a lie about a whole class of runs: a re-pass // holds the WHOLE book's projection, so it funds the book-level pass more often than any // ordinary purchase — and the field that exists so the platform does not stay silent about // an unfunded pass was reporting every re-pass as unfunded. BondFunded: bondFunded, }, offset, s.enqueue) if err != nil { return pgstore.Run{}, err } s.log().InfoContext(ctx, "re-pass admitted", "run", started.ID) return started.Run, nil } quote, order, err := s.resolveOrder(ctx, in, priced, pb, acct.Balance) if err != nil { return pgstore.Run{}, err } if quote.Chapters == 0 { return pgstore.Run{}, fmt.Errorf("%w: there is nothing left of this book to buy", ErrCeilingOutOfBounds) } if quote.Hold > acct.Balance { // WHY the order does not fit decides what the client can offer next: a balance that moved is // waited out or topped up, while another book's hold is somewhere the user can go and act. // ⚠ THE TEST IS «WOULD THIS ORDER HAVE FIT WITHOUT THAT HOLD», and the first edition of it was // a TAUTOLOGY: it compared this order's hold against a re-quote of the SAME order plus the // held amount, which is true whenever the amount is positive — so every refusal named another // book, including the ones that had nothing to do with it. The honest question compares // against the BALANCE the account would have, and it is the same question `Order` answers when // it decides whether to fill `blocked` at all. if held, amount, err := s.Store.CreditHeldBy(ctx, in.UserID, in.BookID); err == nil && held != "" && quote.Hold <= acct.Balance+amount { return pgstore.Run{}, &CreditHeldError{BookID: held} } return pgstore.Run{}, fmt.Errorf("%w: it reserves %s", ErrBalanceCannotCarry, quote.Hold.USD()) } offset, err := journalSize(book.Workdir) if err != nil { return pgstore.Run{}, err } started, err := s.Store.StartRun(ctx, pgstore.StartRunInput{ UserID: in.UserID, BookID: in.BookID, VerifyBank: in.VerifyBank, // How many chapters the order SPANS. For an order phrased in chapters that is what was bought, // and it is what the bar counts and what the wire publishes. For one phrased in CHARACTERS it // is only stored: the span overstates what was bought — a single unit of a four-unit chapter // spans `1` — so the wire publishes `ordered_units` instead and nulls this // (pgstore.runOrderedChapters). ⚠ It is never 0 for either shape (`max(chapters, 1)` in // QuoteUnits), and that is load-bearing: 0 is the RE-PASS's mark. Pinned by // runs.TestACharacterOrdersVolumeIsPublishedInTheUnitItWasSoldIn and its chapter-shaped twin. // The ORDER itself lives on the book, in units. OrderedChapters: quote.Chapters, Ceiling: quote.Hold, Now: s.now(), Resnapshot: resnapshot, AcceptRebill: rebillConsent(resnapshot, quote.Hold), // ⚠ RECORDED FROM THE QUOTE THAT WAS ACTUALLY ACTED ON, not from the one the form showed. The // form is a read and the balance moves under it, so a buyer shown «funded» can be sold a run // that is not — and the passes degrade rather than halt, which makes the difference invisible // in the book. The run carries what it really got. BondFunded: quote.BondFunded, // ⚠ SET FOR EVERY ORDER PHRASED IN CHARACTERS — including one that happens to land on a chapter // boundary, see pricing.Quote.UnitShaped. It is what switches the run's bar to units: counted // in chapters, an order that stops inside one reads `0/N` for its whole life, which is the // state this very file refuses a book for at admission (PD-405). See migration 00033. OrderedUnits: unitShapedOrder(quote), Order: &order, }, offset, s.enqueue) if err != nil { return pgstore.Run{}, err } // Neither the book id nor the money: an INFO line names the run and nothing that identifies a // user's library or the size of their wallet (D39.84, PD-99). s.log().InfoContext(ctx, "run admitted", "run", started.ID, "ordered_chapters", quote.Chapters) return started.Run, nil } // rebillConsent is the sum a re-pinning run may spend on re-payment: its own hold, and never the // blanket form. // // FUNDED BY CONSTRUCTION, which is the property the adversarial pass K7 bought: a cap larger than // the hold lets a run burn a budget it does not have on re-billing, and a cap of «yes» lets a // projection grown past what the buyer saw be paid silently. Zero when nothing is being re-pinned. func rebillConsent(resnapshot bool, hold money.MicroUSD) money.MicroUSD { if !resnapshot { return 0 } return hold } // resolveOrder turns the three kinds of order into ONE quote and the BOUNDARY the book row stores. // // ⚠ The boundary is an IDENTITY and never a count: a book cut again re-numbers its chapters and // re-mints every unit id, so a stored count would silently become an order for different text // (migration 00033, ratified 05.09). An order for the WHOLE BOOK has no boundary at all, which is // the only honest way to say «all of it, however it is cut». func (s *Service) resolveOrder(ctx context.Context, in StartRequest, priced pgstore.PricedBook, pb pricing.Book, balance money.MicroUSD) (pricing.Quote, pgstore.BookOrder, error) { switch { case in.Chapters != nil: if !(ingest.Manifest{Structure: priced.Structure}).ChapterOrdersOffered() { return pricing.Quote{}, pgstore.BookOrder{}, fmt.Errorf("%w: its structure is %q", ErrChapterOrdersUnavailable, priced.Structure) } if *in.Chapters < 1 { return pricing.Quote{}, pgstore.BookOrder{}, fmt.Errorf("%w: %d chapters", ErrCeilingOutOfBounds, *in.Chapters) } q := s.Pricing.Quote(pb, balance, *in.Chapters) if q.ThroughChapterID == "" { // The order reached the end of the book, so it IS the whole book — recorded as such rather // than as a boundary, for the reason above. return q, pgstore.BookOrder{}, nil } return q, pgstore.BookOrder{ ThroughChapterID: q.ThroughChapterID, ThroughChapterNumber: q.ThroughChapter, }, nil case in.Characters != nil: rows, err := s.Store.RemainingUnits(ctx, in.UserID, in.BookID) if err != nil { return pricing.Quote{}, pgstore.BookOrder{}, err } units := make([]pricing.Unit, len(rows)) for i, u := range rows { units[i] = pricing.Unit{ID: u.ID, Expected: u.Expected, SourceChars: u.SourceChars} } n, ok := pricing.UnitsFor(units, *in.Characters) if !ok || n < 1 { return pricing.Quote{}, pgstore.BookOrder{}, fmt.Errorf("%w: %d characters buys no unit of this book", ErrCeilingOutOfBounds, *in.Characters) } if n >= len(units) { return s.Pricing.Quote(pb, balance, 0), pgstore.BookOrder{}, nil } // ⛔ PRICED FROM THE UNITS THEMSELVES, not from the chapters they fall in. Quoting a unit // order at the price of every chapter it touches is what made this order meaningless exactly // where it is the only one available: a book with no chapter structure is ONE chapter, so a // thousand characters of it reserved the whole book. The chapter span below goes into the ROW // and no further: nothing publishes it any more (pgstore.runOrderedChapters nulls the view for // a unit-shaped run, because the span overstates what was bought), and the bar counts units // because `UnitShaped` is set two lines down. What the stored span is still FOR is the re-pass // discriminator — `ceiling_chapters = 0` is the re-pass's own mark, and a zero written here // would make this order look like one to three internal readers. q := s.Pricing.QuoteUnits(pb, balance, units, n, chaptersSpanning(pb, n)) q.UnitShaped = true return q, pgstore.BookOrder{ThroughUnitID: units[n-1].ID}, nil default: // Neither handle touched: the whole book, and no boundary to store. return s.Pricing.Quote(pb, balance, 0), pgstore.BookOrder{}, nil } } // chaptersSpanning is how many remaining chapters a prefix of `units` units reaches into — at least // one, because a bar with a denominator of zero is not a bar. func chaptersSpanning(b pricing.Book, units int) int { seen, n := 0, 0 for _, c := range b.Remaining { n++ seen += c.Units if seen >= units { break } } return max(n, 1) } // readyToTranslate is the half of the book vocabulary a run may start from: everything a finished // intake can leave behind, plus every state a previous run can leave a parsed book in. // // Written as an allowlist and not as "not uploading, not parsing, not rejected": a status added // later is one this function has never thought about, and refusing it is the safe direction — the // user gets a 409 they can report, rather than an engine started against a book in a state nobody // designed for. func readyToTranslate(status string) bool { switch status { case "not_started", "translating", "awaiting_bank", "ready", "paused", "stopped", "failed": return true } return false } // startable is what THIS BOOK cannot do, beside runnable's what this DEPLOYMENT cannot do: the whole // rule a start is judged by before money or disk is looked at, and the one definition of it. // // ⚠ IT HAS TWO BINDINGS OF DIFFERENT AUTHORITY, and that asymmetry is the design rather than a // shortcut. `Start` binds it under the book lock, where its answer decides whether the run happens; // `Order` binds it on the path a client POLLS, where the same answer is advisory and can be stale // before the user clicks — the canon says as much in its own words, «the only authority on whether // THIS start may happen; every read below can be stale» (§RunOptions). What one definition buys is // that the two cannot drift apart: while the live run was known to the door alone, the form answered // `covers_all` over a book the door was about to refuse, and the click met a refusal the form had // the facts to predict (PD-455). A rule changed here changes both. func startable(book pgstore.BookRunContext) error { if !readyToTranslate(book.Status) { // A book whose intake has not finished has no chapter tree to translate and, while it is // `uploading`, half a file on disk. Refused here rather than discovered by the engine inside a // transient unit, where the only trace would be a marker reading "exit-code 1" — and after the // account's money had already been held for it. return fmt.Errorf("%w: it is %s", ErrBookNotReady, book.Status) } if book.ChapterCount > 0 && !book.HasTree { // The book DECLARES chapters and its tree has not been materialised: the intake committed // `not_started` and the materialisation that follows it — in a separate transaction, outside // the intake's — failed or has not run. Refused HERE, before the hold, because there is no // honest bar for such a run and no way out of it either (PD-405). Every counter the screen // shows is a count over `chapters`, so the run would read 0/total for its entire life while // spending; and the tree's own debt is FROZEN while it runs, because the sweep that pays that // debt skips a book with a live run. The window is the intake's normal one — measured at // 0.018 s on the healthy path — and unbounded exactly when the materialisation broke, which is // the only shape worth refusing. // // `book_not_ready` is the word for it, not a new one: the canon's own gloss is "still // arriving, still being cut, or was rejected", and a book that owes its tree is still being // cut. The remedy is to wait for the intake, which is what the status says. // // ⚠ ONE population does not clear by waiting, and the operator's handle for it exists: a book // whose reading-surface debt was WRITTEN OFF after its attempts (AbandonReadModelDebt) owes // nothing, so no sweep will build its tree and this refusal stands until somebody asks again — // `tmplatformctl book refresh --book `, which is exactly what that command is for, and the // books behind it are listed by `tmplatformctl books --abandoned`. Named here because this // guard is what turns a book that merely showed a frozen bar into one that cannot be started: // the refusal is the honest half, and being able to find the remedy is the other. return fmt.Errorf("%w: its chapters are still being materialised", ErrBookNotReady) } if book.HasLiveRun { return pgstore.ErrRunInFlight } return nil } // sourceThere is the DISK half of admission: the book's project directory, asked for before any // money moves. // // ⛔ IT IS THE DOOR'S ALONE AND DELIBERATELY NOT THE FORM'S, unlike startable above. The form is // polled, and a stat of a hung mount would take the poll down with it; the door is one call the user // is already waiting on. The price of the asymmetry is named rather than hidden: the form stays // silent about a vanished directory and says `covers_all` over it — what it no longer does is stay // silent while the money is TAKEN, which is the half PD-162 is about. // // ⚠ There is no "the journal has not been written yet" case to tell apart here, and looking for one // sends a reader to the wrong file: the directory is created ON INTAKE, before the row that points // at it (books.Accept, MkdirAll ahead of the insert), so a row always names a directory that existed. // The first-run exception belongs to the JOURNAL inside it — journalSize maps ENOENT to a zero // offset, and that is the line this check stands beside rather than duplicates. func (s *Service) sourceThere(workdir string) error { if st, err := os.Stat(workdir); err == nil { if st.IsDir() { return nil } // Something is at the path and it is not a project directory. The platform never wrote that, // so it is not this book's own end to announce — it is an anomaly an operator is owed. return fmt.Errorf("%w: the book's directory is not a directory", ErrStorageUnavailable) } else if !errors.Is(err, fs.ErrNotExist) { // A directory that is there and cannot be read — a permission, an I/O error — is the host's // business exactly as an absent volume is, and the run is refused before the money either way. // // ⚠ The PATH is dropped and only the operation and the errno are kept. `os.Stat` answers a // *fs.PathError, whose text carries the book's own directory, and this error reaches an ERROR // log through the handler: a user's book has no business being in one (PD-139, PD-99). The run // id the log line already carries says which book to an operator who is allowed to ask. var pe *fs.PathError if errors.As(err, &pe) { return fmt.Errorf("%w: %s: %w", ErrStorageUnavailable, pe.Op, pe.Err) } return fmt.Errorf("%w: %w", ErrStorageUnavailable, err) } // WHICH absence this is decides who is at fault, and the two are identical from one book: an // unmounted volume leaves an empty mountpoint behind, so every book under it reads as "gone". // Blaming the book there is the defect PD-192 cost the intake — a host-wide fault, answered as // this book's own end — so the question is put to the storage marker the first upload wrote, not // to the root's existence, and the two answers travel under different words: a deployment's 503 // and the book's own 409. // // ⛔ AND THE MARKER IS ONLY EVIDENCE ABOUT THE ROOT IT LIVES IN. A book whose directory is NOT // under the intake's root — `tmplatformctl book add --workdir` puts one anywhere, and so does an // instance with no `BooksDir` at all — sits on a volume this platform never wrote and has no // sentinel on. Reading the intake's healthy marker as evidence about THAT volume would announce a // vanished mount as the book's own end, which is PD-192 again with a longer path. Where there is // nothing to ask, the answer is the deployment's: this platform does not tell anyone their book is // finished on the strength of a directory it never created. if !books.Owns(s.Cfg.BooksDir, workdir) || !books.StorageIsThere(s.Cfg.BooksDir) { return ErrStorageUnavailable } return ErrSourceGone } func (s *Service) enqueue(ctx context.Context, tx pgstore.Tx, runID string) error { if s.Queue == nil { return nil // no queue configured: the reconciler still picks the run up on its next sweep } return s.Queue.EnqueueRun(ctx, tx, runID) } // runnable is what this DEPLOYMENT cannot do, as opposed to what this run cannot do. Both refusals // are of the run and not of the service: an instance whose runner is misconfigured still serves // every read, and taking the library down with it would turn a partial outage into a total one // during exactly the operation that causes it, a binary upgrade. func (s *Service) runnable() error { if len(s.Cfg.MarkerArgv) == 0 { return ErrRunnerIncomplete } if len(s.Cfg.Ceiling) == 0 { return runner.ErrCeilingNotWired } return nil } // ceilingFor renders the engine's ceiling argument for an amount. func (s *Service) ceilingFor(amount money.MicroUSD) ([]string, error) { return s.Cfg.Ceiling.Args(amount) } // unitShapedOrder is the run's own volume when its order was phrased in CHARACTERS, and nil for every // other shape. Nil is the ordinary case and leaves the bar counted in chapters exactly as before. // // ⚠ The condition is the PHRASING (`q.UnitShaped`), not where the order landed — see // pricing.Quote.UnitShaped for why, and for the measurement that killed the older wording. func unitShapedOrder(q pricing.Quote) *int { if !q.UnitShaped || q.Units <= 0 { return nil } n := q.Units return &n }