package runs import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "textmachine/platform/internal/ingest" "textmachine/platform/internal/pgstore" "textmachine/platform/internal/runner" ) // bank.go: the platform half of the bank-correction door (D39.156 pack 2в) — the service behind // `POST /books/{bookId}/bank/corrections`. The call is SYNCHRONOUS by design: the engine's verb is // $0, exits rather than waits on the signing stop, and its 5000-decision cap was chosen so one call // fits a caller's timeout (backend/internal/pipeline/bankdecisions.go, maxDecisions) — so the // handler accepts the document, runs the verb, and answers its report as the receipt. // // What makes the synchronous form safe is one per-book rule, held HERE and shared with Start and // Resume: a corrections call and the spawn of a fresh run must not interleave on one book. The // engine's flock already makes the interleaving harmless — the loser refuses with class 12 — but a // spawned `translate` that dies on the flock is a wasted attempt with money machinery around it. // // The rule also decides which WORD each refusal carries, and the split is exact: `run_in_flight` // is answered from the platform's own run row (below, under the mutex), and ONLY from it; a class // 12 from the verb itself is by construction NOT a run — Start and Resume wait on this same mutex, // the reconciler only restarts rows the check below already sees — so it is a transient holder of // the project (a boundary materialization reading the book, an operator's manual tmctl) and is // answered «busy, retry later», never with a run that does not exist (PD-400, первая половина). // // ⚠ NAMED BOUNDARY OF v1: the mutex is in-process. A second platform replica over one store would // not see this one's calls, and the serialization narrows to per-replica: the cost is bounded and // is exactly the noise the mutex prevents — a translate spawned into the verb's flock dies with a // wasted attempt, a sibling verb answers «retry later». The boundary stops holding the day a second // replica is deployed (PD-400, вторая половина); the cure then is an arbiter in the store, not a // bigger mutex. // BankCorrectionsInput is one validated correction document, in seam vocabulary. The HANDLER owns // the wire form and its validation; by this point the decisions are renderable as they stand. type BankCorrectionsInput struct { UserID string BookID string Preview bool Decisions []ingest.BankDecision } // BankReceipt is the whole answer of an accepted call, still in the engine's vocabulary where the // two overlap — the projection into contract words (`depth`, pointers) is the HTTP layer's. type BankReceipt struct { Preview bool Changed bool Depth string Accepted []ingest.AcceptedDecision PreexistingFaults int // Signature is nil when no run has reached a signing stop yet — there is nothing to count // against (the report's empty `map`). Signature *ingest.SignatureState } // ErrBankRefused is the all-or-nothing refusal: the document was read and declined whole, nothing // was written, and the USER re-decides. The per-decision reasons ride the error. type ErrBankRefused struct { Refusals []ingest.RejectedDecision } func (e *ErrBankRefused) Error() string { return fmt.Sprintf("runs: the correction document was refused whole (%d reasons)", len(e.Refusals)) } // ErrBankIncomplete — the document was ACCEPTED and did not land whole (the engine's class 15). The // remedy is to re-send the SAME document: the retry converges on the engine's byte no-op. var ErrBankIncomplete = errors.New("runs: the correction write did not complete; re-sending the same document converges") // ErrBankUnavailable — the deployment cannot serve this call RIGHT NOW and the remedy is neither a // re-decision nor a blind resend: an unmigrated project schema, a refusal class this build has no // number for, a stop that landed mid-call. Temporary from the caller's seat; the operator has the // log line. var ErrBankUnavailable = errors.New("runs: the correction door cannot answer right now") // ErrBankDocumentTooLarge — the RENDERED decision document is over the engine's byte ceiling // (ingest.MaxDecisionsDocument). The canon's word for the size bound is 413 «split the document», // and it must be said before the verb is spawned: the engine's own copy of this cap answers as a // refusal, which reads «re-decide». var ErrBankDocumentTooLarge = errors.New("runs: the rendered decision document is over the engine's byte ceiling") // ApplyBankCorrections runs one correction document against a book, synchronously. func (s *Service) ApplyBankCorrections(ctx context.Context, in BankCorrectionsInput) (BankReceipt, error) { if s.Cfg.EngineBinary == "" { // Unreachable through the mounted route — the door is not mounted without an engine — and // still answered rather than assumed: a read replica reached directly must not panic. return BankReceipt{}, errors.New("runs: this deployment has no engine to apply corrections with") } // The budget covers the WAIT as well as the verb: it is set before the queue, so K callers piled // on one book cost each of them at most one budget, not K of them — and a caller whose client or // deadline is gone leaves the queue instead of holding a place in it to do work for a dead // request (workflow finding, P9). // // ⚠ The budget is the sweep's own TM_PLATFORM_RUN_BUDGET knob, and the engine sized its // 5000-decision cap against the SHIPPED 60 s — a maximum document measures ~11.3 s of // uninterruptible fold (backend bankdecisions.go, maxDecisions). An operator lowering the knob // toward that figure makes a legal maximum document permanently unservable: every retry answers // 503 and the «split it» word never comes (PD-408 carries this as the knob's named cost). ctx, cancel := context.WithTimeout(ctx, s.runBudget()) defer cancel() // The book lock FIRST, the live-run check UNDER it: read before the lock, the answer could be // a moment from wrong — a resume admitted while this call waited would spawn into the verb's // flock. Serialized, either this call finishes before the spawn happens, or the run row exists // and the refusal below names it truthfully. unlock, err := s.lockBook(ctx, in.BookID) if err != nil { // The queue outlived the caller's budget or the caller itself: «busy, retry later» is the // honest word — nothing was read, spawned or written. s.log().InfoContext(ctx, "a correction call left the book queue before its turn", "err", err) return BankReceipt{}, ErrBankUnavailable } defer unlock() book, err := s.Store.ReadBookForRun(ctx, in.UserID, in.BookID) if err != nil { return BankReceipt{}, err } if !readyToTranslate(book.Status) { // The same gate Start holds, for the same reason wearing this door's hat: a book still // arriving or being cut has no configuration the verb could open — reaching the engine // anyway came back as its config class, which the caller saw as a 500 about our own wiring, // and the parse worker's transient hold on the project could even dress it as a run // (refuter finding, P9). The remedy is «wait for the intake», which is what book_not_ready // says and internal_error does not. return BankReceipt{}, fmt.Errorf("%w: it is %s", ErrBookNotReady, book.Status) } if book.HasLiveRun && !book.LiveRunAwaitingBank { // The book is being translated (admitted, queued or running). A signing stop is NOT this — // but its row closes one sweep LATER than its screen opens: the journal's bank_stop event // moves the status to `awaiting_bank` while `finished_at` waits for the exit marker // (reconcile.finish), so for that window the row is live and the flock is already free or // about to be. Refusing run_in_flight there refused the very screen the platform had just // announced (workflow finding, P9) — the door opens on it instead, and if the engine is // still winding down, its flock answers class 12: an honest transient «retry later». return BankReceipt{}, pgstore.ErrRunInFlight } doc, err := ingest.EncodeDecisions(in.BookID, in.Decisions) if err != nil { return BankReceipt{}, err } if len(doc) > ingest.MaxDecisionsDocument { // The engine's byte ceiling, measured HERE on the RENDERED document — the only form the // engine will see — so its cap is never reached and the canon's 413 word is kept: the // engine's own copy answers as a refusal of the set, which reads «re-decide» (workflow // finding, P9). With HTML escaping off the band above the wire cap is the envelope's few // bytes, and this gate is what keeps that band from answering with the wrong remedy. return BankReceipt{}, ErrBankDocumentTooLarge } path, cleanup, err := s.decisionsFile(doc) if err != nil { return BankReceipt{}, err } defer cleanup() out, err := s.Bank.BankApply(ctx, s.Cfg.EngineBinary, book.Workdir, path, in.Preview) if err != nil { s.log().ErrorContext(ctx, "the correction verb could not be run", "err", err) if ctx.Err() != nil { // The budget or the caller ended the call — and reaching THIS branch means the verb did // NOT exit on its own (a SIGTERM the engine catches comes back as exit 5 through the // verdict, with res.Exited true): the process was SIGKILLed after bankStopGrace, or // never started. A kill can land between the verb's two renames — exactly the // half-landed pair class 15 exists to report — but there is no report to carry that // word, so the honest remainder is «retry later»: a re-send converges on the engine's // byte no-op from either state (PD-407 keeps the imprecision). return BankReceipt{}, ErrBankUnavailable } return BankReceipt{}, err } rec, err := s.bankVerdict(ctx, in, out) if err == nil && !in.Preview && corrected(rec) { // The fact «the bank moved» is stamped from the door's OWN receipt — the one thing that // cannot be blind to what the door just did. // // ⚠ THE REASON THIS COMMENT USED TO GIVE IS DEAD, and it is dead by a LANDING and not by an // argument (unified backlog row 234). It said asking the engine's status here was the first // edition's mistake because «status projects the STORED memory», so right after an apply it // would honestly answer «nothing moved» (adversarial pass, K1). `D39.170` retired that // premise: `foldMemoryForRead` became the read path's FIRST answer and `projectStoredMemory` // dropped to a fallback (`backend/internal/pipeline/status.go`, grep // `IT IS NO LONGER THE READ PATH'S FIRST ANSWER`). The sibling carrier of the same dead // premise was rewritten on 30.08 (`internal/ingest/resync.go`, register row `PD-427`); this // was the second one, and it stood. // // THE DECISION IS UNCHANGED and now rests on what it should have rested on all along: a // receipt is the door's OWN observation of its OWN act, and a status call is a second // opinion about it from a process that has to be spawned, can fail, and costs seconds of CPU // per call. Even a status that now answers correctly would make the fact depend on being // able to ask — which is exactly the dependency this whole file is built to avoid. // // ⚠ On a context of its OWN, detached from the caller — the same rule as the intake's // `writeCtx` (books/parse.go) and the idempotency receipt's `settleCtx` // (httpapi/idempotency.go) — and here it is the difference between a fact and nothing. // // ⚠ The references above and below are by SYMBOL, not by line: this pack moved four of the six // numbered anchors an earlier edition of this comment carried, which is the same rot the // register's own anchors are graded for. Grep the name. // // Two mechanics make the request's context the wrong one to write on. The verb comes back // CLEAN on an already-cancelled call: `runner.BankApply` returns (outcome, nil) whenever the // process exited, and the engine's last look at its context is BEFORE its writes — so a // correction whose files LANDED routinely arrives here with ctx already dead, either because // the tab closed or because the verb ate the whole budget set at the top of this function. // And the remedy this door then names — «re-send the same document» — is addressed to // precisely the client that is gone. The fact has no second writer and no sweep (grep // `bank_moved_at =` — one statement sets it, in `pgstore.RecordBankMove`), so on r.Context() // it is lost FOREVER: `bank_moved_at` stays NULL, the next ordinary run is admitted with // resnapshot=false (`runs.Start`, grep `if book.BankMoved`) and dies on the engine's snapshot // guard AFTER its hold was taken (`pgstore.StartRun`, the hold is taken before anything is // spawned). That is PD-425, and it is money. // // recordBudget rather than the intake's 30 s: this write runs UNDER the book lock taken at // the top, so its tail is a delay the next caller of this book pays. c, cancel := context.WithTimeout(context.WithoutCancel(ctx), recordBudget) defer cancel() if ferr := s.Store.RecordBankMove(c, in.BookID); ferr != nil { // The correction LANDED — the engine's files moved — but the fact did not reach the // store, and the next run's consents are decided from it. «Re-send the same document» // is the honest remedy and it CONVERGES: the engine answers the re-send with its byte // no-op, whose receipt reads already_applied — which is exactly why corrected() accepts // that state too, or a failed write here would never be retried into place. s.log().ErrorContext(ctx, "a correction landed but the bank-move fact did not", "err", ferr) return BankReceipt{}, ErrBankIncomplete } } var refused *ErrBankRefused if errors.As(err, &refused) { // The engine's cap refusals echo the document's PATH — this platform's own temp file. File // paths are server topology and do not cross the wire (the same rule that keeps the report's // `files` off it); the reason survives, the address does not. for i := range refused.Refusals { refused.Refusals[i].Reason = strings.ReplaceAll(refused.Refusals[i].Reason, path, "the decision document") } } return rec, err } // bankVerdict maps the verb's exit and report onto the door's answers. The mapping of each class is // the contract's or argued in the zone journal (pack P9); what this function must never do is let // two different remedies reach the caller as one word. func (s *Service) bankVerdict(ctx context.Context, in BankCorrectionsInput, out runner.BankApplyOutcome) (BankReceipt, error) { fail := func(what string) (BankReceipt, error) { s.log().ErrorContext(ctx, "the correction verb answered outside its contract", "what", what, "exit", out.ExitCode, "stderr", out.Stderr, "decode_err", out.DecodeErr) return BankReceipt{}, fmt.Errorf("runs: bank-apply: %s (exit %d)", what, out.ExitCode) } switch out.ExitCode { case ingest.ExitClean: if !out.Decoded { // Exit 0 always prints the report; a clean exit without one is not the engine. return fail("a clean exit carried no report") } return s.bankReceipt(in, out.Report) case ingest.ExitDecisionsRejected: if !out.Decoded || len(out.Report.Rejected) == 0 { // The DECISIONS class prints a report in every spelling, the caps included, and a refusal // names its reasons — that is the class's own contract (pipeline.ApplyBankDecisions). A // 409 whose `refusals` came out empty would violate the canon's own minItems. return fail("a refusal carried no report or no reasons") } return BankReceipt{}, &ErrBankRefused{Refusals: out.Report.Rejected} case ingest.ExitWriteIncomplete: // The report says which file landed; that truth is server-side. What the caller needs is // the remedy, and it is the same whether nothing landed or half did. s.log().ErrorContext(ctx, "a correction write did not complete; the client re-sends the same document", "stderr", out.Stderr) return BankReceipt{}, ErrBankIncomplete case ingest.ExitProjectLocked: // NOT run_in_flight on THIS replica: a live run was excluded under the mutex before the verb // was spawned (ApplyBankCorrections), and no path of this process admits one while it is held // — so the holder is transient (a boundary materialization, an operator's manual tmctl, the // stopped run's engine still winding down) and «busy, retry later» is honest. ⚠ Across the v1 // boundary above the word CAN be wrong: a sibling replica's Start sees its own mutex and a // clean runs table, and its translate — a real, hours-long run — wins this flock; the honest // word there would be run_in_flight (PD-400, вторая половина: the accepted risk INCLUDES this // false word, not just a wasted attempt). s.log().ErrorContext(ctx, "corrections met a held project with no live run: a transient holder of the book's flock", "stderr", out.Stderr) return BankReceipt{}, ErrBankUnavailable case ingest.ExitSchemaMismatch: // The book's project predates the deployed engine: `tmctl migrate` is owed (deploy order, // row 174). An operator's condition, not the user's — and temporary, which 503 says and a // 500 would not. s.log().ErrorContext(ctx, "corrections refused: the book's project schema is not the engine's; it awaits tmctl migrate", "stderr", out.Stderr) return BankReceipt{}, ErrBankUnavailable case ingest.ExitStopped: // The verb was wound down before it wrote anything (its own SIGTERM contract) — a deploy // restart from the caller's seat. Retrying later converges either way. s.log().ErrorContext(ctx, "the correction verb was stopped mid-call", "stderr", out.Stderr) return BankReceipt{}, ErrBankUnavailable case ingest.ExitConfigInvalid, ingest.ExitSourceUnreadable: // Named BEFORE the band catch below: these are known classes whose fault sits on THIS side // of the seam — the platform renders both of the verb's inputs — so they are an internal // failure, not a "retry later" that would never come true. return fail("the verb refused its caller's own wiring") } if ingest.Refused(out.ExitCode) { // A refusal class this build has no mapping for (19 included): nothing was spent or written, // but WHICH remedy applies is exactly what an unknown number cannot say — so the answer is // the operator's pair of hands, not a guessed one of the other two. s.log().ErrorContext(ctx, "corrections met a refusal class this build cannot map", "exit", out.ExitCode, "stderr", out.Stderr) return BankReceipt{}, ErrBankUnavailable } // Exit 1, the config class (10 — this platform renders both of the verb's inputs, so a broken // one is OUR wiring), 11, and anything else: the deployment is what needs fixing. return fail("an exit outside the door's contract") } // bankReceipt shapes the report of an accepted call. The report's own preview echo is `mode`; it is // cross-checked against what was asked rather than trusted, because the receipt of an APPLY served // for a preview would tell a user their correction landed when nothing did. func (s *Service) bankReceipt(in BankCorrectionsInput, rep ingest.BankReport) (BankReceipt, error) { wantMode := "apply" if in.Preview { wantMode = "projection" } if rep.Mode != wantMode { return BankReceipt{}, fmt.Errorf("runs: bank-apply answered mode %q to a %q call", rep.Mode, wantMode) } out := BankReceipt{ Preview: in.Preview, Changed: rep.Changed, Depth: rep.Depth, Accepted: rep.Accepted, PreexistingFaults: len(rep.PreexistingProblems), } if rep.Signature.Map != "" { sig := rep.Signature out.Signature = &sig } return out, nil } // corrected answers whether this receipt proves the bank carries the document: a write that // changed files, or a retry whose every accepted state reads already_applied — the byte no-op of // a document that landed before. Both prove the move; a no-op that neither changed nor re-found // anything proves none. func corrected(rec BankReceipt) bool { if rec.Changed { return true } for _, a := range rec.Accepted { if a.State == "already_applied" { return true } } return false } // SweepCorrectionScratch removes decision documents an unclean death left behind: cleanup rides a // defer and dies with the process (SIGKILL, OOM, a deploy's expired grace), nothing else deletes // by this mask, and each orphan carries up to a megabyte of a user's own corrections sitting in // the state directory indefinitely (workflow finding, P9). Called once at boot, before the door // serves — a file present THEN belongs to no live call by definition. func (s *Service) SweepCorrectionScratch() { stale, err := filepath.Glob(filepath.Join(s.Cfg.StateDir, "bank-corrections-*.json")) if err != nil || len(stale) == 0 { return } for _, f := range stale { _ = os.Remove(f) } s.log().Info("swept correction documents an earlier process left behind", "count", len(stale)) } // decisionsFile puts the rendered document where the engine can read it — the platform's OWN state // directory, never the book's (D39.110: the engine owns that one). func (s *Service) decisionsFile(doc []byte) (string, func(), error) { if err := os.MkdirAll(s.Cfg.StateDir, 0o750); err != nil { return "", nil, fmt.Errorf("runs: state directory: %w", err) } f, err := os.CreateTemp(s.Cfg.StateDir, "bank-corrections-*.json") if err != nil { return "", nil, fmt.Errorf("runs: write the decision document: %w", err) } cleanup := func() { _ = os.Remove(f.Name()) } if _, err := f.Write(doc); err != nil { _ = f.Close() cleanup() return "", nil, fmt.Errorf("runs: write the decision document: %w", err) } if err := f.Close(); err != nil { cleanup() return "", nil, fmt.Errorf("runs: write the decision document: %w", err) } return f.Name(), cleanup, nil } // lockBook serializes this book's admissions, resumes and corrections against each other, and // watches the caller's context while it WAITS: the critical sections are one engine verb or one // database transaction, but the WAIT for them is only bounded when a caller whose deadline or // client is gone can leave the queue — a sync.Mutex cannot be waited on under a context, and with // one the K-th caller held its place for K budgets and then did the work for a dead request // (workflow finding, P9). A one-slot channel is that same mutex, waitable. func (s *Service) lockBook(ctx context.Context, bookID string) (unlock func(), err error) { s.booksMu.Lock() if s.books == nil { s.books = make(map[string]*bookLock) } l := s.books[bookID] if l == nil { l = &bookLock{slot: make(chan struct{}, 1)} s.books[bookID] = l } l.refs++ s.booksMu.Unlock() release := func() { s.booksMu.Lock() if l.refs--; l.refs == 0 { delete(s.books, bookID) // books are unbounded over a process's life; idle locks are not kept } s.booksMu.Unlock() } select { case l.slot <- struct{}{}: return func() { <-l.slot; release() }, nil case <-ctx.Done(): release() return nil, ctx.Err() } } type bookLock struct { slot chan struct{} // capacity 1: the token in it IS the lock refs int }