textmachine/platform/internal/runs/bank.go

package runs

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/pgstore"
	"textmachine/platform/internal/runner"
)

// bank.go: the platform half of the bank-correction door (D39.156 pack 2в) — the service behind
// `POST /books/{bookId}/bank/corrections`. The call is SYNCHRONOUS by design: the engine's verb is
// $0, exits rather than waits on the signing stop, and its 5000-decision cap was chosen so one call
// fits a caller's timeout (backend/internal/pipeline/bankdecisions.go, maxDecisions) — so the
// handler accepts the document, runs the verb, and answers its report as the receipt.
//
// What makes the synchronous form safe is one per-book rule, held HERE and shared with Start and
// Resume: a corrections call and the spawn of a fresh run must not interleave on one book. The
// engine's flock already makes the interleaving harmless — the loser refuses with class 12 — but a
// spawned `translate` that dies on the flock is a wasted attempt with money machinery around it.
//
// The rule also decides which WORD each refusal carries, and the split is exact: `run_in_flight`
// is answered from the platform's own run row (below, under the mutex), and ONLY from it; a class
// 12 from the verb itself is by construction NOT a run — Start and Resume wait on this same mutex,
// the reconciler only restarts rows the check below already sees — so it is a transient holder of
// the project (a boundary materialization reading the book, an operator's manual tmctl) and is
// answered «busy, retry later», never with a run that does not exist (PD-400, первая половина).
//
// ⚠ NAMED BOUNDARY OF v1: the mutex is in-process. A second platform replica over one store would
// not see this one's calls, and the serialization narrows to per-replica: the cost is bounded and
// is exactly the noise the mutex prevents — a translate spawned into the verb's flock dies with a
// wasted attempt, a sibling verb answers «retry later». The boundary stops holding the day a second
// replica is deployed (PD-400, вторая половина); the cure then is an arbiter in the store, not a
// bigger mutex.

// BankCorrectionsInput is one validated correction document, in seam vocabulary. The HANDLER owns
// the wire form and its validation; by this point the decisions are renderable as they stand.
type BankCorrectionsInput struct {
	UserID    string
	BookID    string
	Preview   bool
	Decisions []ingest.BankDecision
}

// BankReceipt is the whole answer of an accepted call, still in the engine's vocabulary where the
// two overlap — the projection into contract words (`depth`, pointers) is the HTTP layer's.
type BankReceipt struct {
	Preview           bool
	Changed           bool
	Depth             string
	Accepted          []ingest.AcceptedDecision
	PreexistingFaults int
	// Signature is nil when no run has reached a signing stop yet — there is nothing to count
	// against (the report's empty `map`).
	Signature *ingest.SignatureState
}

// ErrBankRefused is the all-or-nothing refusal: the document was read and declined whole, nothing
// was written, and the USER re-decides. The per-decision reasons ride the error.
type ErrBankRefused struct {
	Refusals []ingest.RejectedDecision
}

func (e *ErrBankRefused) Error() string {
	return fmt.Sprintf("runs: the correction document was refused whole (%d reasons)", len(e.Refusals))
}

// ErrBankIncomplete — the document was ACCEPTED and did not land whole (the engine's class 15). The
// remedy is to re-send the SAME document: the retry converges on the engine's byte no-op.
var ErrBankIncomplete = errors.New("runs: the correction write did not complete; re-sending the same document converges")

// ErrBankUnavailable — the deployment cannot serve this call RIGHT NOW and the remedy is neither a
// re-decision nor a blind resend: an unmigrated project schema, a refusal class this build has no
// number for, a stop that landed mid-call. Temporary from the caller's seat; the operator has the
// log line.
var ErrBankUnavailable = errors.New("runs: the correction door cannot answer right now")

// ErrBankDocumentTooLarge — the RENDERED decision document is over the engine's byte ceiling
// (ingest.MaxDecisionsDocument). The canon's word for the size bound is 413 «split the document»,
// and it must be said before the verb is spawned: the engine's own copy of this cap answers as a
// refusal, which reads «re-decide».
var ErrBankDocumentTooLarge = errors.New("runs: the rendered decision document is over the engine's byte ceiling")

// ApplyBankCorrections runs one correction document against a book, synchronously.
func (s *Service) ApplyBankCorrections(ctx context.Context, in BankCorrectionsInput) (BankReceipt, error) {
	if s.Cfg.EngineBinary == "" {
		// Unreachable through the mounted route — the door is not mounted without an engine — and
		// still answered rather than assumed: a read replica reached directly must not panic.
		return BankReceipt{}, errors.New("runs: this deployment has no engine to apply corrections with")
	}
	// The budget covers the WAIT as well as the verb: it is set before the queue, so K callers piled
	// on one book cost each of them at most one budget, not K of them — and a caller whose client or
	// deadline is gone leaves the queue instead of holding a place in it to do work for a dead
	// request (workflow finding, P9).
	//
	// ⚠ The budget is the sweep's own TM_PLATFORM_RUN_BUDGET knob, and the engine sized its
	// 5000-decision cap against the SHIPPED 60 s — a maximum document measures ~11.3 s of
	// uninterruptible fold (backend bankdecisions.go, maxDecisions). An operator lowering the knob
	// toward that figure makes a legal maximum document permanently unservable: every retry answers
	// 503 and the «split it» word never comes (PD-408 carries this as the knob's named cost).
	ctx, cancel := context.WithTimeout(ctx, s.runBudget())
	defer cancel()
	// The book lock FIRST, the live-run check UNDER it: read before the lock, the answer could be
	// a moment from wrong — a resume admitted while this call waited would spawn into the verb's
	// flock. Serialized, either this call finishes before the spawn happens, or the run row exists
	// and the refusal below names it truthfully.
	unlock, err := s.lockBook(ctx, in.BookID)
	if err != nil {
		// The queue outlived the caller's budget or the caller itself: «busy, retry later» is the
		// honest word — nothing was read, spawned or written.
		s.log().InfoContext(ctx, "a correction call left the book queue before its turn", "err", err)
		return BankReceipt{}, ErrBankUnavailable
	}
	defer unlock()
	book, err := s.Store.ReadBookForRun(ctx, in.UserID, in.BookID)
	if err != nil {
		return BankReceipt{}, err
	}
	if !readyToTranslate(book.Status) {
		// The same gate Start holds, for the same reason wearing this door's hat: a book still
		// arriving or being cut has no configuration the verb could open — reaching the engine
		// anyway came back as its config class, which the caller saw as a 500 about our own wiring,
		// and the parse worker's transient hold on the project could even dress it as a run
		// (refuter finding, P9). The remedy is «wait for the intake», which is what book_not_ready
		// says and internal_error does not.
		return BankReceipt{}, fmt.Errorf("%w: it is %s", ErrBookNotReady, book.Status)
	}
	if book.HasLiveRun && !book.LiveRunAwaitingBank {
		// The book is being translated (admitted, queued or running). A signing stop is NOT this —
		// but its row closes one sweep LATER than its screen opens: the journal's bank_stop event
		// moves the status to `awaiting_bank` while `finished_at` waits for the exit marker
		// (reconcile.finish), so for that window the row is live and the flock is already free or
		// about to be. Refusing run_in_flight there refused the very screen the platform had just
		// announced (workflow finding, P9) — the door opens on it instead, and if the engine is
		// still winding down, its flock answers class 12: an honest transient «retry later».
		return BankReceipt{}, pgstore.ErrRunInFlight
	}
	doc, err := ingest.EncodeDecisions(in.BookID, in.Decisions)
	if err != nil {
		return BankReceipt{}, err
	}
	if len(doc) > ingest.MaxDecisionsDocument {
		// The engine's byte ceiling, measured HERE on the RENDERED document — the only form the
		// engine will see — so its cap is never reached and the canon's 413 word is kept: the
		// engine's own copy answers as a refusal of the set, which reads «re-decide» (workflow
		// finding, P9). With HTML escaping off the band above the wire cap is the envelope's few
		// bytes, and this gate is what keeps that band from answering with the wrong remedy.
		return BankReceipt{}, ErrBankDocumentTooLarge
	}
	path, cleanup, err := s.decisionsFile(doc)
	if err != nil {
		return BankReceipt{}, err
	}
	defer cleanup()
	// ⛔ THE WRITE-AHEAD MARK, COMMITTED BEFORE THE VERB, and it is the only channel a death with no
	// channel leaves open (unified backlog row 400). What this door does is run the verb and then
	// stamp the fact from its own receipt; the stamp has ONE writer and no sweep, so a process that
	// dies in between leaves the book looking untouched — and the next run is admitted without
	// `--resnapshot` and dies on the engine's snapshot guard AFTER its hold was taken (PD-425). The
	// store-error half of that window is already answered below («re-send the same document», which
	// converges on the engine's byte no-op); this is the half where nobody is left to re-send.
	//
	// ⚠ A PREVIEW SETS NOTHING: it writes no files by construction, so a mark it left would cost a
	// later run a --resnapshot for an act that never happened.
	//
	// ⚠ AND THE DOOR REFUSES WHEN THE MARK CANNOT BE WRITTEN, rather than proceeding without a
	// backstop. The asymmetry is the same one that chose the mark's direction: refusing costs this
	// caller a «retry later» on a document whose re-send converges, while proceeding risks a paid run
	// that dies after the money moved. It is one statement against a database this process is already
	// connected to, on a path that has already made several reads.
	if !in.Preview {
		if err := s.Store.MarkBankMovePending(ctx, in.BookID, s.now()); err != nil {
			s.log().ErrorContext(ctx, "the correction was not started: its write-ahead mark could not be recorded", "err", err)
			return BankReceipt{}, ErrBankUnavailable
		}
	}
	out, err := s.Bank.BankApply(ctx, s.Cfg.EngineBinary, book.Workdir, path, in.Preview)
	if err != nil {
		s.log().ErrorContext(ctx, "the correction verb could not be run", "err", err)
		if ctx.Err() != nil {
			// The budget or the caller ended the call — and reaching THIS branch means the verb did
			// NOT exit on its own (a SIGTERM the engine catches comes back as exit 5 through the
			// verdict, with res.Exited true): the process was SIGKILLed after bankStopGrace, or
			// never started. A kill can land between the verb's two renames — exactly the
			// half-landed pair class 15 exists to report — but there is no report to carry that
			// word, so the honest remainder is «retry later»: a re-send converges on the engine's
			// byte no-op from either state (PD-407 keeps the imprecision).
			return BankReceipt{}, ErrBankUnavailable
		}
		return BankReceipt{}, err
	}
	rec, err := s.bankVerdict(ctx, in, out)
	if err == nil && !in.Preview && corrected(rec) {
		// The fact «the bank moved» is stamped from the door's OWN receipt — the one thing that
		// cannot be blind to what the door just did.
		//
		// ⚠ THE REASON THIS COMMENT USED TO GIVE IS DEAD, and it is dead by a LANDING and not by an
		// argument (unified backlog row 234). It said asking the engine's status here was the first
		// edition's mistake because «status projects the STORED memory», so right after an apply it
		// would honestly answer «nothing moved» (adversarial pass, K1). `D39.170` retired that
		// premise: `foldMemoryForRead` became the read path's FIRST answer and `projectStoredMemory`
		// dropped to a fallback (`backend/internal/pipeline/status.go`, grep
		// `IT IS NO LONGER THE READ PATH'S FIRST ANSWER`). The sibling carrier of the same dead
		// premise was rewritten on 30.08 (`internal/ingest/resync.go`, register row `PD-427`); this
		// was the second one, and it stood.
		//
		// THE DECISION IS UNCHANGED and now rests on what it should have rested on all along: a
		// receipt is the door's OWN observation of its OWN act, and a status call is a second
		// opinion about it from a process that has to be spawned, can fail, and costs seconds of CPU
		// per call. Even a status that now answers correctly would make the fact depend on being
		// able to ask — which is exactly the dependency this whole file is built to avoid.
		//
		// ⚠ On a context of its OWN, detached from the caller — the same rule as the intake's
		// `writeCtx` (books/parse.go) and the idempotency receipt's `settleCtx`
		// (httpapi/idempotency.go) — and here it is the difference between a fact and nothing.
		//
		// ⚠ The references above and below are by SYMBOL, not by line: this pack moved four of the six
		// numbered anchors an earlier edition of this comment carried, which is the same rot the
		// register's own anchors are graded for. Grep the name.
		//
		// Two mechanics make the request's context the wrong one to write on. The verb comes back
		// CLEAN on an already-cancelled call: `runner.BankApply` returns (outcome, nil) whenever the
		// process exited, and the engine's last look at its context is BEFORE its writes — so a
		// correction whose files LANDED routinely arrives here with ctx already dead, either because
		// the tab closed or because the verb ate the whole budget set at the top of this function.
		// And the remedy this door then names — «re-send the same document» — is addressed to
		// precisely the client that is gone. The fact has no second writer and no sweep (grep
		// `bank_moved_at =` — one statement sets it, in `pgstore.RecordBankMove`), so on r.Context()
		// it is lost FOREVER: `bank_moved_at` stays NULL, the next ordinary run is admitted with
		// resnapshot=false (`runs.Start`, grep `if book.BankMoved`) and dies on the engine's snapshot
		// guard AFTER its hold was taken (`pgstore.StartRun`, the hold is taken before anything is
		// spawned). That is PD-425, and it is money.
		//
		// recordBudget rather than the intake's 30 s: this write runs UNDER the book lock taken at
		// the top, so its tail is a delay the next caller of this book pays.
		c, cancel := context.WithTimeout(context.WithoutCancel(ctx), recordBudget)
		defer cancel()
		if ferr := s.Store.RecordBankMove(c, in.BookID); ferr != nil {
			// The correction LANDED — the engine's files moved — but the fact did not reach the
			// store, and the next run's consents are decided from it. «Re-send the same document»
			// is the honest remedy and it CONVERGES: the engine answers the re-send with its byte
			// no-op, whose receipt reads already_applied — which is exactly why corrected() accepts
			// that state too, or a failed write here would never be retried into place.
			s.log().ErrorContext(ctx, "a correction landed but the bank-move fact did not", "err", ferr)
			return BankReceipt{}, ErrBankIncomplete
		}
	}
	// ⚠ THE MARK IS RETIRED ONLY AGAINST A RECEIPT, never against the absence of one. Two outcomes
	// PROVE that nothing landed: the verb's own all-or-nothing refusal (it read the document and
	// declined it whole) and a clean report that neither changed anything nor re-found it already
	// applied. Everything else keeps the mark, and deliberately: a killed verb can die between its
	// own two renames — the half-landed pair class 15 exists to report — so «no report» may not be
	// read as «no change». The price of keeping it is one harmless --resnapshot on the next run.
	var refused *ErrBankRefused
	if !in.Preview && ((err == nil && !corrected(rec)) || errors.As(err, &refused)) {
		// Detached like the stamp below it and for the same reason: the commonest way to arrive here
		// with a dead context is a client that left, and the mark's whole job is to outlive that.
		c, cancel := context.WithTimeout(context.WithoutCancel(ctx), recordBudget)
		defer cancel()
		if ferr := s.Store.ClearBankMovePending(c, in.BookID); ferr != nil {
			// Not an error to the caller: nothing landed, so the state is honest either way — the
			// leftover mark costs the next run one harmless --resnapshot and nothing else.
			s.log().ErrorContext(ctx, "a correction that landed nothing left its write-ahead mark behind", "err", ferr)
		}
	}
	if errors.As(err, &refused) {
		// The engine's cap refusals echo the document's PATH — this platform's own temp file. File
		// paths are server topology and do not cross the wire (the same rule that keeps the report's
		// `files` off it); the reason survives, the address does not.
		for i := range refused.Refusals {
			refused.Refusals[i].Reason = strings.ReplaceAll(refused.Refusals[i].Reason, path, "the decision document")
		}
	}
	return rec, err
}

// bankVerdict maps the verb's exit and report onto the door's answers. The mapping of each class is
// the contract's or argued in the zone journal (pack P9); what this function must never do is let
// two different remedies reach the caller as one word.
func (s *Service) bankVerdict(ctx context.Context, in BankCorrectionsInput, out runner.BankApplyOutcome) (BankReceipt, error) {
	fail := func(what string) (BankReceipt, error) {
		s.log().ErrorContext(ctx, "the correction verb answered outside its contract",
			"what", what, "exit", out.ExitCode, "stderr", out.Stderr, "decode_err", out.DecodeErr)
		return BankReceipt{}, fmt.Errorf("runs: bank-apply: %s (exit %d)", what, out.ExitCode)
	}
	switch out.ExitCode {
	case ingest.ExitClean:
		if !out.Decoded {
			// Exit 0 always prints the report; a clean exit without one is not the engine.
			return fail("a clean exit carried no report")
		}
		return s.bankReceipt(in, out.Report)
	case ingest.ExitDecisionsRejected:
		if !out.Decoded || len(out.Report.Rejected) == 0 {
			// The DECISIONS class prints a report in every spelling, the caps included, and a refusal
			// names its reasons — that is the class's own contract (pipeline.ApplyBankDecisions). A
			// 409 whose `refusals` came out empty would violate the canon's own minItems.
			return fail("a refusal carried no report or no reasons")
		}
		return BankReceipt{}, &ErrBankRefused{Refusals: out.Report.Rejected}
	case ingest.ExitWriteIncomplete:
		// The report says which file landed; that truth is server-side. What the caller needs is
		// the remedy, and it is the same whether nothing landed or half did.
		s.log().ErrorContext(ctx, "a correction write did not complete; the client re-sends the same document",
			"stderr", out.Stderr)
		return BankReceipt{}, ErrBankIncomplete
	case ingest.ExitProjectLocked:
		// NOT run_in_flight on THIS replica: a live run was excluded under the mutex before the verb
		// was spawned (ApplyBankCorrections), and no path of this process admits one while it is held
		// — so the holder is transient (a boundary materialization, an operator's manual tmctl, the
		// stopped run's engine still winding down) and «busy, retry later» is honest. ⚠ Across the v1
		// boundary above the word CAN be wrong: a sibling replica's Start sees its own mutex and a
		// clean runs table, and its translate — a real, hours-long run — wins this flock; the honest
		// word there would be run_in_flight (PD-400, вторая половина: the accepted risk INCLUDES this
		// false word, not just a wasted attempt).
		s.log().ErrorContext(ctx, "corrections met a held project with no live run: a transient holder of the book's flock",
			"stderr", out.Stderr)
		return BankReceipt{}, ErrBankUnavailable
	case ingest.ExitSchemaMismatch:
		// The book's project predates the deployed engine: `tmctl migrate` is owed (deploy order,
		// row 174). An operator's condition, not the user's — and temporary, which 503 says and a
		// 500 would not.
		s.log().ErrorContext(ctx, "corrections refused: the book's project schema is not the engine's; it awaits tmctl migrate",
			"stderr", out.Stderr)
		return BankReceipt{}, ErrBankUnavailable
	case ingest.ExitStopped:
		// The verb was wound down before it wrote anything (its own SIGTERM contract) — a deploy
		// restart from the caller's seat. Retrying later converges either way.
		s.log().ErrorContext(ctx, "the correction verb was stopped mid-call", "stderr", out.Stderr)
		return BankReceipt{}, ErrBankUnavailable
	case ingest.ExitConfigInvalid, ingest.ExitSourceUnreadable:
		// Named BEFORE the band catch below: these are known classes whose fault sits on THIS side
		// of the seam — the platform renders both of the verb's inputs — so they are an internal
		// failure, not a "retry later" that would never come true.
		return fail("the verb refused its caller's own wiring")
	}
	if ingest.Refused(out.ExitCode) {
		// A refusal class this build has no mapping for (19 included): nothing was spent or written,
		// but WHICH remedy applies is exactly what an unknown number cannot say — so the answer is
		// the operator's pair of hands, not a guessed one of the other two.
		s.log().ErrorContext(ctx, "corrections met a refusal class this build cannot map",
			"exit", out.ExitCode, "stderr", out.Stderr)
		return BankReceipt{}, ErrBankUnavailable
	}
	// Exit 1, the config class (10 — this platform renders both of the verb's inputs, so a broken
	// one is OUR wiring), 11, and anything else: the deployment is what needs fixing.
	return fail("an exit outside the door's contract")
}

// bankReceipt shapes the report of an accepted call. The report's own preview echo is `mode`; it is
// cross-checked against what was asked rather than trusted, because the receipt of an APPLY served
// for a preview would tell a user their correction landed when nothing did.
func (s *Service) bankReceipt(in BankCorrectionsInput, rep ingest.BankReport) (BankReceipt, error) {
	wantMode := "apply"
	if in.Preview {
		wantMode = "projection"
	}
	if rep.Mode != wantMode {
		return BankReceipt{}, fmt.Errorf("runs: bank-apply answered mode %q to a %q call", rep.Mode, wantMode)
	}
	out := BankReceipt{
		Preview: in.Preview, Changed: rep.Changed, Depth: rep.Depth,
		Accepted:          rep.Accepted,
		PreexistingFaults: len(rep.PreexistingProblems),
	}
	if rep.Signature.Map != "" {
		sig := rep.Signature
		out.Signature = &sig
	}
	return out, nil
}

// corrected answers whether this receipt proves the bank carries the document: a write that
// changed files, or a retry whose every accepted state reads already_applied — the byte no-op of
// a document that landed before. Both prove the move; a no-op that neither changed nor re-found
// anything proves none.
func corrected(rec BankReceipt) bool {
	if rec.Changed {
		return true
	}
	for _, a := range rec.Accepted {
		if a.State == "already_applied" {
			return true
		}
	}
	return false
}

// SweepCorrectionScratch removes decision documents an unclean death left behind: cleanup rides a
// defer and dies with the process (SIGKILL, OOM, a deploy's expired grace), nothing else deletes
// by this mask, and each orphan carries up to a megabyte of a user's own corrections sitting in
// the state directory indefinitely (workflow finding, P9). Called once at boot, before the door
// serves — a file present THEN belongs to no live call by definition.
func (s *Service) SweepCorrectionScratch() {
	stale, err := filepath.Glob(filepath.Join(s.Cfg.StateDir, "bank-corrections-*.json"))
	if err != nil || len(stale) == 0 {
		return
	}
	for _, f := range stale {
		_ = os.Remove(f)
	}
	s.log().Info("swept correction documents an earlier process left behind", "count", len(stale))
}

// decisionsFile puts the rendered document where the engine can read it — the platform's OWN state
// directory, never the book's (D39.110: the engine owns that one).
func (s *Service) decisionsFile(doc []byte) (string, func(), error) {
	if err := os.MkdirAll(s.Cfg.StateDir, 0o750); err != nil {
		return "", nil, fmt.Errorf("runs: state directory: %w", err)
	}
	f, err := os.CreateTemp(s.Cfg.StateDir, "bank-corrections-*.json")
	if err != nil {
		return "", nil, fmt.Errorf("runs: write the decision document: %w", err)
	}
	cleanup := func() { _ = os.Remove(f.Name()) }
	if _, err := f.Write(doc); err != nil {
		_ = f.Close()
		cleanup()
		return "", nil, fmt.Errorf("runs: write the decision document: %w", err)
	}
	if err := f.Close(); err != nil {
		cleanup()
		return "", nil, fmt.Errorf("runs: write the decision document: %w", err)
	}
	return f.Name(), cleanup, nil
}

// lockBook serializes this book's admissions, resumes and corrections against each other, and
// watches the caller's context while it WAITS: the critical sections are one engine verb or one
// database transaction, but the WAIT for them is only bounded when a caller whose deadline or
// client is gone can leave the queue — a sync.Mutex cannot be waited on under a context, and with
// one the K-th caller held its place for K budgets and then did the work for a dead request
// (workflow finding, P9). A one-slot channel is that same mutex, waitable.
func (s *Service) lockBook(ctx context.Context, bookID string) (unlock func(), err error) {
	s.booksMu.Lock()
	if s.books == nil {
		s.books = make(map[string]*bookLock)
	}
	l := s.books[bookID]
	if l == nil {
		l = &bookLock{slot: make(chan struct{}, 1)}
		s.books[bookID] = l
	}
	l.refs++
	s.booksMu.Unlock()
	release := func() {
		s.booksMu.Lock()
		if l.refs--; l.refs == 0 {
			delete(s.books, bookID) // books are unbounded over a process's life; idle locks are not kept
		}
		s.booksMu.Unlock()
	}
	select {
	case l.slot <- struct{}{}:
		return func() { <-l.slot; release() }, nil
	case <-ctx.Done():
		release()
		return nil, ctx.Err()
	}
}

type bookLock struct {
	slot chan struct{} // capacity 1: the token in it IS the lock
	refs int
}