textmachine/platform/internal/books/parse.go

package books

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/jobs"
	"textmachine/platform/internal/pgstore"
	"textmachine/platform/internal/runner"
)

// Graces and budgets of the intake walk. Constants rather than settings: they are properties of the
// walk itself, and every one of them is a number an operator would have to reason about the
// reconciler to choose.
const (
	// claimGrace is how long a parse may be somebody's business before another pass may take it. It
	// covers the ordinary case — the queue job is claimed in milliseconds — and the failure it exists
	// for: the process holding the claim was restarted mid-parse.
	//
	// ⚠ It MUST outlive the queue's own job timeout, and it is written as that constant plus a margin
	// so the two cannot drift apart. Shorter, the sweep steals the claim from a parse that is still
	// legitimately running: thief and holder meet on one project directory, the loser dies on the
	// engine's exclusive lock with exit 1 — and exit 1 is how the engine says "I cannot cut this
	// source", so a book that was parsing perfectly well would be rejected for it.
	claimGrace = jobs.JobTimeout + 5*time.Minute
	// UploadGrace is how long a book may stay `uploading`. A request that is still arriving holds the
	// row, so anything older than this is an upload whose request is gone — and that reasoning holds
	// only while the route's own read deadline is SHORTER. Exported so the boot can refuse a
	// configuration where it is not: an operator who raises TM_PLATFORM_UPLOAD_DEADLINE past this
	// would have the sweep delete a book's row and directory out from under a request still writing
	// into it.
	UploadGrace = time.Hour
	// parseAttempts is how many times a book may be handed to the engine before intake gives up. EVERY
	// answer of the engine goes through it — a refusal of the source as much as a host that cannot run
	// the binary — because the engine gives both the same exit code, so what this really bounds is how
	// long a broken host re-runs the engine over every book uploaded to it, and how many chances a
	// genuinely unreadable file gets before its source is removed.
	parseAttempts = 5
)

// Reasons a book is rejected. The platform's OWN closed vocabulary: the engine's text reads like
// pipeline internals and never crosses this seam (contract §Problem, PT-33), and these are stored
// for an operator rather than projected — contract v0 gives a rejected book no reason field.
const (
	// ReasonSourceUnreadable — the engine ran and refused the file. Terminal, but NOT on the first
	// answer: the engine maps every failure it has onto exit 1, so a refusal goes through the attempt
	// budget like everything else and only the last one deletes the source (see defer_).
	ReasonSourceUnreadable = "source_unreadable"
	// ReasonNotConfigured — the book has no project configuration, so there is nothing to parse
	// against. See ErrNotProvisioned: today that is a deployment's state, not a user's mistake.
	ReasonNotConfigured = "not_configured"
	// ReasonParserUnavailable — the engine could not be RUN, repeatedly, until the attempt budget was
	// spent.
	ReasonParserUnavailable = "parser_unavailable"
	// ReasonStorageUnavailable — the storage root itself is not there. Never terminal, and never
	// stored on a book: it says something about the host, and the book it happened to be read for is
	// no more at fault than any other. See ErrStorageGone.
	ReasonStorageUnavailable = "storage_unavailable"
)

// ErrNotProvisioned is a book directory with no engine configuration in it.
//
// ⚠ This is the seam of an OPEN question, and it is named here rather than papered over. A book's
// project configuration (`book.yaml`) carries the pair, the ceilings and the paths the engine reads,
// and D39.110 §2b says the platform does not write that file. Creating a directory for an uploaded
// book therefore leaves somebody having to put the first one there, and who that is — the operator
// from a deployment template, or the platform rendering one at intake and never touching it again —
// is a ratification the zone asked for and does not have (zone journal, P5).
//
// Until it arrives the intake is complete up to this line: the file is received, stored, counted and
// handed to the engine, and a book whose directory carries no configuration is rejected with a
// reason that says so to an operator.
var ErrNotProvisioned = errors.New("books: the book has no engine configuration")

// ErrDirectoryGone is a book whose whole project directory is missing.
//
// Told apart from ErrNotProvisioned deliberately, and the difference is terminal-vs-not: a missing
// CONFIGURATION is a deployment question somebody can still answer, while a missing DIRECTORY means
// the source this platform received is not there any more and no amount of waiting brings it back.
// It is also the crash window of the rejection path — the directory is removed before the row is
// written — and reading it as "not configured" left such a book in `parsing` for good.
var ErrDirectoryGone = errors.New("books: the book's directory is gone")

// ErrStorageGone is the ROOT of the book storage missing — an unmounted volume, or a deployment
// pointed at a path that is not there yet.
//
// Told apart from ErrDirectoryGone because the two look identical from one book (both are ENOENT on
// the same Stat) and mean opposite things. A book's own directory being gone is that book's own
// terminal end; the root being gone is the host's, and reading it as the first would have ONE sweep
// reject EVERY book in intake with "the source cannot be read" — a reason that blames the user's
// file, is terminal by design, and has no way back (there is no re-parse and no un-reject). Found by
// cross-family review of the acceptance dofix.
var ErrStorageGone = errors.New("books: the book storage root is gone")

// Parse turns a received file into a chapter tree, or into a reason it is not one.
//
// It is the body of the queue's worker AND of the backstop sweep, and it is safe to call twice: the
// claim is a compare-and-set, so the loser does nothing. Losing that race is the ordinary case, not
// a failure — two `tmctl manifest` processes on one project directory would be two writers of a file
// the engine holds exclusively.
func (s *Service) Parse(ctx context.Context, bookID string) error {
	now := s.now()
	claim, err := s.Store.ClaimParse(ctx, bookID, now, now.Add(-claimGrace))
	if errors.Is(err, pgstore.ErrParseClaimed) {
		return nil
	}
	if err != nil {
		return err
	}
	m, err := s.manifest(ctx, claim.Workdir)
	if err == nil {
		if m.ChaptersTotal < 1 {
			// A source the engine read and found nothing in. The book's own fault — but it goes through
			// the same budget as everything else, because "the engine answered no" and "the engine could
			// not answer" are told apart by an exit code the engine gives BOTH.
			return s.defer_(ctx, claim, ReasonSourceUnreadable)
		}
		// Counts and versions, no book id: the same rule as everywhere else on this side of the log
		// (ENGINEERING_STANDARDS §Наблюдаемость). What an operator needs per book — including why a
		// book was rejected — is a column, not a log line.
		s.log().InfoContext(ctx, "book parsed", "chapters", m.ChaptersTotal,
			"manifest_version", m.Version, "chunker", m.ChunkerVersion)
		c, cancel := writeCtx(ctx)
		defer cancel()
		return s.Store.FinishParse(c, bookID, claim.At, pgstore.ParsedBook{
			Chapters:       m.ChaptersTotal,
			SourceSHA256:   m.SourceSHA256Bytes(),
			ChunkerVersion: m.ChunkerVersion,
		})
	}
	if errors.Is(err, ErrStorageGone) {
		// The host cannot see its own storage. Nothing about this book is known yet, so it waits with
		// the budget untouched — exactly like a book waiting for a configuration.
		s.log().ErrorContext(ctx, "the book storage root is not there: intake waits rather than rejects",
			"reason", ReasonStorageUnavailable)
		return s.defer_(ctx, claim, ReasonStorageUnavailable)
	}
	if errors.Is(err, ErrDirectoryGone) {
		// Nothing to wait for and nothing to retry: what this platform received is not on disk. It is
		// also how the rejection path heals from a crash between removing the directory and writing
		// the row — the next pass finds no directory and finishes the job.
		s.log().ErrorContext(ctx, "book rejected: its directory is gone", "reason", ReasonSourceUnreadable)
		return s.reject(ctx, claim, ReasonSourceUnreadable)
	}
	reason := ReasonParserUnavailable
	switch {
	case errors.Is(err, ErrNotProvisioned):
		reason = ReasonNotConfigured
	case refusedTheSource(err):
		reason = ReasonSourceUnreadable
	}
	// ⚠ err can carry the book's path (the engine is asked about a directory), which is the open class
	// of PD-139, so it travels only at WARN/ERROR and never at INFO.
	return s.defer_(ctx, claim, reason)
}

// defer_ spends one attempt of the budget and, when the budget is gone, ends the intake.
//
// ONE path for every way a parse can fail, and that is the correction the cross-family review
// forced: the engine maps ALL of its failures onto exit 1 (its own comment says so), so "this source
// cannot be cut" is not distinguishable from "the disk was full", "an operator's own tmctl held the
// lock", or "another pass of this platform took the claim and is holding it". Rejecting — and
// DELETING the user's file — on the first exit 1 turned any of those into irreversible data loss.
// Spending an attempt instead costs a broken host five engine calls spread over the claim grace and
// costs a genuinely unreadable book the same five before it is rejected.
//
// ⚠ `not_configured` is the one reason that NEVER becomes terminal. The platform does not write
// `book.yaml` (D39.110 §2b) and who does is a ratification this zone does not have, so a book whose
// directory has no configuration is waiting on a DEPLOYMENT question. Rejecting it — and it would be
// every uploaded book on a deployment without that answer — would destroy a user's upload over a
// gap they cannot see. It stays `parsing`, visible in the intake metric, and an operator dropping
// the file in is all it takes.
func (s *Service) defer_(ctx context.Context, claim pgstore.ParseClaim, reason string) error {
	if !waitsForTheDeployment(reason) && claim.Attempts >= parseAttempts {
		s.log().ErrorContext(ctx, "book rejected: intake has spent its attempts on it",
			"attempts", claim.Attempts, "reason", reason)
		return s.reject(ctx, claim, reason)
	}
	// The claim is NOT given back: it is what spaces the retries. Released, the sweep would re-offer
	// the book on its very next tick (the staleness predicate falls back to `added_at`, which is
	// already old), and the attempts would burn in as many ticks — 75 seconds at the default sweep
	// interval — rather than over the time this budget is for.
	//
	// The ATTEMPT, though, is given back when the engine was never asked. `ClaimParse` counts every
	// claim, and a book waiting for its configuration claims once per grace forever — so without this
	// the budget was spent by WAITING, and the first real answer from the engine afterwards was
	// terminal on arrival. A typo in a hand-written `book.yaml` would then delete the user's upload
	// on the first attempt at reading it. Waiting must not bring deletion closer.
	if waitsForTheDeployment(reason) {
		c, cancel := writeCtx(ctx)
		defer cancel()
		if err := s.Store.RefundParseAttempt(c, claim.BookID, claim.At); err != nil {
			s.log().ErrorContext(ctx, "the parse attempt could not be refunded", "err", err)
		}
	}
	s.log().WarnContext(ctx, "parse deferred", "attempts", claim.Attempts, "reason", reason)
	return nil
}

// waitsForTheDeployment reports the reasons that never end an intake and never spend its budget: the
// engine was not asked at all, and what is missing belongs to the deployment — a configuration
// nobody has written yet, or the storage root itself. Both are answerable by a human, and both would
// otherwise destroy every upload on a host that is merely misconfigured.
func waitsForTheDeployment(reason string) bool {
	return reason == ReasonNotConfigured || reason == ReasonStorageUnavailable
}

// writeCtx is the context a TERMINAL write uses: detached from the caller's deadline and bounded on
// its own.
//
// The engine call this follows can legitimately consume the whole budget of the pass — and then the
// write that records what happened would run on an already-expired context and be lost, leaving the
// book to be retried and the attempt to be spent again, forever. What must survive is the record.
func writeCtx(ctx context.Context) (context.Context, context.CancelFunc) {
	return context.WithTimeout(context.WithoutCancel(ctx), writeBudget)
}

// writeBudget is what a terminal write gets. Short: it is one statement against a database this
// process is already connected to.
const writeBudget = 30 * time.Second

// manifest asks the engine to cut the book, once its configuration is there to cut it against.
func (s *Service) manifest(ctx context.Context, workdir string) (ingest.Manifest, error) {
	if s.Engine == nil {
		return ingest.Manifest{}, errors.New("books: no engine is configured")
	}
	if _, err := os.Stat(workdir); err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			// Which of the two absences this is decides whether the book dies, so it is decided by the
			// SENTINEL rather than by the root's own existence (see storageIsThere).
			if !s.storageIsThere() {
				return ingest.Manifest{}, ErrStorageGone
			}
			return ingest.Manifest{}, ErrDirectoryGone
		}
		return ingest.Manifest{}, fmt.Errorf("books: read book directory: %w", err)
	}
	if _, err := os.Stat(filepath.Join(workdir, runner.ConfigFile)); err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return ingest.Manifest{}, ErrNotProvisioned
		}
		return ingest.Manifest{}, fmt.Errorf("books: read book configuration: %w", err)
	}
	return s.Engine.Manifest(ctx, s.Cfg.EngineBinary, workdir)
}

// refusedTheSource tells the engine's ANSWER from the engine's absence.
//
// The engine maps every failure that is not one of its two sentinels onto exit 1 (backend
// cmd/tmctl/main.go exitCode), so "the source cannot be cut" arrives as an ExitError — a process
// that ran and said no. Everything else here is this host: no binary at that path, no permission to
// execute it, a working directory that is gone, a context that expired. Those retry; an answer does
// not.
func refusedTheSource(err error) bool {
	var exit *exec.ExitError
	if !errors.As(err, &exit) {
		return false
	}
	// A signal is not an answer: an engine the machine killed said nothing about the book.
	return exit.Exited()
}

// reject records the terminal end of an intake and removes what is left of it.
//
// The source is deleted, and it is a decision rather than housekeeping: the file cannot be parsed,
// no path in the contract re-parses or downloads it, and an authenticated route that writes bytes to
// an operator's disk and never removes them is a hole this pack would otherwise be opening. The ROW
// stays — the user has to be able to see that the book they uploaded did not make it.
func (s *Service) reject(ctx context.Context, claim pgstore.ParseClaim, reason string) error {
	// The DIRECTORY goes first and the row second, and the order is the crash window: dying between
	// the two then leaves a book still `parsing` with no source, which the next attempt fails on and
	// rejects properly. The other order leaves a directory with a full source and a row that says
	// `rejected` — and nothing ever looks at a rejected book again, so those bytes stay forever.
	//
	// ⚠ Only where the source is the BOOK's fault. A deployment that could not run the engine keeps
	// the file: deleting a user's upload because this host was misconfigured is not a decision to
	// make on their behalf.
	if reason == ReasonSourceUnreadable {
		s.removeDir(claim.Workdir)
	}
	// On a context of its own: this write is the only record that the intake is over, and the engine
	// call that led here may have used up everything the pass had.
	c, cancel := writeCtx(ctx)
	defer cancel()
	return s.Store.RejectBook(c, claim.BookID, claim.At, reason)
}

// Sweep finishes the intake walk of every book whose own process did not.
//
// The two halves are the two ways a walk stops: a request that went away mid-upload, and a parse
// whose process is gone. Neither is reachable from the thing that started it, which is what makes
// this the only cure — the same reason the run reconciler exists.
//
// One book's failure never stops the sweep: these are independent books of independent accounts.
func (s *Service) Sweep(ctx context.Context) error {
	now := s.now()
	stuck, err := s.Store.StuckIntake(ctx, now.Add(-UploadGrace), now.Add(-claimGrace))
	if err != nil {
		return err
	}
	for _, b := range stuck {
		if err := ctx.Err(); err != nil {
			return err
		}
		switch b.Status {
		case "uploading":
			s.log().InfoContext(ctx, "removing an upload that never finished")
			s.abandon(ctx, b.ID, b.Workdir)
		case "parsing":
			// Each book gets its OWN budget, and it has to be one a parse can live inside: the engine
			// call is the same one the queue gives fifteen minutes. Sharing the pass's deadline meant a
			// large book was killed by it, the kill was read as a host that cannot run the engine, and
			// the attempt was spent — every pass, until the book was rejected for being big.
			//
			// ⚠ A budget the book gets is not a budget the PASS still has, and that gap was the rest of
			// the same defect (re-check of the dofix): the second book of a pass inherited whatever the
			// first one left, so a slow first parse handed the second a stub of a deadline and burned
			// its attempt on the timeout. A book that cannot be given its whole budget is therefore not
			// STARTED — the claim is taken inside Parse, so a book left for the next tick has spent
			// nothing — and the pass says so once rather than per book.
			if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < jobs.JobTimeout {
				s.log().InfoContext(ctx, "intake pass ends early: what is left of it is shorter than a parse",
					"remaining", time.Until(deadline).String())
				return nil
			}
			c, cancel := context.WithTimeout(ctx, jobs.JobTimeout)
			err := s.Parse(c, b.ID)
			cancel()
			if err != nil {
				s.log().ErrorContext(ctx, "book could not be parsed", "err", err)
			}
		}
	}
	return nil
}