textmachine/platform/internal/books/books.go

// Package books owns a book's INTAKE: receiving the file the user uploads, putting it where the
// engine will look for it, and turning it into a chapter tree.
//
// The shape mirrors the run lifecycle deliberately (package runs): the request only gets the book as
// far as "the bytes are here", and everything after that is a step some later sweep can finish. A
// parse is one $0 call of the engine that takes seconds on a large book, and holding a request open
// for it — or losing the book when the process doing it is restarted — are the two failures this
// split exists to avoid.
package books

import (
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"log/slog"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/pgstore"
)

// Manifester is the engine's $0 producer of a chapter tree: `tmctl manifest`. An interface so the
// intake's decisions can be pinned without an engine binary.
type Manifester interface {
	Manifest(ctx context.Context, binary, workdir string) (ingest.Manifest, error)
}

// Enqueuer hands a book to the queue inside the caller's transaction.
type Enqueuer interface {
	EnqueueParse(ctx context.Context, tx pgstore.Tx, bookID string) error
}

// Config is what an operator chooses about intake.
type Config struct {
	// BooksDir is the root the platform creates book directories under. Absolute, and NOT the place
	// an operator keeps hand-made books: everything below it is written and, on a rejected intake,
	// removed by this service.
	BooksDir string
	// EngineBinary is the versioned tmctl path used to parse. Intake is not mounted without it: a
	// deployment that cannot parse would take uploads it can only reject.
	EngineBinary string
}

// Service is the intake.
type Service struct {
	Store  *pgstore.Store
	Engine Manifester
	Queue  Enqueuer
	Cfg    Config
	Log    *slog.Logger
	// Now is injectable so the graces below are testable without sleeping.
	Now func() time.Time
}

func (s *Service) now() time.Time {
	if s.Now != nil {
		return s.Now()
	}
	return time.Now()
}

func (s *Service) log() *slog.Logger {
	if s.Log == nil {
		return slog.New(slog.DiscardHandler)
	}
	return s.Log
}

// ErrBadIntake is a request this route cannot make a book out of. It is the contract's 400.
var ErrBadIntake = errors.New("books: the intake form is not usable")

// SourceName is the file name the intake writes a book's source under, extension aside. The name is
// FIXED and predictable because the engine finds the source through `source_file:` in the book's own
// config, which somebody else writes (see the provisioning seam in parse.go).
const SourceName = "source"

// maxTitle bounds the title taken from the uploaded file's name. The library lists it, and the
// client-supplied name is otherwise the one unbounded string on that screen.
const maxTitle = 200

// langCode is the contract's LangCode: a code, never a name (§LangCode). Validated here as well as
// in the read model because this is where a value from a browser enters.
var langCode = regexp.MustCompile(`^[a-z]{2,3}(-[A-Za-z0-9]{2,8})*$`)

// Intake is one accepted call of POST /books.
type Intake struct {
	UserID     string
	SourceLang string
	TargetLang string
	Genre      string
	// Filename is the name the client gave the part. It is used for two things and trusted for
	// neither: a title for the library and the extension that tells the engine which reader to use.
	Filename string
	// File is the body of the upload, already bounded by the route's own limit. It is streamed to
	// disk and never held in memory.
	File io.Reader
}

// Accept receives one book.
//
// The order — row, then bytes — is what makes `uploading` a state anything can observe: a second tab
// listing the library while a 60 MB file is still on the wire sees the book with its languages and
// without its size. It also makes an abandoned upload FINDABLE, which is the half that matters
// operationally: the row is the only record that a directory under BooksDir belongs to anyone.
func (s *Service) Accept(ctx context.Context, in Intake) (pgstore.Book, error) {
	if !langCode.MatchString(in.SourceLang) || !langCode.MatchString(in.TargetLang) {
		return pgstore.Book{}, fmt.Errorf("%w: the languages must be codes", ErrBadIntake)
	}
	if s.Cfg.BooksDir == "" {
		return pgstore.Book{}, errors.New("books: no books directory is configured")
	}
	id := pgstore.NewBookID()
	dir := filepath.Join(s.Cfg.BooksDir, id)
	if err := os.MkdirAll(dir, 0o750); err != nil {
		return pgstore.Book{}, fmt.Errorf("books: create book directory: %w", err)
	}
	// The first upload is what marks this storage as ours — see storageIsThere. Written here and
	// nowhere else, and deliberately NOT at boot: a marker the boot recreates says "the storage is
	// here" about a directory the boot itself just made.
	if err := s.markStorage(); err != nil {
		s.removeDir(dir)
		return pgstore.Book{}, err
	}
	// The created row is not kept: what the caller gets back is the row as it stands AFTER the file
	// landed, and between the two the status has moved from `uploading` to `parsing`.
	_, err := s.Store.CreateUpload(ctx, id, pgstore.NewUpload{
		OwnerID:    in.UserID,
		Title:      titleFrom(in.Filename),
		SourceLang: in.SourceLang,
		TargetLang: in.TargetLang,
		Genre:      in.Genre,
		Workdir:    dir,
		Now:        s.now(),
	})
	if err != nil {
		s.removeDir(dir)
		return pgstore.Book{}, err
	}
	characters, err := s.receive(filepath.Join(dir, SourceName+extensionOf(in.Filename)), in.File)
	if err != nil {
		// The upload did not finish, so there is nothing to parse and nothing to keep. The row is
		// DELETED rather than rejected: `rejected` means the file could not be parsed (contract
		// §BookStatus), and there is no delete handle in the contract for the user to clear a row an
		// abandoned upload would otherwise leave in their library forever.
		//
		// On its own context: the ordinary cause of getting here is the client going away, and the
		// request's context is already cancelled by then.
		c, cancel := writeCtx(ctx)
		defer cancel()
		s.abandon(c, id, dir)
		return pgstore.Book{}, err
	}
	// On a context that outlives the request: every byte is in, and a client that hung up while
	// waiting for the 201 must not cost the upload it already finished. The ROW is what makes the
	// book findable, so it is written even when nobody is left to read the answer. Bounded on its own
	// (writeCtx): detached is not the same as unlimited, and a hung statement here would hold the
	// goroutine of a request that is already over.
	start, cancelStart := writeCtx(ctx)
	defer cancelStart()
	book, err := s.Store.StartParsing(start, id, characters, s.enqueue)
	if err != nil {
		// The row is gone or unreachable, and the directory holds a file nothing points at — which is
		// the one thing the row-first order exists to prevent, so it is undone here too. The ordinary
		// cause is the sweep having abandoned this upload while it was still arriving.
		c, cancel := writeCtx(ctx)
		defer cancel()
		s.abandon(c, id, dir)
		return pgstore.Book{}, err
	}
	// No book id: an INFO line must not identify a user's library (ENGINEERING_STANDARDS
	// §Наблюдаемость, the same rule that keeps raw paths out of the access log — PD-3). The request
	// id the handler carries is what ties this line to the response the user got.
	s.log().InfoContext(ctx, "book accepted", "characters", characters)
	return book, nil
}

// receive streams the upload to disk and counts what went past.
//
// Streamed, never buffered: the route's limit is tens of megabytes and reading that into memory
// would make one upload per concurrent request the platform's memory profile. The file is created
// with O_EXCL so a book directory can never be written twice by two requests that somehow minted the
// same id.
func (s *Service) receive(path string, body io.Reader) (int64, error) {
	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o640)
	if err != nil {
		return 0, fmt.Errorf("books: create source file: %w", err)
	}
	c := &counter{w: f}
	if _, err := io.Copy(c, body); err != nil {
		f.Close()
		// The error is returned as it came: the handler tells a body that outgrew the route's limit
		// from a client that went away, and both are the caller's to name (http.MaxBytesError).
		return 0, err
	}
	if err := f.Close(); err != nil {
		return 0, fmt.Errorf("books: close source file: %w", err)
	}
	return c.characters, nil
}

// counter writes through and counts CHARACTERS, which is what the contract's Book carries — a size
// in characters, known to the platform from intake and to nobody else (§Book.character_count).
//
// It counts every byte that is not a UTF-8 continuation byte, which is exactly the rune count of
// valid UTF-8 and needs no state across the chunk boundaries io.Copy hands it. ⚠ For a source in
// another encoding — the engine accepts GB18030 and UTF-16 and decodes them itself — the figure is
// an approximation of the character count, not the count. Naming a size for a file whose encoding is
// not yet known is what intake can do; the exact one would need the engine's decode, which happens a
// step later and reports bytes.
type counter struct {
	w          io.Writer
	characters int64
}

func (c *counter) Write(p []byte) (int, error) {
	n, err := c.w.Write(p)
	for _, b := range p[:n] {
		if b&0xC0 != 0x80 {
			c.characters++
		}
	}
	return n, err
}

func (s *Service) enqueue(ctx context.Context, tx pgstore.Tx, bookID string) error {
	if s.Queue == nil {
		return nil // no queue configured: the sweep picks the book up on its next pass
	}
	return s.Queue.EnqueueParse(ctx, tx, bookID)
}

// abandon undoes an upload that did not finish. Both halves are best effort and both are logged:
// what must not happen is a silent leak of either the row or the directory.
func (s *Service) abandon(ctx context.Context, id, dir string) {
	// The ROW is deleted first here, and unlike the reject path that order is forced: the directory
	// may only go once it is certain nobody owns it. The sweep decides from a snapshot, and a request
	// that finished in the meantime has moved the book to `parsing` — DeleteUpload's status guard then
	// refuses, and removing the directory anyway would delete the source of a live book under it.
	//
	// The crash window that leaves is a directory with no row, and it is named rather than closed:
	// nothing walks BooksDir looking for orphans (register row PD-175, where the retention sweep that
	// would is filed).
	if err := s.Store.DeleteUpload(ctx, id); err != nil {
		if errors.Is(err, pgstore.ErrNoBook) {
			// The ordinary race: another instance's sweep got there first, or the upload finished. Not
			// an error — an ERROR line on a routine race is noise that teaches operators to skim.
			s.log().InfoContext(ctx, "the abandoned upload was already gone; its directory is left alone")
			return
		}
		s.log().ErrorContext(ctx, "an abandoned upload was not removed; its directory is left alone", "err", err)
		return
	}
	s.removeDir(dir)
}

// removeDir deletes a directory this service created, and NOTHING else.
//
// The guard is not ceremony: a book registered by the dev CLI carries a workdir the operator chose —
// their own project directory, with their own source and their own project database — and no path
// here may ever remove one. Everything under BooksDir was created by this service and holds nothing
// the platform did not put there.
func (s *Service) removeDir(dir string) {
	if !s.owns(dir) {
		s.log().Error("refusing to remove a directory this service did not create", "dir", dir)
		return
	}
	if err := os.RemoveAll(dir); err != nil {
		s.log().Error("book directory could not be removed", "err", err)
	}
}

// StorageMarker is the file that says "this is the storage this platform has been writing books
// into". Exported so an operator provisioning a volume by hand can put one there.
const StorageMarker = ".tmplatform-books"

const storageMarkerText = `This directory holds book sources written by tmplatformd.
Its presence is what tells the intake that the storage is mounted: without it a book whose directory
is missing is treated as "the storage is gone" and WAITS, instead of being rejected with its source
deleted. Do not remove it while books live here.
`

// markStorage writes the marker if it is not there. Idempotent and race-free by O_EXCL: two uploads
// arriving together are both correct, and the loser's EEXIST is the state it wanted.
func (s *Service) markStorage() error {
	f, err := os.OpenFile(filepath.Join(s.Cfg.BooksDir, StorageMarker),
		os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o640)
	if errors.Is(err, fs.ErrExist) {
		return nil
	}
	if err != nil {
		return fmt.Errorf("books: mark the storage: %w", err)
	}
	defer f.Close()
	if _, err := f.WriteString(storageMarkerText); err != nil {
		return fmt.Errorf("books: mark the storage: %w", err)
	}
	return nil
}

// storageIsThere reports whether the books storage is the one this platform wrote into.
//
// ⚠ It asks about the MARKER and not about the directory, and the difference is the whole point. A
// volume mounted AT BooksDir leaves an empty mountpoint behind when it is unmounted, so the directory
// still exists and `Stat` still succeeds — which is how the first version of this guard (PD-192) let
// an unmount reject every book in intake, with their sources deleted, exactly as if each book's own
// directory had been removed. The boot's MkdirAll made it worse by recreating the root after a
// restart. The marker is written by the FIRST upload and by nothing else, so neither an unmount nor a
// fresh MkdirAll can forge it (re-check of the dofix, FP5-10).
//
// A host that has never taken an upload has no marker either, and answers "not there" — which is the
// safe direction: it has no books to reject.
func (s *Service) storageIsThere() bool {
	if s.Cfg.BooksDir == "" {
		return false
	}
	_, err := os.Stat(filepath.Join(s.Cfg.BooksDir, StorageMarker))
	return err == nil
}

func (s *Service) owns(dir string) bool {
	if s.Cfg.BooksDir == "" || dir == "" {
		return false
	}
	rel, err := filepath.Rel(s.Cfg.BooksDir, dir)
	if err != nil {
		return false
	}
	return rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}

// titleFrom is the book's name until something better exists.
//
// ⚠ Named limitation, not a design: `BookIntake` carries no title field (contract §BookIntake) and
// inventing one is not this zone's right, while the engine's manifest reports counts and identity
// and no title either. What is left is the name the user gave the file, which is a real thing they
// chose. An empty result stays empty: a synthesized "Book 1" would be a label nobody wrote.
func titleFrom(filename string) string {
	name := filepath.Base(filepath.FromSlash(filename))
	if i := strings.LastIndexByte(name, '.'); i > 0 {
		name = name[:i]
	}
	name = strings.Map(func(r rune) rune {
		if unicode.IsControl(r) {
			return -1
		}
		return r
	}, name)
	name = strings.TrimSpace(name)
	if name == "." || name == ".." || name == string(filepath.Separator) {
		return ""
	}
	if utf8.RuneCountInString(name) > maxTitle {
		name = string([]rune(name)[:maxTitle])
	}
	return name
}

// extensionOf is the one thing about the FORMAT the platform is allowed to know: the engine
// dispatches its reader by extension (`.epub` → the epub reader, anything else → plain text,
// backend/internal/chunk/ingest.go), so the extension has to survive intake or an EPUB is read as
// text.
//
// It is not a format allowlist, and that is deliberate: which formats exist is the engine's
// question, and a platform that refused an extension the engine had just learned would be a second
// place to teach. What it does refuse is a name that is not an extension — the value goes into a
// path, so anything but lowercase alphanumerics is dropped and the source becomes plain text.
func extensionOf(filename string) string {
	ext := strings.ToLower(filepath.Ext(filepath.Base(filepath.FromSlash(filename))))
	if len(ext) < 2 || len(ext) > 9 {
		return ".txt"
	}
	for _, r := range ext[1:] {
		if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
			return ".txt"
		}
	}
	return ext
}