textmachine/platform/internal/books/books_test.go

package books

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"log/slog"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/jackc/pgx/v5"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/jobs"
	"textmachine/platform/internal/pgstore"
	"textmachine/platform/internal/runner"
)

// Intake is a walk with three ends — parsed, rejected, or removed — and every one of them is reached
// by a DIFFERENT process from the one that started it. That is what these tests are about: the row,
// the directory and the file agree at each step, and no step is reachable only from the request that
// began it.
func intakeDB(t *testing.T) (*pgstore.Store, context.Context) {
	t.Helper()
	admin := os.Getenv("TM_PLATFORM_TEST_DSN")
	if admin == "" {
		t.Skip("TM_PLATFORM_TEST_DSN not set: the intake's statuses need a live Postgres")
	}
	ctx := t.Context()
	var suffix [6]byte
	if _, err := rand.Read(suffix[:]); err != nil {
		t.Fatal(err)
	}
	name := "tm_books_test_" + hex.EncodeToString(suffix[:])
	conn, err := pgx.Connect(ctx, admin)
	if err != nil {
		t.Fatalf("connect: %v", err)
	}
	if _, err := conn.Exec(ctx, "create database "+pgx.Identifier{name}.Sanitize()); err != nil {
		conn.Close(ctx)
		t.Skipf("cannot create a scratch database (%v): grant CREATEDB or point the DSN at one", err)
	}
	t.Cleanup(func() {
		c, cancel := context.WithTimeout(context.Background(), 15*time.Second)
		defer cancel()
		_, _ = conn.Exec(c, "drop database if exists "+pgx.Identifier{name}.Sanitize()+" with (force)")
		conn.Close(c)
	})
	u, err := url.Parse(admin)
	if err != nil {
		t.Fatal(err)
	}
	u.Path = "/" + name
	if err := pgstore.Migrate(ctx, u.String()); err != nil {
		t.Fatalf("migrate: %v", err)
	}
	s, err := pgstore.Open(ctx, u.String())
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(s.Close)
	if _, err := s.Pool().Exec(ctx, `insert into users (id, email) values ('u1','u1@example.org')`); err != nil {
		t.Fatal(err)
	}
	return s, ctx
}

// fakeEngine stands in for `tmctl manifest`. It models the two answers that lead to different ends —
// a manifest, and a process that RAN and refused — because a fake that can only succeed proves
// nothing about the branch that rejects a book.
type fakeEngine struct {
	mu       sync.Mutex
	manifest ingest.Manifest
	err      error
	calls    int
	// onManifest runs INSIDE the call, which is where the deadline the caller granted is visible.
	onManifest func(ctx context.Context)
}

func (f *fakeEngine) Manifest(ctx context.Context, _, _ string) (ingest.Manifest, error) {
	f.mu.Lock()
	f.calls++
	hook := f.onManifest
	m, err := f.manifest, f.err
	f.mu.Unlock()
	if hook != nil {
		hook(ctx)
	}
	return m, err
}

func (f *fakeEngine) set(m ingest.Manifest, err error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.manifest, f.err = m, err
}

func (f *fakeEngine) called() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.calls
}

// refusal is what the engine's own "no" looks like from here: a process that ran and exited non-zero
// (backend cmd/tmctl maps every failure that is not one of its two sentinels onto exit 1). Produced
// by running a real command rather than by hand-building an ExitError, so the type the classifier
// keys on is the type a real refusal carries.
func refusal(t *testing.T) error {
	t.Helper()
	err := exec.CommandContext(t.Context(), "/bin/sh", "-c", "exit 1").Run()
	if err == nil {
		t.Fatal("the refusal helper did not fail")
	}
	return err
}

type fixture struct {
	svc    *Service
	store  *pgstore.Store
	engine *fakeEngine
	ctx    context.Context
	root   string
	now    time.Time
}

func newFixture(t *testing.T) *fixture {
	t.Helper()
	store, ctx := intakeDB(t)
	root := t.TempDir()
	eng := &fakeEngine{manifest: ingest.Manifest{
		Version: "tm-manifest-v2", ChaptersTotal: 500, UnitsTotal: 950,
		SourceSHA256: strings.Repeat("ab", 32), ChunkerVersion: "chunk-2026.07",
	}}
	now := time.Now().UTC().Truncate(time.Millisecond)
	f := &fixture{store: store, engine: eng, ctx: ctx, root: root, now: now}
	f.svc = &Service{
		Store:  store,
		Engine: eng,
		Cfg:    Config{BooksDir: root, EngineBinary: "/opt/engine/2026.08.01/tmctl"},
		Now:    func() time.Time { return f.now },
	}
	return f
}

// accept runs one upload of the given bytes and returns the book.
func (f *fixture) accept(t *testing.T, filename, body string) pgstore.Book {
	t.Helper()
	b, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
		Genre: "xianxia", Filename: filename, File: strings.NewReader(body)})
	if err != nil {
		t.Fatal(err)
	}
	return b
}

// provision puts an engine configuration in the book's directory. Who does this in production is the
// open question the intake stops at (ErrNotProvisioned); here it stands for the operator.
func (f *fixture) provision(t *testing.T, b pgstore.Book) string {
	t.Helper()
	dir := filepath.Join(f.root, b.ID)
	if err := os.WriteFile(filepath.Join(dir, runner.ConfigFile), []byte("book_id: x\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	return dir
}

func (f *fixture) libraryRevision(t *testing.T) int64 {
	t.Helper()
	lib, err := f.store.ListBooks(f.ctx, "u1", 50, "")
	if err != nil {
		t.Fatal(err)
	}
	return lib.Revision
}

func (f *fixture) card(t *testing.T, id string) pgstore.Book {
	t.Helper()
	b, _, err := f.store.GetBook(f.ctx, "u1", id)
	if err != nil {
		t.Fatal(err)
	}
	return b
}

// The whole walk, and the assertion is that each status is written by somebody: the row exists while
// the file arrives, the engine turns it into a chapter tree, and the library reads the result exactly
// as it reads a book the dev intake put there.
func TestAnUploadedBookWalksFromUploadingToNotStarted(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "蛊真人.txt", "первая глава\fвторая глава")
	if book.Status != "parsing" {
		t.Fatalf("the response says %q; the file is in, so the book is being parsed", book.Status)
	}
	if book.Title != "蛊真人" {
		t.Fatalf("title %q, want the name the user gave the file", book.Title)
	}
	// Characters, not bytes: the contract's figure is a size in characters and this source is Cyrillic.
	if book.CharacterCount != 25 {
		t.Fatalf("character_count %d, want 25 runes", book.CharacterCount)
	}
	f.provision(t, book)
	if err := f.svc.Parse(f.ctx, book.ID); err != nil {
		t.Fatal(err)
	}
	got := f.card(t, book.ID)
	if got.Status != "not_started" || got.ChapterCount != 500 {
		t.Fatalf("after parsing: %q with %d chapters, want not_started with 500", got.Status, got.ChapterCount)
	}
	// The manifest's identity is kept: it says WHICH cut of WHICH bytes produced that chapter count.
	var sha []byte
	var chunker string
	if err := f.store.Pool().QueryRow(f.ctx,
		`select source_sha256, chunker_version from books where id = $1`, book.ID).Scan(&sha, &chunker); err != nil {
		t.Fatal(err)
	}
	if len(sha) != 32 || chunker != "chunk-2026.07" {
		t.Fatalf("the manifest's provenance was not recorded: sha %d bytes, chunker %q", len(sha), chunker)
	}
}

// `uploading` is a state something can observe, and that is the whole reason the row is written
// before the body is read: a second reader of the library sees the book while its file is still on
// the wire.
func TestABookIsVisibleAsUploadingWhileItsFileIsStillArriving(t *testing.T) {
	f := newFixture(t)
	reading := make(chan struct{})
	release := make(chan struct{})
	body := &blockingReader{data: "the whole book", gate: reading, release: release}
	done := make(chan error, 1)
	go func() {
		_, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
			Filename: "book.txt", File: body})
		done <- err
	}()
	<-reading
	lib, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	if len(lib.Books) != 1 || lib.Books[0].Status != "uploading" {
		t.Fatalf("the library shows %+v while the file is arriving, want one book in uploading", lib.Books)
	}
	close(release)
	if err := <-done; err != nil {
		t.Fatal(err)
	}
	if got := f.card(t, lib.Books[0].ID); got.Status != "parsing" {
		t.Fatalf("after the body arrived the book is %q, want parsing", got.Status)
	}
}

// blockingReader hands over its first chunk and then waits, so a test can look at the world in the
// middle of an upload.
type blockingReader struct {
	data    string
	gate    chan struct{}
	release chan struct{}
	sent    bool
}

func (b *blockingReader) Read(p []byte) (int, error) {
	if !b.sent {
		b.sent = true
		n := copy(p, b.data)
		close(b.gate)
		<-b.release
		return n, nil
	}
	return 0, io.EOF
}

// An upload that did not finish leaves NOTHING: not a row the user cannot delete — the contract has
// no handle for that — and not a directory nothing points at.
//
// The client going AWAY is the ordinary way to get here, so that is what the test does: the request's
// own context is cancelled before the body fails. Cleaning up on it would then clean up nothing.
func TestAnAbandonedUploadLeavesNeitherARowNorADirectory(t *testing.T) {
	f := newFixture(t)
	ctx, cancel := context.WithCancel(f.ctx)
	defer cancel()
	_, err := f.svc.Accept(ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
		Filename: "book.txt", File: &failingReader{cancel: cancel}})
	if err == nil {
		t.Fatal("an upload whose body failed was accepted")
	}
	lib, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	if len(lib.Books) != 0 {
		t.Fatalf("the library holds %+v after a failed upload", lib.Books)
	}
	entries, err := os.ReadDir(f.root)
	if err != nil {
		t.Fatal(err)
	}
	// Everything except the storage marker, which is the root's own and outlives every book: it is
	// written by the first upload and never removed, because its absence is what tells the intake the
	// volume is gone (FP5-10).
	var left []string
	for _, e := range entries {
		if e.Name() != StorageMarker {
			left = append(left, e.Name())
		}
	}
	if len(left) != 0 {
		t.Fatalf("the books root holds %v after a failed upload", left)
	}
}

// failingReader is a client that goes away mid-body. gate/release, when set, let a test look at the
// world while the upload is still in flight.
type failingReader struct {
	cancel  context.CancelFunc
	gate    chan struct{}
	release chan struct{}
}

func (f failingReader) Read([]byte) (int, error) {
	if f.gate != nil {
		close(f.gate)
		<-f.release
	}
	if f.cancel != nil {
		f.cancel()
	}
	return 0, errors.New("the client went away")
}

// A source the engine READ and refused is terminal, and its bytes go: nothing re-parses them, no
// path downloads them, and an authenticated route that writes to an operator's disk and never
// removes anything is a hole this one must not open.
func TestASourceTheEngineRefusesIsRejectedAndItsFileRemoved(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "not-a-book.pdf", "%PDF-1.7 …")
	dir := f.provision(t, book)
	f.engine.set(ingest.Manifest{}, refusal(t))
	// ⚠ The FIRST refusal is not terminal, and that is the correction the cross-family review forced:
	// the engine maps every failure it has onto exit 1 — a full disk, an operator's own tmctl holding
	// the project lock, a claim taken over mid-parse — so rejecting on the first one turned any of
	// those into the irreversible deletion of a user's upload. It spends an attempt instead.
	if err := f.svc.Parse(f.ctx, book.ID); err != nil {
		t.Fatal(err)
	}
	if got := f.card(t, book.ID); got.Status != "parsing" {
		t.Fatalf("one refusal made the book %q; a single exit 1 must not be terminal", got.Status)
	}
	if _, err := os.Stat(filepath.Join(dir, SourceName+".pdf")); err != nil {
		t.Fatalf("the source was deleted on the first refusal: %v", err)
	}
	for range parseAttempts {
		f.now = f.now.Add(claimGrace + time.Minute)
		if err := f.svc.Parse(f.ctx, book.ID); err != nil {
			t.Fatal(err)
		}
	}
	got := f.card(t, book.ID)
	if got.Status != "rejected" {
		t.Fatalf("a source the engine refused is %q, want rejected", got.Status)
	}
	if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("the rejected book's directory is still there: %v", err)
	}
	// The reason is the platform's own word and is kept for an operator — contract v0 has no field
	// for it, so it is deliberately not on the wire.
	var reason string
	if err := f.store.Pool().QueryRow(f.ctx,
		`select reject_reason from books where id = $1`, book.ID).Scan(&reason); err != nil {
		t.Fatal(err)
	}
	if reason != ReasonSourceUnreadable {
		t.Fatalf("reject_reason %q, want %q", reason, ReasonSourceUnreadable)
	}
}

// An engine that cannot be RUN is this host's problem and not this book's. The parse is retried —
// and bounded, because a book that waits forever is the failure this must not trade for (PD-162).
func TestAnEngineThatCannotBeRunIsRetriedAndThenGivenUpOn(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "book.txt", "text")
	f.provision(t, book)
	f.engine.set(ingest.Manifest{}, &exec.Error{Name: "tmctl", Err: os.ErrNotExist})
	// ⚠ The clock moves between attempts because the CLAIM is what spaces them: a failed parse keeps
	// it, so the book is not claimable again until the grace passes. Without that the five attempts
	// would burn in five ticks of the sweep — 75 seconds — instead of in the time this budget is
	// meant to bound, and an engine outage of a couple of minutes would reject every book waiting.
	for i := 1; i < parseAttempts; i++ {
		if err := f.svc.Parse(f.ctx, book.ID); err != nil {
			t.Fatal(err)
		}
		if got := f.card(t, book.ID); got.Status != "parsing" {
			t.Fatalf("attempt %d left the book %q, want it still parsing", i, got.Status)
		}
		// The very next call, before the grace, must do NOTHING at all.
		before := f.engine.called()
		if err := f.svc.Parse(f.ctx, book.ID); err != nil {
			t.Fatal(err)
		}
		if f.engine.called() != before {
			t.Fatalf("attempt %d was retried before the grace passed", i)
		}
		f.now = f.now.Add(claimGrace + time.Minute)
	}
	if err := f.svc.Parse(f.ctx, book.ID); err != nil {
		t.Fatal(err)
	}
	got := f.card(t, book.ID)
	if got.Status != "rejected" {
		t.Fatalf("after %d attempts the book is %q, want rejected", parseAttempts, got.Status)
	}
	// The file STAYS: the fault was the deployment's, and deleting a user's upload because this host
	// was misconfigured is not a decision to make on their behalf.
	if _, err := os.Stat(filepath.Join(f.root, book.ID, SourceName+".txt")); err != nil {
		t.Fatalf("the source of a book rejected for a deployment fault was removed: %v", err)
	}
}

// The seam of the open question: a book directory with no engine configuration cannot be parsed, and
// the platform does not write that file (D39.110 §2b).
//
// ⚠ This test asserts the OPPOSITE of what it did before the cross-family review, and the reversal is
// deliberate. It used to pin "five attempts, then rejected" — which meant that on any deployment
// without an answer to the `book.yaml` question (that is, every deployment today) EVERY upload was
// destroyed within the hour, over a gap the user cannot see and an operator can fix in a second.
// A missing configuration is a DEPLOYMENT state, so the book waits in `parsing`, visibly, with its
// source intact, and one file dropped in is all it takes.
func TestABookWithNoEngineConfigurationWaitsRatherThanDies(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "book.txt", "text")
	for range parseAttempts * 2 {
		if err := f.svc.Parse(f.ctx, book.ID); err != nil {
			t.Fatal(err)
		}
		f.now = f.now.Add(claimGrace + time.Minute) // the claim spaces the retries
	}
	if f.engine.called() != 0 {
		t.Fatalf("the engine was asked %d times about a book it has no configuration for", f.engine.called())
	}
	var status, reason string
	if err := f.store.Pool().QueryRow(f.ctx,
		`select status, reject_reason from books where id = $1`, book.ID).Scan(&status, &reason); err != nil {
		t.Fatal(err)
	}
	if status != "parsing" || reason != "" {
		t.Fatalf("a book with no configuration is %q/%q; it must WAIT, not be destroyed", status, reason)
	}
	if _, err := os.Stat(filepath.Join(f.root, book.ID, SourceName+".txt")); err != nil {
		t.Fatalf("the source of a book waiting for its configuration was removed: %v", err)
	}
	// And the moment somebody provisions it, the very next pass parses it.
	f.provision(t, pgstore.Book{ID: book.ID})
	f.now = f.now.Add(claimGrace + time.Minute)
	if err := f.svc.Parse(f.ctx, book.ID); err != nil {
		t.Fatal(err)
	}
	if got := f.card(t, book.ID); got.Status != "not_started" {
		t.Fatalf("after provisioning the book is %q, want not_started", got.Status)
	}
}

// Two parsers on one book would be two writers of the engine's project file. The claim is a
// compare-and-set, and losing that race is the ordinary case rather than a failure.
func TestOnlyOneOfTwoParsersOfOneBookAsksTheEngine(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "book.txt", "text")
	f.provision(t, book)
	var wg sync.WaitGroup
	errs := make([]error, 8)
	for i := range errs {
		wg.Add(1)
		go func() {
			defer wg.Done()
			errs[i] = f.svc.Parse(f.ctx, book.ID)
		}()
	}
	wg.Wait()
	for i, err := range errs {
		if err != nil {
			t.Errorf("parser %d: %v", i, err)
		}
	}
	if n := f.engine.called(); n != 1 {
		t.Fatalf("the engine was asked %d times for one book, want exactly 1", n)
	}
}

// The backstop: both halves of a stuck intake are reachable only from the sweep, because the process
// that could have finished them is the one that died.
func TestTheSweepFinishesAWalkNobodyElseCan(t *testing.T) {
	f := newFixture(t)
	stale := f.accept(t, "abandoned.txt", "half a file")
	// This one never got past `uploading`: its request is gone.
	if _, err := f.store.Pool().Exec(f.ctx,
		`update books set status = 'uploading' where id = $1`, stale.ID); err != nil {
		t.Fatal(err)
	}
	waiting := f.accept(t, "waiting.txt", "text")
	f.provision(t, waiting)
	f.now = f.now.Add(2 * time.Hour) // past both graces
	if err := f.svc.Sweep(f.ctx); err != nil {
		t.Fatal(err)
	}
	lib, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	if len(lib.Books) != 1 || lib.Books[0].ID != waiting.ID {
		t.Fatalf("the library holds %+v, want only the book that finished uploading", lib.Books)
	}
	if got := f.card(t, waiting.ID); got.Status != "not_started" {
		t.Fatalf("the waiting book is %q, want the sweep to have parsed it", got.Status)
	}
	if _, err := os.Stat(filepath.Join(f.root, stale.ID)); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("the abandoned upload's directory survived the sweep: %v", err)
	}
}

// Adding a book must MOVE the library's revision. The contract asks a client to drop a read whose
// revision is not above what it applied, so a book that joins at a number the library already had is
// a book the screen that uploaded it never sees (register row PD-122).
func TestABookThatJoinsTheLibraryMovesItsRevision(t *testing.T) {
	f := newFixture(t)
	before, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	first := f.accept(t, "one.txt", "text")
	after, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	if after.Revision <= before.Revision {
		t.Fatalf("the library revision stayed at %d after a book joined it", after.Revision)
	}
	f.provision(t, first)
	if err := f.svc.Parse(f.ctx, first.ID); err != nil {
		t.Fatal(err)
	}
	parsed, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	if parsed.Revision <= after.Revision {
		t.Fatalf("the library revision stayed at %d when a book's status changed", parsed.Revision)
	}
}

// A book the DEV intake registered carries a workdir the operator chose — their own project
// directory, with their own source in it. No path here may remove one.
func TestADirectoryOutsideTheBooksRootIsNeverRemoved(t *testing.T) {
	f := newFixture(t)
	outside := t.TempDir()
	if err := os.WriteFile(filepath.Join(outside, "irreplaceable.txt"), []byte("x"), 0o600); err != nil {
		t.Fatal(err)
	}
	f.svc.removeDir(outside)
	if _, err := os.Stat(filepath.Join(outside, "irreplaceable.txt")); err != nil {
		t.Fatalf("a directory outside the books root was removed: %v", err)
	}
	for _, dir := range []string{f.root, filepath.Join(f.root, ".."), "", "/"} {
		if f.svc.owns(dir) {
			t.Errorf("owns(%q) is true; only paths BELOW the root are this service's", dir)
		}
	}
	if !f.svc.owns(filepath.Join(f.root, "bk_1")) {
		t.Error("a book directory under the root is not recognised as this service's")
	}
}

// The two things the intake takes from a name it did not write.
func TestWhatTheUploadedFileNameIsAllowedToDecide(t *testing.T) {
	titles := []struct{ in, want string }{
		{"蛊真人.epub", "蛊真人"},
		{"/etc/passwd", "passwd"},
		{"../../secret.txt", "secret"},
		{"", ""},
		{"no-extension", "no-extension"},
		{".hidden", ".hidden"},
		{strings.Repeat("x", 400) + ".txt", strings.Repeat("x", maxTitle)},
		{"line\nbreak.txt", "linebreak"},
	}
	for _, tc := range titles {
		if got := titleFrom(tc.in); got != tc.want {
			t.Errorf("titleFrom(%q) = %q, want %q", tc.in, got, tc.want)
		}
	}
	exts := []struct{ in, want string }{
		{"book.epub", ".epub"},
		{"BOOK.EPUB", ".epub"},
		{"book.txt", ".txt"},
		{"book", ".txt"},
		{"book.", ".txt"},
		{"book.tar.gz", ".gz"},
		{"evil.sh;rm -rf", ".txt"},
		{"evil../../x", ".txt"},
		{"book.verylongextension", ".txt"},
	}
	for _, tc := range exts {
		if got := extensionOf(tc.in); got != tc.want {
			t.Errorf("extensionOf(%q) = %q, want %q", tc.in, got, tc.want)
		}
	}
}

// The languages are CODES and they enter here from a browser.
func TestALanguageThatIsNotACodeIsRefused(t *testing.T) {
	f := newFixture(t)
	for _, bad := range []string{"", "Chinese", "zh_CN", "ZH", "../zh", "z"} {
		_, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: bad, TargetLang: "ru",
			Filename: "b.txt", File: strings.NewReader("x")})
		if !errors.Is(err, ErrBadIntake) {
			t.Errorf("source_lang %q: %v, want ErrBadIntake", bad, err)
		}
	}
	if entries, err := os.ReadDir(f.root); err != nil || len(entries) != 0 {
		t.Fatalf("a refused intake left %v in the books root (%v)", entries, err)
	}
}

// The engine dispatches its reader by EXTENSION (backend/internal/chunk/ingest.go: `.epub` → the
// epub reader, anything else → plain text), so the extension has to survive intake or an EPUB is
// read as text and the book comes out as one chapter of markup.
func TestTheUploadedFilesExtensionSurvivesIntake(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "蛊真人.EPUB", "PK\x03\x04 not really an epub")
	if _, err := os.Stat(filepath.Join(f.root, book.ID, SourceName+".epub")); err != nil {
		t.Fatalf("the source was not written as source.epub: %v", err)
	}
}

// The zone's log discipline (ENGINEERING_STANDARDS §Наблюдаемость, PD-3): an INFO line does not
// identify a user's library. The intake writes several, so this asserts the whole walk at once.
func TestNoBookIdentifierReachesAnInfoLine(t *testing.T) {
	f := newFixture(t)
	var buf bytes.Buffer
	f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
	book := f.accept(t, "book.txt", "text")
	f.provision(t, book)
	if err := f.svc.Parse(f.ctx, book.ID); err != nil {
		t.Fatal(err)
	}
	f.now = f.now.Add(2 * time.Hour)
	if err := f.svc.Sweep(f.ctx); err != nil {
		t.Fatal(err)
	}
	for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") {
		if line == "" {
			continue
		}
		var rec map[string]any
		if err := json.Unmarshal([]byte(line), &rec); err != nil {
			t.Fatal(err)
		}
		if rec["level"] != "INFO" && rec["level"] != "WARN" {
			continue // an ERROR may carry the path an operator needs — the open class of PD-139
		}
		if strings.Contains(line, book.ID) {
			t.Errorf("a book id reached a %s line: %s", rec["level"], line)
		}
	}
}

// The library's revision may never go BACKWARDS, and an abandoned upload is the one deletion this
// zone has: the book that joined at the top of the account's scale is removed when its request goes
// away, and the maximum would fall with it. A client that had applied the higher number must then, by
// the contract, drop every later read of the library — the screen keeps rendering a book that is gone.
func TestCancellingAnUploadDoesNotWindTheLibraryBack(t *testing.T) {
	f := newFixture(t)
	kept := f.accept(t, "kept.txt", "text")
	f.provision(t, kept)
	before, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	// The number that matters is the one a CLIENT could have seen — the library WHILE the doomed
	// upload was in it, not the library before it arrived. Read from inside the upload, which is
	// exactly where a second tab would read it.
	reading, release := make(chan struct{}), make(chan struct{})
	ctx, cancel := context.WithCancel(f.ctx)
	defer cancel()
	done := make(chan error, 1)
	go func() {
		_, err := f.svc.Accept(ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
			Filename: "abandoned.txt",
			File:     &failingReader{cancel: cancel, gate: reading, release: release}})
		done <- err
	}()
	<-reading
	during, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	if during.Revision <= before.Revision {
		t.Fatalf("the upload joined the library at revision %d, which is not above %d",
			during.Revision, before.Revision)
	}
	close(release)
	if err := <-done; err == nil {
		t.Fatal("an upload whose body failed was accepted")
	}
	after, err := f.store.ListBooks(f.ctx, "u1", 10, "")
	if err != nil {
		t.Fatal(err)
	}
	if after.Revision < during.Revision {
		t.Fatalf("the library revision went back from %d to %d when the upload was cancelled",
			during.Revision, after.Revision)
	}
	if len(after.Books) != 1 || after.Books[0].ID != kept.ID {
		t.Fatalf("the library holds %+v, want only the book that stayed", after.Books)
	}
}

// The sweep decides from a snapshot too, and its cure is the same: the directory of a book goes only
// when the ROW went with it. A request that finished while the sweep was looking moves the book to
// `parsing`, DeleteUpload's status guard then refuses — and removing the directory anyway would
// delete the source of a live book, leaving one that can only end rejected and an hour of somebody's
// upload gone without a word.
func TestTheSweepNeverRemovesTheSourceOfABookItCouldNotDelete(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "book.txt", "text") // already `parsing`: its upload finished
	f.svc.abandon(f.ctx, book.ID, filepath.Join(f.root, book.ID))
	if got := f.card(t, book.ID); got.Status != "parsing" {
		t.Fatalf("the row is %q, want the parsing book it became", got.Status)
	}
	if _, err := os.Stat(filepath.Join(f.root, book.ID, SourceName+".txt")); err != nil {
		t.Fatalf("the source of a book that was NOT deleted was removed: %v", err)
	}
}

// The two constants are a PAIR and neither package can see the other's reasoning, so the pair is
// asserted here. A claim grace shorter than the queue's job timeout means the backstop sweep takes a
// parse away from the worker that is still running it: both then meet on one project directory, the
// loser dies on the engine's exclusive lock with exit 1 — and exit 1 is how the engine says "I cannot
// cut this source".
func TestTheClaimGraceOutlivesTheQueuesJobTimeout(t *testing.T) {
	if claimGrace <= jobs.JobTimeout {
		t.Fatalf("claimGrace %s does not outlive the queue's job timeout %s: the sweep would steal a claim from a running parse",
			claimGrace, jobs.JobTimeout)
	}
}

// Every byte is in and the client hung up while waiting for the 201 — a phone that went to sleep, a
// tab that was closed. The book must still be theirs: the row is what makes it findable at all, and
// throwing away a completed upload because nobody is left to read the answer costs the user the whole
// file and the whole wait.
func TestAnUploadSurvivesAClientThatHangsUpAfterTheLastByte(t *testing.T) {
	f := newFixture(t)
	ctx, cancel := context.WithCancel(f.ctx)
	book, err := f.svc.Accept(ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
		Filename: "book.txt", File: &hangUpReader{data: "первая глава", cancel: cancel}})
	if err != nil {
		t.Fatalf("an upload whose client left after the last byte was dropped: %v", err)
	}
	if book.Status != "parsing" {
		t.Fatalf("the book is %q, want parsing", book.Status)
	}
	if _, err := os.Stat(filepath.Join(f.root, book.ID, SourceName+".txt")); err != nil {
		t.Fatalf("the received source is gone: %v", err)
	}
}

// hangUpReader delivers the whole body and then cancels the request's context, which is what a client
// that walks away after sending everything looks like from here.
type hangUpReader struct {
	data   string
	cancel context.CancelFunc
	sent   bool
}

func (h *hangUpReader) Read(p []byte) (int, error) {
	if !h.sent {
		h.sent = true
		return copy(p, h.data), nil
	}
	h.cancel()
	return 0, io.EOF
}

// FP5-1 (acceptance, HIGH). A library revision must identify ONE state of the library. The floor a
// cancelled upload raises is only half the rule: if the next book joins from the maximum alone it
// lands at or below that floor, and the same number then answers for three different libraries —
// before the cancelled upload, during it, and after the next one arrived.
func TestABookThatJoinsAfterACancelledUploadStillMovesTheRevision(t *testing.T) {
	f := newFixture(t)
	first := f.accept(t, "kept.txt", "text")
	f.provision(t, first)
	before := f.libraryRevision(t)

	ctx, cancel := context.WithCancel(f.ctx)
	defer cancel()
	if _, err := f.svc.Accept(ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
		Filename: "abandoned.txt", File: &failingReader{cancel: cancel}}); err == nil {
		t.Fatal("an upload whose body failed was accepted")
	}
	afterCancel := f.libraryRevision(t)
	if afterCancel < before {
		t.Fatalf("the revision went back after a cancelled upload: %d → %d", before, afterCancel)
	}
	next := f.accept(t, "next.txt", "text")
	afterNext := f.libraryRevision(t)
	if afterNext <= afterCancel {
		t.Fatalf("a book that joined after a cancelled upload left the revision at %d (was %d): the same number now answers for two different libraries",
			afterNext, afterCancel)
	}
	// And the book itself carries a revision above the floor, so its own card is not stale on arrival.
	var rev int64
	if err := f.store.Pool().QueryRow(f.ctx, `select revision from books where id = $1`, next.ID).Scan(&rev); err != nil {
		t.Fatal(err)
	}
	if rev <= afterCancel {
		t.Fatalf("the new book joined at revision %d, at or below the floor %d", rev, afterCancel)
	}
}

// FP5-4 (acceptance). Waiting for a configuration must not bring DELETION closer: every claim counted
// against the budget, so a book that waited out five graces had its budget spent before the engine
// was asked once — and the first answer, which the engine gives as exit 1 for a typo in `book.yaml`
// as much as for an unreadable file, was terminal on arrival and took the source with it.
func TestWaitingForAConfigurationDoesNotBringDeletionCloser(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "book.txt", "text")
	for range parseAttempts * 3 {
		if err := f.svc.Parse(f.ctx, book.ID); err != nil {
			t.Fatal(err)
		}
		f.now = f.now.Add(claimGrace + time.Minute)
	}
	var attempts int
	if err := f.store.Pool().QueryRow(f.ctx,
		`select parse_attempts from books where id = $1`, book.ID).Scan(&attempts); err != nil {
		t.Fatal(err)
	}
	if attempts != 0 {
		t.Fatalf("waiting for a configuration spent %d attempts of the budget", attempts)
	}
	// Now it is provisioned and the engine refuses the source once: with the budget intact, that first
	// answer must NOT be terminal and must NOT delete anything.
	f.provision(t, pgstore.Book{ID: book.ID})
	f.engine.set(ingest.Manifest{}, refusal(t))
	if err := f.svc.Parse(f.ctx, book.ID); err != nil {
		t.Fatal(err)
	}
	if got := f.card(t, book.ID); got.Status != "parsing" {
		t.Fatalf("the first engine refusal after a long wait made the book %q", got.Status)
	}
	if _, err := os.Stat(filepath.Join(f.root, book.ID, SourceName+".txt")); err != nil {
		t.Fatalf("the source was deleted on the first refusal after a wait: %v", err)
	}
}

// FP5-3 (acceptance). The rejection path removes the directory BEFORE it writes the row, so a crash
// between the two leaves a book in `parsing` with nothing on disk. That must be terminal — otherwise
// the missing directory reads as "not configured", which is the one reason that never ends, and the
// book waits for a configuration nobody can put anywhere.
func TestABookWhoseDirectoryIsGoneIsRejectedRatherThanLeftWaiting(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "book.txt", "text")
	f.provision(t, book)
	if err := os.RemoveAll(filepath.Join(f.root, book.ID)); err != nil {
		t.Fatal(err)
	}
	if err := f.svc.Parse(f.ctx, book.ID); err != nil {
		t.Fatal(err)
	}
	var status, reason string
	if err := f.store.Pool().QueryRow(f.ctx,
		`select status, reject_reason from books where id = $1`, book.ID).Scan(&status, &reason); err != nil {
		t.Fatal(err)
	}
	if status != "rejected" || reason != ReasonSourceUnreadable {
		t.Fatalf("a book with no directory is %q/%q, want rejected/%s", status, reason, ReasonSourceUnreadable)
	}
	if f.engine.called() != 0 {
		t.Fatalf("the engine was asked %d times about a book with no directory", f.engine.called())
	}
}

// FP5-5 (acceptance): the backstop's re-drive of a parse is the SAME engine call the queue gives
// fifteen minutes. Under the run sweep's two-minute pass it was killed by the deadline, the kill was
// read as a host that cannot run the engine, and the attempt was spent — every pass, until the book
// was rejected for being large.
func TestOneBookInTheSweepGetsABudgetAParseCanLiveIn(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "book.txt", "text")
	f.provision(t, book)
	// The engine takes longer than a second and reports the deadline it was given.
	var granted time.Duration
	f.engine.onManifest = func(ctx context.Context) {
		if dl, ok := ctx.Deadline(); ok {
			granted = time.Until(dl)
		}
	}
	f.now = f.now.Add(2 * time.Hour) // past the graces, so the sweep picks the book up
	if err := f.svc.Sweep(f.ctx); err != nil {
		t.Fatal(err)
	}
	// Not an equality: the deadline is read a few microseconds after it was granted. What matters is
	// that a parse gets the same ORDER of time from the sweep as it does from the queue, rather than
	// the two minutes the run pass runs on.
	if granted < jobs.JobTimeout-time.Minute {
		t.Fatalf("the sweep gave one book's parse %s; the same call gets %s from the queue",
			granted, jobs.JobTimeout)
	}
}

// Cross-family review of the acceptance dofix (H1): the storage ROOT being gone — an unmounted
// volume, a deployment pointed at a path that is not there yet — looks exactly like one book's
// directory being gone, and the two mean opposite things. Read as the second, ONE sweep rejects
// every book in intake with "the source cannot be read": a reason that blames the user's file, is
// terminal by design, and has no way back.
func TestAVanishedStorageRootIsNotEveryBooksFault(t *testing.T) {
	f := newFixture(t)
	first := f.accept(t, "one.txt", "text")
	second := f.accept(t, "two.txt", "text")
	f.provision(t, first)
	f.provision(t, second)
	if err := os.RemoveAll(f.root); err != nil {
		t.Fatal(err)
	}
	// Well past the budget: waiting for a host to come back must never bring a rejection closer.
	for range parseAttempts * 2 {
		for _, b := range []pgstore.Book{first, second} {
			if err := f.svc.Parse(f.ctx, b.ID); err != nil {
				t.Fatal(err)
			}
		}
		f.now = f.now.Add(claimGrace + time.Minute)
	}
	if f.engine.called() != 0 {
		t.Fatalf("the engine was asked %d times while the storage root was gone", f.engine.called())
	}
	for _, b := range []pgstore.Book{first, second} {
		var status, reason string
		var attempts int
		if err := f.store.Pool().QueryRow(f.ctx,
			`select status, reject_reason, parse_attempts from books where id = $1`, b.ID).
			Scan(&status, &reason, &attempts); err != nil {
			t.Fatal(err)
		}
		if status != "parsing" || reason != "" {
			t.Fatalf("book %s is %q/%q after the storage root vanished; the host is at fault, not the book",
				b.ID, status, reason)
		}
		if attempts != 0 {
			t.Fatalf("book %s spent %d attempts on a host that could not see its own storage: the first "+
				"real answer after this would be terminal on arrival", b.ID, attempts)
		}
	}
	// And when the volume comes back, the very next pass parses the book.
	if err := os.MkdirAll(filepath.Join(f.root, first.ID), 0o700); err != nil {
		t.Fatal(err)
	}
	f.provision(t, first)
	if err := f.svc.Parse(f.ctx, first.ID); err != nil {
		t.Fatal(err)
	}
	if got := f.card(t, first.ID); got.Status != "not_started" {
		t.Fatalf("after the storage came back the book is %q, want not_started", got.Status)
	}
}

// Cross-family review of the acceptance dofix (H2): the floor under the library's revision made the
// STATUS walk invisible. A book uploaded before somebody else's upload was cancelled carries a
// counter below that floor, so `uploading → parsing → not_started` — three increments of its own
// number — left `greatest(max, floor)` exactly where it was, and a client obeying the contract drops
// every read that is not above what it applied. The book would sit on the screen as "arriving" until
// an unrelated event moved the library's number.
func TestAStatusChangeIsVisibleEvenUnderTheLibrarysFloor(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "book.txt", "text")
	f.provision(t, book)

	// A cancelled upload raises the floor above the book's own counter.
	ctx, cancel := context.WithCancel(f.ctx)
	defer cancel()
	if _, err := f.svc.Accept(ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
		Filename: "abandoned.txt", File: &failingReader{cancel: cancel}}); err == nil {
		t.Fatal("an upload whose body failed was accepted")
	}
	floor := f.libraryRevision(t)
	var own int64
	if err := f.store.Pool().QueryRow(f.ctx,
		`select revision from books where id = $1`, book.ID).Scan(&own); err != nil {
		t.Fatal(err)
	}
	if own > floor {
		t.Fatalf("the fixture does not reproduce the case: the book is at %d, above the floor %d", own, floor)
	}
	if err := f.svc.Parse(f.ctx, book.ID); err != nil {
		t.Fatal(err)
	}
	if got := f.card(t, book.ID); got.Status != "not_started" {
		t.Fatalf("the book is %q, want not_started", got.Status)
	}
	if got := f.libraryRevision(t); got <= floor {
		t.Fatalf("the book finished parsing and the library still answers %d (was %d): the client drops "+
			"that read and never sees the status", got, floor)
	}
}

// Re-check of the dofix (FP5-10): the SHAPE the first guard missed. A volume mounted at BooksDir
// leaves an empty directory behind when it is unmounted — the mountpoint — and the boot recreates
// that directory anyway, so "does the root exist" answers yes about storage that is gone. Every book
// in intake would then be rejected as unreadable and its source deleted. The marker is what tells the
// two apart: it is written by an upload and cannot be forged by an unmount or by MkdirAll.
func TestAnUnmountedVolumeLooksLikeAnEmptyRootAndStillIsNotTheBooksFault(t *testing.T) {
	f := newFixture(t)
	book := f.accept(t, "one.txt", "text")
	f.provision(t, book)
	// The upload is what writes the marker, and nothing else does — asserted here rather than assumed,
	// because a marker that is never written makes every missing directory look like an unmount and
	// the crash window of the rejection path (FP5-3) would then wait for ever instead of closing.
	if _, err := os.Stat(filepath.Join(f.root, StorageMarker)); err != nil {
		t.Fatalf("an accepted upload left no storage marker: %v", err)
	}
	// Exactly what an unmount leaves: the root is there, everything under it is not — including the
	// marker, which lived on the volume.
	if err := os.RemoveAll(f.root); err != nil {
		t.Fatal(err)
	}
	if err := os.MkdirAll(f.root, 0o750); err != nil { // the boot's own MkdirAll, or the mountpoint
		t.Fatal(err)
	}
	for range parseAttempts * 2 {
		if err := f.svc.Parse(f.ctx, book.ID); err != nil {
			t.Fatal(err)
		}
		f.now = f.now.Add(claimGrace + time.Minute)
	}
	if f.engine.called() != 0 {
		t.Fatalf("the engine was asked %d times about storage that is not mounted", f.engine.called())
	}
	var status, reason string
	var attempts int
	if err := f.store.Pool().QueryRow(f.ctx,
		`select status, reject_reason, parse_attempts from books where id = $1`, book.ID).
		Scan(&status, &reason, &attempts); err != nil {
		t.Fatal(err)
	}
	if status != "parsing" || reason != "" {
		t.Fatalf("an unmounted volume left the book %q/%q: the storage is the host's problem, not the book's",
			status, reason)
	}
	if attempts != 0 {
		t.Fatalf("the book spent %d attempts while its storage was unmounted", attempts)
	}
	// And the marker is not something the platform re-creates behind an operator's back: only an
	// upload writes it, so the volume coming back is what restores the answer.
	if _, err := os.Stat(filepath.Join(f.root, StorageMarker)); err == nil {
		t.Fatal("the marker was re-created while the volume was away: it can no longer tell mounted from not")
	}
}

// Re-check of the dofix (tail в): the rest of the same defect. A budget the BOOK gets is not a budget
// the PASS still has — the second book of a pass inherited whatever the first one left, so a slow
// first parse handed the second a stub of a deadline, the engine call died on it, and the attempt was
// spent on a timeout that says nothing about the book. A pass that cannot give a book its whole
// budget must not start it: the claim is taken inside Parse, so a book left for the next tick has
// spent nothing.
func TestAPassTooShortForAParseStartsNoneAtAll(t *testing.T) {
	f := newFixture(t)
	first := f.accept(t, "one.txt", "text")
	second := f.accept(t, "two.txt", "text")
	f.provision(t, first)
	f.provision(t, second)
	f.now = f.now.Add(2 * time.Hour) // past the graces, so the sweep picks both up
	// What is left of a pass whose first book was slow.
	ctx, cancel := context.WithTimeout(f.ctx, time.Minute)
	defer cancel()
	if err := f.svc.Sweep(ctx); err != nil {
		t.Fatal(err)
	}
	if f.engine.called() != 0 {
		t.Fatalf("the engine was called %d times on a pass shorter than a parse", f.engine.called())
	}
	for _, b := range []pgstore.Book{first, second} {
		var attempts int
		var status string
		if err := f.store.Pool().QueryRow(f.ctx,
			`select parse_attempts, status from books where id = $1`, b.ID).Scan(&attempts, &status); err != nil {
			t.Fatal(err)
		}
		if attempts != 0 || status != "parsing" {
			t.Fatalf("book %s spent %d attempts and is %q after a pass that had no time to parse it",
				b.ID, attempts, status)
		}
	}
}