textmachine/platform/internal/exports/exports_test.go

807 lines
35 KiB
Go

package exports
import (
"bytes"
"context"
"errors"
"io"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
"time"
"textmachine/platform/internal/ingest"
"textmachine/platform/internal/pgstore"
"textmachine/platform/internal/runner"
)
// fakeStore is the storage the door talks to, with every write recorded so a test can ask what the
// door decided rather than what it logged.
type fakeStore struct {
book pgstore.BookRunContext
bookErr error
created pgstore.Export
createEr error
rows map[string]pgstore.Export
build pgstore.ExportBuild
buildErr error
finished bool
finishedPath string
finishedSize int64
finishComp bool
finishErr error
failedCode string
failCalls int
claimErr error
claims int
honourCtx bool
lapsed int
unlinked []pgstore.Artifact
forgot []string
staleIDs []string
}
func (f *fakeStore) ReadBookForRun(context.Context, string, string) (pgstore.BookRunContext, error) {
return f.book, f.bookErr
}
func (f *fakeStore) CreateExport(ctx context.Context, bookID, format string, now time.Time,
enqueue func(context.Context, pgstore.Tx, string) error) (pgstore.Export, error) {
if f.createEr != nil {
return pgstore.Export{}, f.createEr
}
f.created = pgstore.Export{ID: "exp_1", BookID: bookID, Format: format,
State: pgstore.ExportPending, RequestedAt: now}
// The real store calls the enqueue inside its transaction; the fake calls it for the same reason
// — a door that forgot to hand the build to the queue would otherwise pass every test here.
if err := enqueue(ctx, nil, f.created.ID); err != nil {
return pgstore.Export{}, err
}
return f.created, nil
}
func (f *fakeStore) ReadExport(_ context.Context, _, _, exportID string) (pgstore.Export, error) {
e, ok := f.rows[exportID]
if !ok {
return pgstore.Export{}, pgstore.ErrNoExport
}
return e, nil
}
func (f *fakeStore) ReadExportForBuild(context.Context, string) (pgstore.ExportBuild, error) {
return f.build, f.buildErr
}
func (f *fakeStore) FinishExport(ctx context.Context, _, path string, size int64, complete bool, _, _ time.Time) error {
if f.honourCtx && ctx.Err() != nil {
return ctx.Err() // what a real driver does on a dead context
}
if f.finishErr != nil {
return f.finishErr
}
f.finished, f.finishedPath, f.finishedSize, f.finishComp = true, path, size, complete
return nil
}
func (f *fakeStore) FailExport(_ context.Context, _, code string, _ time.Time) error {
f.failCalls++
f.failedCode = code
return nil
}
func (f *fakeStore) ClaimExportBuild(context.Context, string, time.Time) error {
f.claims++
return f.claimErr
}
func (f *fakeStore) ExpireExports(context.Context, time.Time) (int, error) {
return f.lapsed, nil
}
func (f *fakeStore) Unlinked(context.Context) ([]pgstore.Artifact, error) {
return f.unlinked, nil
}
func (f *fakeStore) ForgetExportPath(_ context.Context, id string) error {
f.forgot = append(f.forgot, id)
return nil
}
func (f *fakeStore) FailStaleExports(context.Context, string, time.Time, time.Time, time.Time) ([]string, error) {
return f.staleIDs, nil
}
// fakeEngine answers one prepared outcome and, when told to, writes the file the engine would have.
type fakeEngine struct {
out runner.BuildOutcome
err error
// writes is what the engine leaves at `--out`; afterBuild runs once the file is on disk, which is
// where a caller's deadline realistically expires.
writes string
afterBuild func()
calls int
lastOut string
}
func (f *fakeEngine) Build(_ context.Context, _, _, _, out string) (runner.BuildOutcome, error) {
f.calls++
f.lastOut = out
if f.writes != "" {
if err := os.WriteFile(out, []byte(f.writes), 0o600); err != nil {
return runner.BuildOutcome{}, err
}
}
if f.afterBuild != nil {
f.afterBuild()
}
return f.out, f.err
}
type fakeQueue struct {
enqueued int
err error
}
func (q *fakeQueue) EnqueueExport(context.Context, pgstore.Tx, string) error {
q.enqueued++
return q.err
}
func newService(t *testing.T, st *fakeStore, eng *fakeEngine, q *fakeQueue) *Service {
t.Helper()
if st.rows == nil {
st.rows = map[string]pgstore.Export{}
}
return &Service{Store: st, Engine: eng, Queue: q, Log: slog.New(slog.DiscardHandler),
Cfg: Config{EngineBinary: "/bin/tmctl", Formats: []string{"epub", "txt"},
Dir: t.TempDir(), TTL: time.Hour, StaleAfter: 20 * time.Minute, QueuedGrace: time.Hour}}
}
func cleanReport() runner.BuildOutcome {
return runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitClean, Decoded: true,
Report: ingest.BuildReport{Version: ingest.KnownBuildVersion, Complete: true}}
}
// A format outside the declared set never reaches the engine. The canon's own answer is `400`, and
// the reason it must be decided BEFORE the row is created is money-free but not free: an accepted
// request means a queued job, a poll, and a failure a user reads as the service breaking.
//
// Mutation caught: dropping the Knows check in Create.
func TestAFormatThisDeploymentDoesNotDeclareIsRefusedBeforeAnythingIsCreated(t *testing.T) {
st := &fakeStore{book: pgstore.BookRunContext{HasTree: true, Status: "ready"}}
q := &fakeQueue{}
s := newService(t, st, &fakeEngine{}, q)
if _, err := s.Create(t.Context(), "u_1", "bk_1", "pdf"); !errors.Is(err, ErrUnknownFormat) {
t.Fatalf("Create(pdf) = %v, want ErrUnknownFormat", err)
}
if q.enqueued != 0 || st.created.ID != "" {
t.Errorf("a refused format still created a row (%q) or queued a build (%d)", st.created.ID, q.enqueued)
}
}
// A book NOBODY HAS CUT is ACCEPTED like any other and answered by the export's own outcome — the
// canon says the door may not refuse by a book's state («Nothing about a book's state conflicts with
// exporting it»), and `409 book_not_ready` here was this code's invention (ratified out 04.09).
//
// ⚠ THE ENGINE IS NOT ASKED, and that is the half that matters. Both shapes of «not cut» reach it as
// the wrong class — exit 10 for a book with no configuration yet (which this door reads as the
// deployment's fault) and exit 11 for a source that cut into nothing (the number the INTAKE acts on
// destructively). The verdict is given without spawning anything.
//
// Mutation caught: restoring a refusal to Create (the request stops being accepted); dropping the
// HasTree branch from Build (the engine is asked and answers a class about somebody else's fault).
func TestABookThatWasNeverCutIsAcceptedAndAnsweredByItsExportRatherThanRefused(t *testing.T) {
st := &fakeStore{book: pgstore.BookRunContext{HasTree: false, Status: "parsing"}}
q := &fakeQueue{}
s := newService(t, st, &fakeEngine{}, q)
e, err := s.Create(t.Context(), "u_1", "bk_1", "epub")
if err != nil {
t.Fatalf("Create on an uncut book was refused: %v — the canon forbids refusing by book state", err)
}
if e.State != pgstore.ExportPending || q.enqueued != 1 {
t.Fatalf("export %+v, enqueued %d", e, q.enqueued)
}
// And what it is answered WITH, without the engine ever being spawned.
st.build = pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "epub",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: false}
eng := &fakeEngine{writes: "should not happen", out: cleanReport()}
s2 := newService(t, st, eng, &fakeQueue{})
if err := s2.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if eng.calls != 0 {
t.Errorf("the engine was spawned %d times for a book with nothing to build from", eng.calls)
}
if st.failedCode != FailureBookEmpty {
t.Errorf("failure code %q, want %q", st.failedCode, FailureBookEmpty)
}
if st.finished {
t.Error("a book with no chapters was published as an artifact")
}
}
// A book being TRANSLATED is explicitly exportable (canon §createExport), and the whole reason it
// can be is that `build` opens the project read-only and takes no flock. Pinned here because the
// tempting shape — take the book lock like the correction door does — would be silently correct
// under every other test and would serialize every export behind a translation.
//
// Mutation caught: adding a live-run refusal to Create.
func TestABookBeingTranslatedIsStillExportable(t *testing.T) {
st := &fakeStore{book: pgstore.BookRunContext{HasTree: true, Status: "translating", HasLiveRun: true}}
q := &fakeQueue{}
s := newService(t, st, &fakeEngine{}, q)
e, err := s.Create(t.Context(), "u_1", "bk_1", "epub")
if err != nil {
t.Fatalf("a book being translated was refused an export: %v", err)
}
if e.State != pgstore.ExportPending || q.enqueued != 1 {
t.Errorf("export %+v, enqueued %d", e, q.enqueued)
}
}
// Every exit the engine can answer with maps onto ONE machine reason, and the map is the door's
// whole contract with the seam. Written as a table because the failure it guards against is a
// SILENT one: an unmapped code falling into a default that says "try again" for a condition that
// will never change.
//
// ⚠ Exit 11 is the entry that matters most. From `build` it means "this book has no output units";
// from the intake's vocabulary the same number DELETES the user's upload. The two must never be one
// branch.
//
// Mutation caught: collapsing any two rows, or answering `true` (a file exists) for a refusal.
func TestEveryExitOfTheBuildVerbHasOneMachineReasonAndTheyAreNotTheSame(t *testing.T) {
s := newService(t, &fakeStore{}, &fakeEngine{}, &fakeQueue{})
cases := []struct {
name string
res runner.BuildOutcome
code string
ok bool
}{
{"clean", cleanReport(), "", true},
{"flagged", runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitFlagged, Decoded: true,
Report: ingest.BuildReport{Version: ingest.KnownBuildVersion}}, "", true},
{"clean without a report", runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitClean},
FailureBuildFailed, false},
{"book with holes (--partial did not reach the engine)",
runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitBookIncomplete}, FailureBuildFailed, false},
{"no output units", runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitSourceUnreadable},
FailureBookEmpty, false},
{"the operator's configuration", runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitConfigInvalid},
FailureDeploymentError, false},
{"the copies were prepared and not all landed",
runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitWriteIncomplete}, FailureBuildFailed, false},
{"a held project", runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitProjectLocked},
FailureBuildInterrupted, false},
{"an unmigrated schema", runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitSchemaMismatch},
FailureBuildInterrupted, false},
{"wound down", runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitStopped},
FailureBuildInterrupted, false},
{"an exit outside the contract", runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitFailure},
FailureBuildFailed, false},
{"killed", runner.BuildOutcome{}, FailureBuildInterrupted, false},
}
for _, tc := range cases {
code, ok := s.verdict(t.Context(), "exp_1", tc.res, nil)
if ok != tc.ok || code != tc.code {
t.Errorf("%s: verdict = (%q, %v), want (%q, %v)", tc.name, code, ok, tc.code, tc.ok)
}
}
}
// The whole happy path of the worker, and the two facts the row must carry afterwards: WHERE the
// file is (server-side, so the download can find it) and WHETHER it came out whole (operator-side,
// so an incomplete copy is not a surprise in a support ticket).
func TestABuiltExportIsPublishedWithItsPathAndItsWholeness(t *testing.T) {
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true}}
eng := &fakeEngine{writes: "a marked copy"}
eng.out = cleanReport()
eng.out.Report.Complete = false
eng.out.Report.PendingUnits = 3
s := newService(t, st, eng, &fakeQueue{})
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if !st.finished {
t.Fatalf("the build was not published; failure code %q", st.failedCode)
}
if st.finishedSize != int64(len("a marked copy")) {
t.Errorf("size %d, want %d", st.finishedSize, len("a marked copy"))
}
if st.finishComp {
t.Error("a copy with three pending units was published as complete")
}
if want := filepath.Join(s.Cfg.Dir, "bk_1", "exp_1.txt"); st.finishedPath != want {
t.Errorf("path %q, want %q", st.finishedPath, want)
}
if _, err := os.Stat(st.finishedPath); err != nil {
t.Errorf("the published artifact is not on disk: %v", err)
}
}
// A build that comes back AFTER the stale sweep has already answered the poll must not leave its
// file behind: nothing will ever serve it (the row says `failed`) and nothing will ever expire it
// (the sweep only knows paths it published). It is the one leak this door can produce, and it is
// invisible — the user's answer is correct either way.
//
// Mutation caught: dropping the os.Remove on ErrExportSettled.
func TestAFileBuiltForAnExportNobodyWaitedForIsNotLeftOnDisk(t *testing.T) {
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true},
finishErr: pgstore.ErrExportSettled}
eng := &fakeEngine{writes: "orphan", out: cleanReport()}
s := newService(t, st, eng, &fakeQueue{})
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
path := filepath.Join(s.Cfg.Dir, "bk_1", "exp_1.txt")
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Errorf("a file nothing will serve or expire was left at %s (%v)", path, err)
}
}
// A row already settled when the worker picks it up is left alone — no second build, no second
// verdict. This is what makes the queue's MaxAttempts:1 and the stale sweep safe TOGETHER.
func TestAWorkerThatFindsItsRowAlreadySettledBuildsNothing(t *testing.T) {
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportFailed, Workdir: t.TempDir(), HasTree: true}}
eng := &fakeEngine{writes: "should not happen", out: cleanReport()}
s := newService(t, st, eng, &fakeQueue{})
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if eng.calls != 0 {
t.Errorf("the engine was called %d times for a row that was already settled", eng.calls)
}
if st.finished || st.failCalls != 0 {
t.Error("a settled row was written again")
}
}
// The engine says it wrote a file and there is none: published would be a link to a 404, so it is
// FAILED instead. Loud on this side, an ending poll on the other.
func TestAnExportTheEngineDidNotActuallyWriteIsFailedRatherThanPublished(t *testing.T) {
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true}}
eng := &fakeEngine{out: cleanReport()} // writes nothing
s := newService(t, st, eng, &fakeQueue{})
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if st.finished {
t.Fatal("an export with no file was published")
}
if st.failedCode != FailureBuildFailed {
t.Errorf("failure code %q, want %q", st.failedCode, FailureBuildFailed)
}
}
// A refusal leaves nothing behind either: whatever the engine wrote before it refused is not an
// artifact, and a file under a `failed` row has the same "nothing expires it" problem as the one
// above.
func TestARefusedBuildLeavesNoFileBehind(t *testing.T) {
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true}}
eng := &fakeEngine{writes: "half a book",
out: runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitConfigInvalid}}
s := newService(t, st, eng, &fakeQueue{})
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if st.failedCode != FailureDeploymentError {
t.Errorf("failure code %q, want %q", st.failedCode, FailureDeploymentError)
}
if _, err := os.Stat(filepath.Join(s.Cfg.Dir, "bk_1", "exp_1.txt")); !os.IsNotExist(err) {
t.Error("a refused build left its half-written file on disk")
}
}
// The engine's own STAGING files go too, and they are the ones nothing else could ever name again.
//
// `tmctl build` writes every format through `os.CreateTemp(dir, "."+base+".tmp-*")` and renames it
// into place, and it cannot be wound down gracefully — it is called without the signal context its
// siblings get, so a deadline kills it with SIGKILL. A kill between the write and the rename leaves
// a dot-file that carries no row, so the GC (which deletes only paths rows carry) never sees it.
//
// Mutation caught: removing only the output path and leaving the glob out.
func TestTheStagingFilesOfAKilledBuildAreRemovedWithIt(t *testing.T) {
dir := t.TempDir()
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true}}
// A build that leaves the engine's staging file behind and then refuses, which is what a SIGKILL
// between the write and the rename produces.
eng := &fakeEngine{out: runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitConfigInvalid}}
s := newService(t, st, eng, &fakeQueue{})
s.Cfg.Dir = dir
staged := filepath.Join(dir, "bk_1", ".exp_1.txt.tmp-123456")
if err := os.MkdirAll(filepath.Dir(staged), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(staged, []byte("half a book"), 0o600); err != nil {
t.Fatal(err)
}
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(staged); !os.IsNotExist(err) {
t.Errorf("%s survived: no row will ever carry it, so nothing else would delete it (%v)", staged, err)
}
// A neighbour that is NOT this export's staging file stays: the glob is anchored on the artifact's
// own name, and a sweep that took the directory would take other exports with it.
other := filepath.Join(dir, "bk_1", ".exp_2.txt.tmp-999")
if err := os.WriteFile(other, []byte("somebody else's"), 0o600); err != nil {
t.Fatal(err)
}
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(other); err != nil {
t.Errorf("another export's staging file was taken too: %v", err)
}
}
// The three answers a download can get and the state each comes from. Pinned together because the
// remedies differ and answering one in place of another sends the user to do the wrong thing: poll
// (pending), build again (expired), read the poll (failed).
func TestADownloadOfSomethingNotReadyNamesWhichKindOfNotReady(t *testing.T) {
now := time.Now()
past, future := now.Add(-time.Hour), now.Add(time.Hour)
dir := t.TempDir()
live := filepath.Join(dir, "live.txt")
if err := os.WriteFile(live, []byte("book"), 0o600); err != nil {
t.Fatal(err)
}
st := &fakeStore{rows: map[string]pgstore.Export{
"pending": {ID: "pending", State: pgstore.ExportPending},
"failed": {ID: "failed", State: pgstore.ExportFailed, FailureCode: FailureBuildFailed},
"expired": {ID: "expired", State: pgstore.ExportExpired},
"lapsed": {ID: "lapsed", State: pgstore.ExportReady, Path: live, ExpiresAt: &past},
"gone": {ID: "gone", State: pgstore.ExportReady, Path: filepath.Join(dir, "nope.txt"), ExpiresAt: &future},
"ready": {ID: "ready", State: pgstore.ExportReady, Path: live, ExpiresAt: &future},
}}
s := newService(t, st, &fakeEngine{}, &fakeQueue{})
s.NowFunc = func() time.Time { return now }
for id, want := range map[string]error{
"pending": ErrNotReady,
"failed": ErrNotReady,
"expired": ErrExpired,
// The row's own expiry decides, not the sweep's tick: between two ticks a link would
// otherwise still work after the moment this service told the client it would stop.
"lapsed": ErrExpired,
// Ready over a file that is gone reads as expired and not as a 500: the remedy is the same
// one, and a 500 invites a retry that can never succeed.
"gone": ErrExpired,
} {
f, _, err := s.Open(t.Context(), "u_1", "bk_1", id)
if !errors.Is(err, want) {
t.Errorf("Open(%s) = %v, want %v", id, err, want)
}
if f != nil {
f.Close()
t.Errorf("Open(%s) handed back a file", id)
}
}
f, e, err := s.Open(t.Context(), "u_1", "bk_1", "ready")
if err != nil {
t.Fatalf("a ready export inside its window was refused: %v", err)
}
defer f.Close()
if e.State != pgstore.ExportReady {
t.Errorf("state %q", e.State)
}
if body, _ := io.ReadAll(f); string(body) != "book" {
t.Errorf("the wrong bytes came back: %q", body)
}
}
// The GC deletes the FILES of the rows it expired, and it survives one that is already gone: the
// sweep is the second half of a promise the row made, not a transaction.
func TestTheSweepTakesTheBytesOfWhatItExpired(t *testing.T) {
dir := t.TempDir()
live := filepath.Join(dir, "a.txt")
if err := os.WriteFile(live, []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
gone := filepath.Join(dir, "already-gone.txt")
st := &fakeStore{lapsed: 2, staleIDs: []string{"exp_9"},
unlinked: []pgstore.Artifact{{ID: "exp_live", Path: live}, {ID: "exp_gone", Path: gone}}}
s := newService(t, st, &fakeEngine{}, &fakeQueue{})
if err := s.Sweep(t.Context()); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(live); !os.IsNotExist(err) {
t.Errorf("an expired export's file survived the sweep: %v", err)
}
// Both rows stop naming a file: one because it was removed, one because it was already gone.
if len(st.forgot) != 2 {
t.Errorf("the sweep cleared %d paths, want both (%v)", len(st.forgot), st.forgot)
}
}
// The unlink is RETRYABLE, and that is the whole reason the path outlives the state change. A file
// the sweep could not remove — a full disk, a read-only mount, a crash between the row's move and
// the unlink — must still be findable on the next pass, and the earlier shape cleared the path in
// the same statement as the state, so it never was: every other read of the table selects `ready`.
//
// Mutation caught: clearing the path before the unlink succeeds (the row forgets a file that is
// still there), or clearing it unconditionally after a failed removal.
func TestAFileTheSweepCouldNotRemoveKeepsItsRowPointingAtIt(t *testing.T) {
dir := t.TempDir()
// A directory in place of the file: os.Remove fails on a non-empty one, which is a failure this
// test can produce without root and without breaking the filesystem.
stuck := filepath.Join(dir, "stuck.txt")
if err := os.MkdirAll(filepath.Join(stuck, "inside"), 0o750); err != nil {
t.Fatal(err)
}
st := &fakeStore{unlinked: []pgstore.Artifact{{ID: "exp_stuck", Path: stuck}}}
s := newService(t, st, &fakeEngine{}, &fakeQueue{})
if err := s.Sweep(t.Context()); err != nil {
t.Fatal(err)
}
if len(st.forgot) != 0 {
t.Errorf("the row forgot a path whose file is still there (%v): nothing would ever find it again", st.forgot)
}
if _, err := os.Stat(stuck); err != nil {
t.Errorf("the fixture's own premise is gone: %v", err)
}
}
// A worker CLAIMS its build before running it, and a claim the stale sweep has already answered
// stops the build dead. Without the claim the GC cannot tell a build whose process is gone from one
// that is still queued behind three parses, and it buries the second (the two-clock rule).
//
// Mutation caught: dropping the ClaimExportBuild call, or ignoring ErrExportSettled from it.
func TestAWorkerClaimsItsBuildAndStopsIfTheClaimIsRefused(t *testing.T) {
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true}}
eng := &fakeEngine{writes: "a copy", out: cleanReport()}
s := newService(t, st, eng, &fakeQueue{})
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if st.claims != 1 {
t.Errorf("the worker claimed %d times, want once — without the stamp the GC judges a queued "+
"build on the clock of a running one", st.claims)
}
refused := &fakeStore{build: st.build, claimErr: pgstore.ErrExportSettled}
eng2 := &fakeEngine{writes: "should not happen", out: cleanReport()}
s2 := newService(t, refused, eng2, &fakeQueue{})
if err := s2.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if eng2.calls != 0 {
t.Errorf("the engine ran %d times for a build whose claim was refused", eng2.calls)
}
}
// The declaration an operator makes is checked at BOOT, and the reason is sharper than tidiness:
// the format becomes the artifact's file extension, so a value carrying a separator would name a
// file outside the export directory.
func TestADeclaredFormatCannotNameAFileOutsideTheExportDirectory(t *testing.T) {
for _, bad := range [][]string{{"../../etc/passwd"}, {"ep/ub"}, {"EPUB"}, {""}, {"epub", "epub"},
{"averyverylongformatname"}} {
if err := ValidateFormats(bad); err == nil {
t.Errorf("ValidateFormats(%q) was accepted", bad)
}
}
if err := ValidateFormats([]string{"epub", "txt", "fb2"}); err != nil {
t.Errorf("a usable declaration was refused: %v", err)
}
}
// The door's ratified SECOND duty: it checks the REPORT and not only the file (D39.175 п.2,
// D39.178 п.3). Nothing pinned it — the whole of the duty is a log line, and a log line is what a
// refactor drops without any test noticing.
//
// ⚠ Both directions, because the duty has two halves: a report with something in it must REACH the
// operator, and a clean one must not — a line on every export is a line an operator filters, and a
// filtered line is not a warning.
//
// ⚠ And it is the OPERATOR's, not the reader's: the honesty a reader is owed about an incomplete
// book is inside the file, and none of these words may cross the wire (checked by the wire test in
// internal/httpapi). No money is in it either (D39.84).
//
// Mutation caught: making `report` return early (the drift warning disappears in silence); dropping
// the `NeedsOperator` guard (every build logs); dropping the build-shape warning.
func TestWhatTheBuildReportCarriesForTheOperatorReachesTheOperator(t *testing.T) {
build := func(t *testing.T, rep ingest.BuildReport) string {
t.Helper()
var logs bytes.Buffer
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true}}
eng := &fakeEngine{writes: "a copy",
out: runner.BuildOutcome{Exited: true, ExitCode: ingest.ExitClean, Decoded: true, Report: rep}}
s := newService(t, st, eng, &fakeQueue{})
s.Log = slog.New(slog.NewJSONHandler(&logs, nil))
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
if !st.finished {
t.Fatalf("the build did not publish; failure %q", st.failedCode)
}
return logs.String()
}
clean := build(t, ingest.BuildReport{Version: ingest.KnownBuildVersion, Complete: true})
if strings.Contains(clean, "for the operator") {
t.Errorf("a clean build spoke to the operator anyway:\n%s", clean)
}
drifted := build(t, ingest.BuildReport{Version: ingest.KnownBuildVersion, ConfigDrift: true,
StaleUnknown: true, RemovedFiles: []string{"/beside/the/db.book.epub"}})
if !strings.Contains(drifted, "for the operator") {
t.Fatalf("CONFIG-DRIFT did not reach the operator, and checking the report is the door's own "+
"ratified duty (D39.175 п.2):\n%s", drifted)
}
for _, key := range []string{`"config_drift":true`, `"stale_unknown":true`, `"removed_files":1`} {
if !strings.Contains(drifted, key) {
t.Errorf("the operator's line does not carry %s:\n%s", key, drifted)
}
}
// ⚠ About THIS line only, and not about the package's logging generally: an operator's log
// legitimately carries the engine's stderr, which names paths. What is asserted is that the
// REPORT line is a summary — an operator needs to know housekeeping happened, and the count says
// it — so that the one line addressed to a dashboard does not become a list of server paths.
if strings.Contains(drifted, "/beside/the/db.book.epub") {
t.Errorf("an artifact path reached a log line:\n%s", drifted)
}
// A report shape this build was not written against is READ — the file exists and is paid for —
// and said out loud, because a deploy skew announces itself exactly here and nowhere else.
skewed := build(t, ingest.BuildReport{Version: "tm-build-v9", Complete: true})
if !strings.Contains(skewed, "shape this platform was not written against") {
t.Errorf("a foreign build shape was read in silence:\n%s", skewed)
}
if !strings.Contains(skewed, "tm-build-v9") {
t.Errorf("the skew line does not name the shape it saw:\n%s", skewed)
}
}
// The POLL answers the same question the link does, and it is asked twice over because the two ways
// to stop being serveable have different causes: the moment passed, or the file went. Both used to
// be visible only to `Open` — the poll went on advertising a `ready` export with a `url`, and the
// url answered 410. It is the same «a poll and its address disagree» this door refuses elsewhere.
//
// Mutation caught: dropping the `asOfNow` call from Read (the expiry direction); dropping the stat
// (the vanished-file direction); treating any stat error as gone (a busy mount would tell a client
// its export expired).
func TestThePollSaysTheSameThingTheLinkDoes(t *testing.T) {
now := time.Now()
past, future := now.Add(-time.Hour), now.Add(time.Hour)
dir := t.TempDir()
live := filepath.Join(dir, "live.txt")
if err := os.WriteFile(live, []byte("book"), 0o600); err != nil {
t.Fatal(err)
}
st := &fakeStore{rows: map[string]pgstore.Export{
"lapsed": {ID: "lapsed", State: pgstore.ExportReady, Path: live, ExpiresAt: &past},
"gone": {ID: "gone", State: pgstore.ExportReady, ExpiresAt: &future,
Path: filepath.Join(dir, "nothing-here.txt")},
"ready": {ID: "ready", State: pgstore.ExportReady, Path: live, ExpiresAt: &future},
}}
s := newService(t, st, &fakeEngine{}, &fakeQueue{})
s.NowFunc = func() time.Time { return now }
for id, why := range map[string]string{
"lapsed": "its moment has passed",
"gone": "its file is not there",
} {
e, err := s.Read(t.Context(), "u_1", "bk_1", id)
if err != nil {
t.Fatalf("%s: %v", id, err)
}
if e.State != pgstore.ExportExpired {
t.Errorf("the poll calls %s %q while the link answers 410 (%s)", id, e.State, why)
}
if e.Path != "" {
t.Errorf("%s: the poll still carries a path", id)
}
// And the two surfaces agree, which is the property rather than either answer alone.
if _, _, err := s.Open(t.Context(), "u_1", "bk_1", id); !errors.Is(err, ErrExpired) {
t.Errorf("%s: the link says %v while the poll says expired", id, err)
}
}
e, err := s.Read(t.Context(), "u_1", "bk_1", "ready")
if err != nil || e.State != pgstore.ExportReady {
t.Fatalf("a live export was reported as %q (%v)", e.State, err)
}
// ⚠ AND ONLY «not there» COUNTS. A stat that fails for any other reason — a directory this
// process may not traverse, a mount gone busy — is THIS deployment's problem, and answering the
// client «your export expired» would send it to build the same book again against the same broken
// disk. The link keeps its own judgement for the same reason (`Open` reports only ErrNotExist).
if os.Getuid() == 0 {
t.Skip("running as root: a permission error cannot be produced, so the narrowness of the stat is unchecked here")
}
locked := filepath.Join(dir, "locked")
if err := os.MkdirAll(locked, 0o700); err != nil {
t.Fatal(err)
}
hidden := filepath.Join(locked, "book.txt")
if err := os.WriteFile(hidden, []byte("book"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Chmod(locked, 0o000); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(locked, 0o700) })
if _, err := os.Stat(hidden); err == nil || errors.Is(err, os.ErrNotExist) {
t.Skipf("this filesystem does not enforce the directory permission (%v): the case cannot be produced", err)
}
st.rows["unreachable"] = pgstore.Export{ID: "unreachable", State: pgstore.ExportReady,
Path: hidden, ExpiresAt: &future}
got, err := s.Read(t.Context(), "u_1", "bk_1", "unreachable")
if err != nil {
t.Fatal(err)
}
if got.State != pgstore.ExportReady {
t.Errorf("a file this process cannot STAT was reported as %q: the deployment's own trouble was "+
"handed to a client as an expiry, and building again would meet the same disk", got.State)
}
}
// The publication survives a context that is already dead, and it must: the engine can come back
// with a file COMMITTED to disk on a call whose deadline has passed (a job timeout, a shutdown), and
// publishing on the caller's context then fails — deleting a book that was built and leaving the row
// `pending` for the stale sweep to end minutes later on a less true reason. The failure path already
// wrote on a detached context; this is its other half.
//
// Mutation caught: passing `ctx` instead of the detached one to FinishExport.
func TestABuiltExportIsPublishedEvenIfTheCallThatBuiltItIsOver(t *testing.T) {
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: "bk_1", Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true}}
// The store refuses a write on a cancelled context, exactly as pgx does.
st.honourCtx = true
eng := &fakeEngine{writes: "a whole book", out: cleanReport()}
s := newService(t, st, eng, &fakeQueue{})
ctx, cancel := context.WithCancel(t.Context())
eng.afterBuild = cancel // the deadline passes while the engine is committing its file
if err := s.Build(ctx, "exp_1"); err != nil {
t.Fatal(err)
}
if !st.finished {
t.Fatalf("a file that WAS built was not published (failure %q): the irreversible act is done, "+
"so its record must survive a cancellation on the same path", st.failedCode)
}
if _, err := os.Stat(st.finishedPath); err != nil {
t.Errorf("the published artifact was deleted: %v", err)
}
}
// ⛔ THE BOOK IS NOT NAMED IN A LOG LINE, and the export id is the handle instead.
//
// The standard of this zone keeps identifiers of users and books out of logs (PD-139, PD-99), and the
// success line of a build carried `book_id` in plain text at INFO — the fifth carrier of that class
// found on 11.09, and the one nobody's census had looked at. What an operator loses is nothing: the
// export id is this service's own opaque identifier, it resolves to the book for whoever may ask, and
// it is what a reader quotes when a download breaks.
func TestTheBuiltExportIsLoggedByItsOwnIdAndNotByItsBook(t *testing.T) {
const bookID = "bk_THEBOOKSOWNIDENTIFIER"
st := &fakeStore{build: pgstore.ExportBuild{ID: "exp_1", BookID: bookID, Format: "txt",
State: pgstore.ExportPending, Workdir: t.TempDir(), HasTree: true}}
s := newService(t, st, &fakeEngine{writes: "a book", out: cleanReport()}, &fakeQueue{})
var buf bytes.Buffer
s.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
if err := s.Build(t.Context(), "exp_1"); err != nil {
t.Fatal(err)
}
logged := buf.String()
// THE BOUNDARY: the line has to have been written, or the assertion below is about nothing.
if !strings.Contains(logged, "export built") {
t.Fatalf("the build wrote no success line, so this test measured nothing:\n%s", logged)
}
// The handle stays…
if !strings.Contains(logged, "exp_1") {
t.Errorf("the export's own id is not in the line, so an operator has no handle at all:\n%s", logged)
}
// …and the book does not.
if strings.Contains(logged, bookID) {
t.Errorf("the book's identifier reached a log line: %s", logged)
}
}