package books import ( "context" "errors" "os" "path/filepath" "strings" "testing" "time" "textmachine/platform/internal/ingest" "textmachine/platform/internal/pgstore" ) // templated puts the deployment's `book.yaml` template in place, which is what lets a book be cut // inside the upload request: without it the intake's pass stops at ErrNotProvisioned. func templated(t *testing.T, f *fixture) { t.Helper() tpl := filepath.Join(t.TempDir(), "book.yaml") if err := os.WriteFile(tpl, []byte(templateYAML), 0o600); err != nil { t.Fatal(err) } f.svc.Cfg.BookTemplate = tpl } // The cut runs inside the upload request, so what a book is — and whether it is one — is answered // before the upload is over (backlog row 285). func TestAnUploadIsCutWhileTheUploaderIsStillWaiting(t *testing.T) { f := newFixture(t) templated(t, f) book := f.accept(t, "蛊真人.txt", "первая глава\fвторая глава") if book.Status != "not_started" { t.Fatalf("the accepted book is %q, want the cut's own verdict", book.Status) } if book.ChapterCount != 500 { t.Fatalf("the accepted book carries %d chapters, want the cut's own 500", book.ChapterCount) } if f.engine.called() != 1 { t.Fatalf("the engine was asked %d times for one upload, want exactly one $0 cut", f.engine.called()) } // This fixture serves no reading surface, so the two fields that arrive with it are absent — and // they are absent TOGETHER, because one call writes both (pgstore.SaveStructure). A response that // carried one without the other would mean the cut and its materialisation had come apart. if book.SourceChars != nil || book.Structure != "" { t.Errorf("a deployment with no reading surface answered source_chars=%v structure=%q", book.SourceChars, book.Structure) } } // A file the engine read, cut, and found nothing in: refused now, and nothing is kept — there is // nothing to delete if we never took it (row 285 closes row 254). func TestAnUnusableSourceIsRefusedAndNothingIsKept(t *testing.T) { f := newFixture(t) templated(t, f) f.engine.set(ingest.Manifest{}, refusal(t, ingest.ExitSourceUnreadable)) _, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru", Filename: "empty.txt", File: strings.NewReader(" ")}) if !errors.Is(err, ErrNoBookInSource) { t.Fatalf("a source the engine found no book in was answered %v, want ErrNoBookInSource", err) } if !errors.Is(err, ErrBadIntake) { t.Fatal("the refusal is not an ErrBadIntake, so the route cannot answer it as the contract's 400") } assertNothingKept(t, f) } // A book that came through fine and that nothing cut into chapters: the reader's book is one document // per engine chapter, so what we would hand back is a single canvas (rows 283/325). // // The predicate is the CUT and not the extension, so both directions are asserted: an `.epub` cut by // its nav is accepted, a `.txt` nobody could cut is not. func TestABookNothingCutIntoChaptersIsRefusedAndAnEpubThatWasCutIsNot(t *testing.T) { f := newFixture(t) templated(t, f) f.engine.set(oneChapterManifest(), nil) _, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru", Filename: "flat.txt", File: strings.NewReader("a book with no headings at all")}) if !errors.Is(err, ErrStructureNotDeliverable) { t.Fatalf("a source cut into one chapter was answered %v, want ErrStructureNotDeliverable", err) } assertNothingKept(t, f) // The same intake, an EPUB, and a cut that found chapters — which the engine draws from a nav or // an NCX. f.engine.set(wholeManifest(12, 2), nil) book, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru", Filename: "novel.epub", File: strings.NewReader("PK\x03\x04 not really a zip")}) if err != nil { t.Fatalf("an epub the engine cut into 12 chapters was refused: %v", err) } if book.Status != "not_started" || book.ChapterCount != 12 { t.Fatalf("the epub came back %q with %d chapters, want a parsed book with its 12", book.Status, book.ChapterCount) } } // A deployment that cannot answer must not answer for the file: every class but the engine's own // verdict leaves the book accepted `parsing` for the queue to finish. // // The claim also goes back. A claim left standing would make the queue job that runs a moment later // do nothing, which no assertion about status alone would catch — so the queue's pass is run here and // asserted to have done something. func TestADeploymentThatCannotCutStillAcceptsTheBookAndTheQueueFinishesIt(t *testing.T) { f := newFixture(t) f.engine.set(wholeManifest(7, 2), nil) // No template: the intake's own pass stops at «this book has no configuration», which is the // deployment's state and never the book's. book := f.accept(t, "蛊真人.txt", "первая глава") if book.Status != "parsing" { t.Fatalf("a book on a deployment that cannot cut it came back %q, want the queue's own `parsing`", book.Status) } f.provision(t, book) if err := f.svc.Parse(f.ctx, book.ID); err != nil { t.Fatal(err) } if got := f.card(t, book.ID); got.Status != "not_started" || got.ChapterCount != 7 { t.Fatalf("the queue's pass left the book %q with %d chapters: the intake's claim was not given back, "+ "so the job found it claimed and did nothing", got.Status, got.ChapterCount) } } // assertNothingKept is the refusal's other half: no row, no bytes on disk. func assertNothingKept(t *testing.T, f *fixture) { t.Helper() var rows int if err := f.store.Pool().QueryRow(f.ctx, `select count(*) from books`).Scan(&rows); err != nil { t.Fatal(err) } if rows != 0 { t.Errorf("a refused upload left %d book rows behind; the contract gives a user no way to clear one", rows) } entries, err := os.ReadDir(f.root) if err != nil { t.Fatal(err) } var dirs []string for _, e := range entries { if e.IsDir() { dirs = append(dirs, e.Name()) } } if len(dirs) != 0 { t.Errorf("a refused upload left %v on disk", dirs) } } // oneChapterManifest is the engine's answer for a source nothing cut: one chapter, `structure: none`. func oneChapterManifest() ingest.Manifest { m := wholeManifest(1, 3) m.Structure = "none" return m } // The response must carry the row as it stands AFTER the cut, and the cut can outlive the budget of // any context opened before it. // // The re-read used to run on the context taken before the body arrived, which is spent by the time a // slow cut returns — so the 201 answered `parsing` with no chapters while the store held a parsed // book. The budget is shortened here rather than waited out. func TestASlowCutStillAnswersWithTheRowAsItStandsAfterIt(t *testing.T) { f := newFixture(t) templated(t, f) f.svc.writeBudget = 50 * time.Millisecond f.engine.onManifest = func(context.Context) { time.Sleep(120 * time.Millisecond) } book := f.accept(t, "蛊真人.txt", "первая глава\fвторая глава") if book.Status != "not_started" || book.ChapterCount != 500 { t.Fatalf("after a cut that outlived the write budget the response says %q with %d chapters, "+ "want the parsed row", book.Status, book.ChapterCount) } if got := f.card(t, book.ID); got.Status != book.Status || got.ChapterCount != book.ChapterCount { t.Fatalf("the response (%q/%d) disagrees with the store (%q/%d)", book.Status, book.ChapterCount, got.Status, got.ChapterCount) } } // A deployment fault must leave no verdict about the file behind: no spent attempt, no `rejected` // row, and the claim back so the queue's job finds work to do. // // Three branches answer before the one the intake used to guard, and each ended the pass its own way: // a manifest this build cannot read spent an attempt, a missing storage root held the claim, and a // missing directory wrote `rejected` — which the 201 then carried out to the uploader. func TestADeploymentFaultLeavesNoVerdictAboutTheFile(t *testing.T) { for _, c := range []struct { name string arrange func(t *testing.T, f *fixture) }{{ // The manifest decodes to a shape this build does not read: not evidence about the user's text. name: "manifest this build cannot read", arrange: func(t *testing.T, f *fixture) { f.engine.set(ingest.Manifest{Version: "tm-manifest-v9", ChaptersTotal: 3}, nil) }, }, { // The engine could not be run at all. name: "engine cannot be run", arrange: func(t *testing.T, f *fixture) { f.engine.set(ingest.Manifest{}, errors.New("exec: no such file")) }, }} { t.Run(c.name, func(t *testing.T) { f := newFixture(t) templated(t, f) c.arrange(t, f) book := f.accept(t, "蛊真人.txt", "первая глава") if book.Status != "parsing" { t.Fatalf("a deployment fault answered %q, want the queue's own `parsing`", book.Status) } var attempts int var claimed *time.Time if err := f.store.Pool().QueryRow(f.ctx, `select parse_attempts, parse_started_at from books where id = $1`, book.ID).Scan(&attempts, &claimed); err != nil { t.Fatal(err) } if attempts != 0 { t.Errorf("the intake's pass spent %d attempts of a budget that bounds how often a broken "+ "HOST is asked", attempts) } if claimed != nil { t.Errorf("the claim is still held (%v): the queue's job will find the book claimed and do nothing", claimed) } // And no verdict about the file was written either: `rejected` is a state the user sees, and // a deployment fault is not the user's file being wrong. if got := f.card(t, book.ID); got.Status == "rejected" { t.Errorf("a deployment fault left the book %q, which the 201 then carries out as a verdict "+ "about the uploader's file", got.Status) } }) } } // The other two branches of the same rule, driven at the level they live on: both are decided before // the engine is called, so no hook inside the engine can reach them, and going through Accept would // pin the fixture rather than the branch. // // `book directory gone` is the one that used to write `rejected` — a verdict about the uploader's // file, produced by the platform losing its own directory. func TestTheBranchesDecidedBeforeTheEngineAlsoLeaveNoVerdict(t *testing.T) { for _, c := range []struct { name string break_ func(t *testing.T, f *fixture, dir string) }{{ name: "book directory gone", break_: func(t *testing.T, f *fixture, dir string) { remove(t, dir) }, }, { name: "storage root gone", break_: func(t *testing.T, f *fixture, dir string) { remove(t, dir) remove(t, filepath.Join(f.root, StorageMarker)) }, }} { t.Run(c.name, func(t *testing.T) { // No template, so the intake's own pass reaches no verdict and leaves the book `parsing` // with its claim given back — which is the state these two branches are decided in. f := newFixture(t) book := f.accept(t, "蛊真人.txt", "первая глава") if book.Status != "parsing" { t.Fatalf("the fixture did not reach `parsing`: %q", book.Status) } c.break_(t, f, filepath.Join(f.root, book.ID)) claim, err := f.store.ClaimParse(f.ctx, book.ID, f.now, f.now.Add(-time.Hour)) if err != nil { t.Fatal(err) } if err := f.svc.parseClaimed(f.ctx, claim, true); !errors.Is(err, errNotConclusive) { t.Fatalf("the intake's own pass answered %v, want errNotConclusive", err) } if got := f.card(t, book.ID); got.Status == "rejected" { t.Errorf("the pass left the book %q: a platform that lost its own directory is not a "+ "verdict about the uploader's file", got.Status) } }) } } // The intake does not materialize the reading surface, and does not enqueue a job it has no work for. // // Both are properties of the same pass and both are invisible from the response, which is why they // are asserted on the collaborators rather than on the book: materializing runs the engine twice more // (readmodel.MaterializeBudget) with the uploader holding the request open, and a job that exists // while the cut is going races it for the parse claim. // // The debt is what brings the materializer back, so it is asserted present — «did not materialize» // and «lost the tree» are different outcomes and only one of them is wanted. func TestTheIntakeNeitherMaterializesNorEnqueuesWhenItsCutSucceeds(t *testing.T) { f := newFixture(t) templated(t, f) reader, queue := &fakeReader{}, &countingQueue{} f.svc.Reader, f.svc.Queue = reader, queue book := f.accept(t, "蛊真人.txt", "первая глава\fвторая глава") if book.Status != "not_started" { t.Fatalf("the cut did not succeed: %q", book.Status) } if len(reader.claims) != 0 || len(reader.refreshed) != 0 { t.Errorf("the intake materialized the reading surface (%d claims, %d refreshes): the uploader "+ "waits for two more engine runs and UploadSettle does not cover them", len(reader.claims), len(reader.refreshed)) } if queue.n != 0 { t.Errorf("the intake enqueued %d jobs for a book it has already cut: such a job races the cut "+ "for the parse claim", queue.n) } // The debt stands, so the materializer's own sweep takes the book. owed, err := f.store.BooksOwedReadModel(f.ctx, 10) if err != nil { t.Fatal(err) } if len(owed) != 1 || owed[0].ID != book.ID { t.Fatalf("the parsed book owes no reading surface (%d owed): nothing brings the materializer to it", len(owed)) } } // The other half of the same rule: a cut that reaches no verdict hands the book to the queue, and the // claim goes back in the SAME transaction — either alone is a state nobody finishes. func TestACutWithNoVerdictGivesTheClaimBackAndEnqueuesTheJob(t *testing.T) { f := newFixture(t) // no template: the pass stops at «this book has no configuration» queue := &countingQueue{} f.svc.Queue = queue book := f.accept(t, "蛊真人.txt", "первая глава") if book.Status != "parsing" { t.Fatalf("the cut reached a verdict it should not have: %q", book.Status) } if queue.n != 1 { t.Errorf("the queue was handed %d jobs, want exactly the one that finishes this book", queue.n) } var claimed *time.Time if err := f.store.Pool().QueryRow(f.ctx, `select parse_started_at from books where id = $1`, book.ID).Scan(&claimed); err != nil { t.Fatal(err) } if claimed != nil { t.Errorf("the claim is still held (%v) while a job is queued: that job will do nothing", claimed) } } // countingQueue counts what the intake handed the queue. The real one inserts a row in the caller's // transaction; what is being asserted here is WHETHER and HOW MANY, not what River does with it. type countingQueue struct{ n int } func (q *countingQueue) EnqueueParse(context.Context, pgstore.Tx, string) error { q.n++; return nil } // remove takes a path away and fails the test if it could not: a case that did not break what it // meant to break is a case that proves nothing. func remove(t *testing.T, path string) { t.Helper() if err := os.RemoveAll(path); err != nil { t.Fatal(err) } } // The job goes in ONLY where the claim was really given back, and the half that decides is // `RowsAffected` on the release itself (pgstore.ReleaseParseClaim). // // Both halves are one transaction because either alone is a state nobody finishes. The guard is the // side nothing covered: a pass whose claim has been taken over by another has nothing to hand the // queue, and a job enqueued for a book somebody else is already parsing is a worker that wakes up, // finds the claim held and does nothing — while the pass that DOES hold it still owes the book its // finish. Both directions are asserted here, because a case that only ever proves the job is ABSENT // passes just as well when the release stopped enqueueing altogether. func TestOnlyAClaimThatWasReallyGivenBackQueuesTheJobThatFinishesTheBook(t *testing.T) { f := newFixture(t) // no template: the intake's own pass reaches no verdict and releases once queue := &countingQueue{} f.svc.Queue = queue book := f.accept(t, "蛊真人.txt", "первая глава") if queue.n != 1 { t.Fatalf("the intake handed the queue %d jobs, want the one that finishes this book", queue.n) } // Somebody takes the book: from here a release carrying any OTHER stamp is a pass talking about // work it is no longer doing. claim, err := f.store.ClaimParse(f.ctx, book.ID, f.now, f.now.Add(-ClaimGrace)) if err != nil { t.Fatal(err) } if err := f.store.ReleaseParseClaim(f.ctx, book.ID, claim.At.Add(-time.Second), f.svc.enqueue); err != nil { t.Fatalf("a release carrying a stale stamp failed instead of doing nothing: %v", err) } if queue.n != 1 { t.Errorf("a release that gave back nothing still queued a job (%d in total): the worker it wakes will find the claim held and do nothing, and the pass that holds it is still the one that owes the book", queue.n) } var stamp *time.Time if err := f.store.Pool().QueryRow(f.ctx, `select parse_started_at from books where id = $1`, book.ID).Scan(&stamp); err != nil { t.Fatal(err) } if stamp == nil { t.Fatal("a release carrying a stale stamp cleared somebody else's claim") } // And the holder's OWN release does both halves: without this the assertion above would hold on a // release that has stopped queueing anything at all. if err := f.store.ReleaseParseClaim(f.ctx, book.ID, claim.At, f.svc.enqueue); err != nil { t.Fatal(err) } if queue.n != 2 { t.Errorf("the holder gave the claim back and the queue has %d jobs, want a second one: the book has nobody to finish it", queue.n) } if err := f.store.Pool().QueryRow(f.ctx, `select parse_started_at from books where id = $1`, book.ID).Scan(&stamp); err != nil { t.Fatal(err) } if stamp != nil { t.Errorf("the holder's own release left the claim standing (%v): the job it queued will do nothing", stamp) } }