405 lines
17 KiB
Go
405 lines
17 KiB
Go
package books
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/platform/internal/ingest"
|
||
"textmachine/platform/internal/pgstore"
|
||
)
|
||
|
||
// templated puts the deployment's `book.yaml` template in place, which is what lets a book be cut
|
||
// inside the upload request: without it the intake's pass stops at ErrNotProvisioned.
|
||
func templated(t *testing.T, f *fixture) {
|
||
t.Helper()
|
||
tpl := filepath.Join(t.TempDir(), "book.yaml")
|
||
if err := os.WriteFile(tpl, []byte(templateYAML), 0o600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
f.svc.Cfg.BookTemplate = tpl
|
||
}
|
||
|
||
// The cut runs inside the upload request, so what a book is — and whether it is one — is answered
|
||
// before the upload is over (backlog row 285).
|
||
func TestAnUploadIsCutWhileTheUploaderIsStillWaiting(t *testing.T) {
|
||
f := newFixture(t)
|
||
templated(t, f)
|
||
book := f.accept(t, "蛊真人.txt", "первая глава\fвторая глава")
|
||
if book.Status != "not_started" {
|
||
t.Fatalf("the accepted book is %q, want the cut's own verdict", book.Status)
|
||
}
|
||
if book.ChapterCount != 500 {
|
||
t.Fatalf("the accepted book carries %d chapters, want the cut's own 500", book.ChapterCount)
|
||
}
|
||
if f.engine.called() != 1 {
|
||
t.Fatalf("the engine was asked %d times for one upload, want exactly one $0 cut", f.engine.called())
|
||
}
|
||
// This fixture serves no reading surface, so the two fields that arrive with it are absent — and
|
||
// they are absent TOGETHER, because one call writes both (pgstore.SaveStructure). A response that
|
||
// carried one without the other would mean the cut and its materialisation had come apart.
|
||
if book.SourceChars != nil || book.Structure != "" {
|
||
t.Errorf("a deployment with no reading surface answered source_chars=%v structure=%q",
|
||
book.SourceChars, book.Structure)
|
||
}
|
||
}
|
||
|
||
// A file the engine read, cut, and found nothing in: refused now, and nothing is kept — there is
|
||
// nothing to delete if we never took it (row 285 closes row 254).
|
||
func TestAnUnusableSourceIsRefusedAndNothingIsKept(t *testing.T) {
|
||
f := newFixture(t)
|
||
templated(t, f)
|
||
f.engine.set(ingest.Manifest{}, refusal(t, ingest.ExitSourceUnreadable))
|
||
_, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||
Filename: "empty.txt", File: strings.NewReader(" ")})
|
||
if !errors.Is(err, ErrNoBookInSource) {
|
||
t.Fatalf("a source the engine found no book in was answered %v, want ErrNoBookInSource", err)
|
||
}
|
||
if !errors.Is(err, ErrBadIntake) {
|
||
t.Fatal("the refusal is not an ErrBadIntake, so the route cannot answer it as the contract's 400")
|
||
}
|
||
assertNothingKept(t, f)
|
||
}
|
||
|
||
// A book that came through fine and that nothing cut into chapters: the reader's book is one document
|
||
// per engine chapter, so what we would hand back is a single canvas (rows 283/325).
|
||
//
|
||
// The predicate is the CUT and not the extension, so both directions are asserted: an `.epub` cut by
|
||
// its nav is accepted, a `.txt` nobody could cut is not.
|
||
func TestABookNothingCutIntoChaptersIsRefusedAndAnEpubThatWasCutIsNot(t *testing.T) {
|
||
f := newFixture(t)
|
||
templated(t, f)
|
||
f.engine.set(oneChapterManifest(), nil)
|
||
_, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||
Filename: "flat.txt", File: strings.NewReader("a book with no headings at all")})
|
||
if !errors.Is(err, ErrStructureNotDeliverable) {
|
||
t.Fatalf("a source cut into one chapter was answered %v, want ErrStructureNotDeliverable", err)
|
||
}
|
||
assertNothingKept(t, f)
|
||
|
||
// The same intake, an EPUB, and a cut that found chapters — which the engine draws from a nav or
|
||
// an NCX.
|
||
f.engine.set(wholeManifest(12, 2), nil)
|
||
book, err := f.svc.Accept(f.ctx, Intake{UserID: "u1", SourceLang: "zh", TargetLang: "ru",
|
||
Filename: "novel.epub", File: strings.NewReader("PK\x03\x04 not really a zip")})
|
||
if err != nil {
|
||
t.Fatalf("an epub the engine cut into 12 chapters was refused: %v", err)
|
||
}
|
||
if book.Status != "not_started" || book.ChapterCount != 12 {
|
||
t.Fatalf("the epub came back %q with %d chapters, want a parsed book with its 12", book.Status, book.ChapterCount)
|
||
}
|
||
}
|
||
|
||
// A deployment that cannot answer must not answer for the file: every class but the engine's own
|
||
// verdict leaves the book accepted `parsing` for the queue to finish.
|
||
//
|
||
// The claim also goes back. A claim left standing would make the queue job that runs a moment later
|
||
// do nothing, which no assertion about status alone would catch — so the queue's pass is run here and
|
||
// asserted to have done something.
|
||
func TestADeploymentThatCannotCutStillAcceptsTheBookAndTheQueueFinishesIt(t *testing.T) {
|
||
f := newFixture(t)
|
||
f.engine.set(wholeManifest(7, 2), nil)
|
||
// No template: the intake's own pass stops at «this book has no configuration», which is the
|
||
// deployment's state and never the book's.
|
||
book := f.accept(t, "蛊真人.txt", "первая глава")
|
||
if book.Status != "parsing" {
|
||
t.Fatalf("a book on a deployment that cannot cut it came back %q, want the queue's own `parsing`", book.Status)
|
||
}
|
||
f.provision(t, book)
|
||
if err := f.svc.Parse(f.ctx, book.ID); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if got := f.card(t, book.ID); got.Status != "not_started" || got.ChapterCount != 7 {
|
||
t.Fatalf("the queue's pass left the book %q with %d chapters: the intake's claim was not given back, "+
|
||
"so the job found it claimed and did nothing", got.Status, got.ChapterCount)
|
||
}
|
||
}
|
||
|
||
// assertNothingKept is the refusal's other half: no row, no bytes on disk.
|
||
func assertNothingKept(t *testing.T, f *fixture) {
|
||
t.Helper()
|
||
var rows int
|
||
if err := f.store.Pool().QueryRow(f.ctx, `select count(*) from books`).Scan(&rows); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rows != 0 {
|
||
t.Errorf("a refused upload left %d book rows behind; the contract gives a user no way to clear one", rows)
|
||
}
|
||
entries, err := os.ReadDir(f.root)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var dirs []string
|
||
for _, e := range entries {
|
||
if e.IsDir() {
|
||
dirs = append(dirs, e.Name())
|
||
}
|
||
}
|
||
if len(dirs) != 0 {
|
||
t.Errorf("a refused upload left %v on disk", dirs)
|
||
}
|
||
}
|
||
|
||
// oneChapterManifest is the engine's answer for a source nothing cut: one chapter, `structure: none`.
|
||
func oneChapterManifest() ingest.Manifest {
|
||
m := wholeManifest(1, 3)
|
||
m.Structure = "none"
|
||
return m
|
||
}
|
||
|
||
// The response must carry the row as it stands AFTER the cut, and the cut can outlive the budget of
|
||
// any context opened before it.
|
||
//
|
||
// The re-read used to run on the context taken before the body arrived, which is spent by the time a
|
||
// slow cut returns — so the 201 answered `parsing` with no chapters while the store held a parsed
|
||
// book. The budget is shortened here rather than waited out.
|
||
func TestASlowCutStillAnswersWithTheRowAsItStandsAfterIt(t *testing.T) {
|
||
f := newFixture(t)
|
||
templated(t, f)
|
||
f.svc.writeBudget = 50 * time.Millisecond
|
||
f.engine.onManifest = func(context.Context) { time.Sleep(120 * time.Millisecond) }
|
||
|
||
book := f.accept(t, "蛊真人.txt", "первая глава\fвторая глава")
|
||
if book.Status != "not_started" || book.ChapterCount != 500 {
|
||
t.Fatalf("after a cut that outlived the write budget the response says %q with %d chapters, "+
|
||
"want the parsed row", book.Status, book.ChapterCount)
|
||
}
|
||
if got := f.card(t, book.ID); got.Status != book.Status || got.ChapterCount != book.ChapterCount {
|
||
t.Fatalf("the response (%q/%d) disagrees with the store (%q/%d)",
|
||
book.Status, book.ChapterCount, got.Status, got.ChapterCount)
|
||
}
|
||
}
|
||
|
||
// A deployment fault must leave no verdict about the file behind: no spent attempt, no `rejected`
|
||
// row, and the claim back so the queue's job finds work to do.
|
||
//
|
||
// Three branches answer before the one the intake used to guard, and each ended the pass its own way:
|
||
// a manifest this build cannot read spent an attempt, a missing storage root held the claim, and a
|
||
// missing directory wrote `rejected` — which the 201 then carried out to the uploader.
|
||
func TestADeploymentFaultLeavesNoVerdictAboutTheFile(t *testing.T) {
|
||
for _, c := range []struct {
|
||
name string
|
||
arrange func(t *testing.T, f *fixture)
|
||
}{{
|
||
// The manifest decodes to a shape this build does not read: not evidence about the user's text.
|
||
name: "manifest this build cannot read",
|
||
arrange: func(t *testing.T, f *fixture) {
|
||
f.engine.set(ingest.Manifest{Version: "tm-manifest-v9", ChaptersTotal: 3}, nil)
|
||
},
|
||
}, {
|
||
// The engine could not be run at all.
|
||
name: "engine cannot be run",
|
||
arrange: func(t *testing.T, f *fixture) { f.engine.set(ingest.Manifest{}, errors.New("exec: no such file")) },
|
||
}} {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
f := newFixture(t)
|
||
templated(t, f)
|
||
c.arrange(t, f)
|
||
book := f.accept(t, "蛊真人.txt", "первая глава")
|
||
if book.Status != "parsing" {
|
||
t.Fatalf("a deployment fault answered %q, want the queue's own `parsing`", book.Status)
|
||
}
|
||
var attempts int
|
||
var claimed *time.Time
|
||
if err := f.store.Pool().QueryRow(f.ctx,
|
||
`select parse_attempts, parse_started_at from books where id = $1`, book.ID).Scan(&attempts, &claimed); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if attempts != 0 {
|
||
t.Errorf("the intake's pass spent %d attempts of a budget that bounds how often a broken "+
|
||
"HOST is asked", attempts)
|
||
}
|
||
if claimed != nil {
|
||
t.Errorf("the claim is still held (%v): the queue's job will find the book claimed and do nothing", claimed)
|
||
}
|
||
// And no verdict about the file was written either: `rejected` is a state the user sees, and
|
||
// a deployment fault is not the user's file being wrong.
|
||
if got := f.card(t, book.ID); got.Status == "rejected" {
|
||
t.Errorf("a deployment fault left the book %q, which the 201 then carries out as a verdict "+
|
||
"about the uploader's file", got.Status)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// The other two branches of the same rule, driven at the level they live on: both are decided before
|
||
// the engine is called, so no hook inside the engine can reach them, and going through Accept would
|
||
// pin the fixture rather than the branch.
|
||
//
|
||
// `book directory gone` is the one that used to write `rejected` — a verdict about the uploader's
|
||
// file, produced by the platform losing its own directory.
|
||
func TestTheBranchesDecidedBeforeTheEngineAlsoLeaveNoVerdict(t *testing.T) {
|
||
for _, c := range []struct {
|
||
name string
|
||
break_ func(t *testing.T, f *fixture, dir string)
|
||
}{{
|
||
name: "book directory gone",
|
||
break_: func(t *testing.T, f *fixture, dir string) {
|
||
remove(t, dir)
|
||
},
|
||
}, {
|
||
name: "storage root gone",
|
||
break_: func(t *testing.T, f *fixture, dir string) {
|
||
remove(t, dir)
|
||
remove(t, filepath.Join(f.root, StorageMarker))
|
||
},
|
||
}} {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
// No template, so the intake's own pass reaches no verdict and leaves the book `parsing`
|
||
// with its claim given back — which is the state these two branches are decided in.
|
||
f := newFixture(t)
|
||
book := f.accept(t, "蛊真人.txt", "первая глава")
|
||
if book.Status != "parsing" {
|
||
t.Fatalf("the fixture did not reach `parsing`: %q", book.Status)
|
||
}
|
||
c.break_(t, f, filepath.Join(f.root, book.ID))
|
||
claim, err := f.store.ClaimParse(f.ctx, book.ID, f.now, f.now.Add(-time.Hour))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := f.svc.parseClaimed(f.ctx, claim, true); !errors.Is(err, errNotConclusive) {
|
||
t.Fatalf("the intake's own pass answered %v, want errNotConclusive", err)
|
||
}
|
||
if got := f.card(t, book.ID); got.Status == "rejected" {
|
||
t.Errorf("the pass left the book %q: a platform that lost its own directory is not a "+
|
||
"verdict about the uploader's file", got.Status)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// The intake does not materialize the reading surface, and does not enqueue a job it has no work for.
|
||
//
|
||
// Both are properties of the same pass and both are invisible from the response, which is why they
|
||
// are asserted on the collaborators rather than on the book: materializing runs the engine twice more
|
||
// (readmodel.MaterializeBudget) with the uploader holding the request open, and a job that exists
|
||
// while the cut is going races it for the parse claim.
|
||
//
|
||
// The debt is what brings the materializer back, so it is asserted present — «did not materialize»
|
||
// and «lost the tree» are different outcomes and only one of them is wanted.
|
||
func TestTheIntakeNeitherMaterializesNorEnqueuesWhenItsCutSucceeds(t *testing.T) {
|
||
f := newFixture(t)
|
||
templated(t, f)
|
||
reader, queue := &fakeReader{}, &countingQueue{}
|
||
f.svc.Reader, f.svc.Queue = reader, queue
|
||
|
||
book := f.accept(t, "蛊真人.txt", "первая глава\fвторая глава")
|
||
if book.Status != "not_started" {
|
||
t.Fatalf("the cut did not succeed: %q", book.Status)
|
||
}
|
||
if len(reader.claims) != 0 || len(reader.refreshed) != 0 {
|
||
t.Errorf("the intake materialized the reading surface (%d claims, %d refreshes): the uploader "+
|
||
"waits for two more engine runs and UploadSettle does not cover them",
|
||
len(reader.claims), len(reader.refreshed))
|
||
}
|
||
if queue.n != 0 {
|
||
t.Errorf("the intake enqueued %d jobs for a book it has already cut: such a job races the cut "+
|
||
"for the parse claim", queue.n)
|
||
}
|
||
// The debt stands, so the materializer's own sweep takes the book.
|
||
owed, err := f.store.BooksOwedReadModel(f.ctx, 10)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(owed) != 1 || owed[0].ID != book.ID {
|
||
t.Fatalf("the parsed book owes no reading surface (%d owed): nothing brings the materializer to it", len(owed))
|
||
}
|
||
}
|
||
|
||
// The other half of the same rule: a cut that reaches no verdict hands the book to the queue, and the
|
||
// claim goes back in the SAME transaction — either alone is a state nobody finishes.
|
||
func TestACutWithNoVerdictGivesTheClaimBackAndEnqueuesTheJob(t *testing.T) {
|
||
f := newFixture(t) // no template: the pass stops at «this book has no configuration»
|
||
queue := &countingQueue{}
|
||
f.svc.Queue = queue
|
||
|
||
book := f.accept(t, "蛊真人.txt", "первая глава")
|
||
if book.Status != "parsing" {
|
||
t.Fatalf("the cut reached a verdict it should not have: %q", book.Status)
|
||
}
|
||
if queue.n != 1 {
|
||
t.Errorf("the queue was handed %d jobs, want exactly the one that finishes this book", queue.n)
|
||
}
|
||
var claimed *time.Time
|
||
if err := f.store.Pool().QueryRow(f.ctx,
|
||
`select parse_started_at from books where id = $1`, book.ID).Scan(&claimed); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if claimed != nil {
|
||
t.Errorf("the claim is still held (%v) while a job is queued: that job will do nothing", claimed)
|
||
}
|
||
}
|
||
|
||
// countingQueue counts what the intake handed the queue. The real one inserts a row in the caller's
|
||
// transaction; what is being asserted here is WHETHER and HOW MANY, not what River does with it.
|
||
type countingQueue struct{ n int }
|
||
|
||
func (q *countingQueue) EnqueueParse(context.Context, pgstore.Tx, string) error { q.n++; return nil }
|
||
|
||
// remove takes a path away and fails the test if it could not: a case that did not break what it
|
||
// meant to break is a case that proves nothing.
|
||
func remove(t *testing.T, path string) {
|
||
t.Helper()
|
||
if err := os.RemoveAll(path); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// The job goes in ONLY where the claim was really given back, and the half that decides is
|
||
// `RowsAffected` on the release itself (pgstore.ReleaseParseClaim).
|
||
//
|
||
// Both halves are one transaction because either alone is a state nobody finishes. The guard is the
|
||
// side nothing covered: a pass whose claim has been taken over by another has nothing to hand the
|
||
// queue, and a job enqueued for a book somebody else is already parsing is a worker that wakes up,
|
||
// finds the claim held and does nothing — while the pass that DOES hold it still owes the book its
|
||
// finish. Both directions are asserted here, because a case that only ever proves the job is ABSENT
|
||
// passes just as well when the release stopped enqueueing altogether.
|
||
func TestOnlyAClaimThatWasReallyGivenBackQueuesTheJobThatFinishesTheBook(t *testing.T) {
|
||
f := newFixture(t) // no template: the intake's own pass reaches no verdict and releases once
|
||
queue := &countingQueue{}
|
||
f.svc.Queue = queue
|
||
book := f.accept(t, "蛊真人.txt", "первая глава")
|
||
if queue.n != 1 {
|
||
t.Fatalf("the intake handed the queue %d jobs, want the one that finishes this book", queue.n)
|
||
}
|
||
// Somebody takes the book: from here a release carrying any OTHER stamp is a pass talking about
|
||
// work it is no longer doing.
|
||
claim, err := f.store.ClaimParse(f.ctx, book.ID, f.now, f.now.Add(-ClaimGrace))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := f.store.ReleaseParseClaim(f.ctx, book.ID, claim.At.Add(-time.Second), f.svc.enqueue); err != nil {
|
||
t.Fatalf("a release carrying a stale stamp failed instead of doing nothing: %v", err)
|
||
}
|
||
if queue.n != 1 {
|
||
t.Errorf("a release that gave back nothing still queued a job (%d in total): the worker it wakes will find the claim held and do nothing, and the pass that holds it is still the one that owes the book",
|
||
queue.n)
|
||
}
|
||
var stamp *time.Time
|
||
if err := f.store.Pool().QueryRow(f.ctx,
|
||
`select parse_started_at from books where id = $1`, book.ID).Scan(&stamp); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if stamp == nil {
|
||
t.Fatal("a release carrying a stale stamp cleared somebody else's claim")
|
||
}
|
||
// And the holder's OWN release does both halves: without this the assertion above would hold on a
|
||
// release that has stopped queueing anything at all.
|
||
if err := f.store.ReleaseParseClaim(f.ctx, book.ID, claim.At, f.svc.enqueue); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if queue.n != 2 {
|
||
t.Errorf("the holder gave the claim back and the queue has %d jobs, want a second one: the book has nobody to finish it", queue.n)
|
||
}
|
||
if err := f.store.Pool().QueryRow(f.ctx,
|
||
`select parse_started_at from books where id = $1`, book.ID).Scan(&stamp); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if stamp != nil {
|
||
t.Errorf("the holder's own release left the claim standing (%v): the job it queued will do nothing", stamp)
|
||
}
|
||
}
|