textmachine/platform/internal/readmodel/readmodel_test.go

package readmodel

import (
	"context"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/pgstore"
)

type fakeEngine struct {
	manifest    ingest.Manifest
	export      ingest.Export
	manifestErr error
	exportErr   error
}

func (f *fakeEngine) Manifest(context.Context, string, string) (ingest.Manifest, error) {
	return f.manifest, f.manifestErr
}

func (f *fakeEngine) Export(context.Context, string, string) (ingest.Export, error) {
	return f.export, f.exportErr
}

type fakeStore struct {
	structure pgstore.Structure
	saved     bool
	bank      []ingest.BankTerm
	bankSaved bool
	owed      []pgstore.OwedBook
	cleared   []pgstore.OwedBook
	deferred  []pgstore.OwedBook
	// deferredUntil is WHEN each deferral pushed the debt to. It is the whole of the fix the register
	// row is about: `now()` put the book back at the head of the queue fifteen seconds later.
	deferredUntil []time.Time
	deferCost     []pgstore.AttemptCost
	attempts      int
	abandoned     []pgstore.OwedBook
	abandonReason string
	claimed       []pgstore.OwedBook
	// unclaimable models a debt somebody else already holds.
	unclaimable bool
	// beforeClear runs just before the discharge, so a test can spend the caller's context exactly
	// where the reads would have spent it; clearCtxErr is what the discharge saw.
	beforeClear func()
	clearCtxErr error
}

func (f *fakeStore) SaveStructure(_ context.Context, _ string, in pgstore.Structure) error {
	f.structure, f.saved = in, true
	return nil
}

func (f *fakeStore) SaveBank(_ context.Context, _ string, terms []ingest.BankTerm) error {
	f.bank, f.bankSaved = terms, true
	return nil
}

func (f *fakeStore) BooksOwedReadModel(_ context.Context, limit int) ([]pgstore.OwedBook, error) {
	if len(f.owed) > limit {
		return f.owed[:limit], nil
	}
	return f.owed, nil
}

func (f *fakeStore) ClaimReadModelDebt(_ context.Context, bookID string, owedAt time.Time, window time.Duration) (time.Time, error) {
	if f.unclaimable {
		return time.Time{}, nil
	}
	f.claimed = append(f.claimed, pgstore.OwedBook{ID: bookID, OwedAt: owedAt})
	return owedAt.Add(window), nil
}

func (f *fakeStore) DeferReadModelDebt(_ context.Context, bookID string, owedAt, next time.Time, reason string, cost pgstore.AttemptCost) (int, error) {
	f.deferred = append(f.deferred, pgstore.OwedBook{ID: bookID, OwedAt: owedAt})
	f.deferredUntil = append(f.deferredUntil, next)
	f.deferCost = append(f.deferCost, cost)
	if cost == pgstore.SpendsAnAttempt {
		f.attempts++
	}
	return f.attempts, nil
}

func (f *fakeStore) AbandonReadModelDebt(_ context.Context, bookID string, owedAt, _ time.Time, reason string) error {
	f.abandoned = append(f.abandoned, pgstore.OwedBook{ID: bookID, OwedAt: owedAt})
	f.abandonReason = reason
	return nil
}

func (f *fakeStore) ClearReadModelDebt(ctx context.Context, bookID string, owedAt time.Time) error {
	if f.beforeClear != nil {
		f.beforeClear()
	}
	f.clearCtxErr = ctx.Err()
	f.cleared = append(f.cleared, pgstore.OwedBook{ID: bookID, OwedAt: owedAt})
	return nil
}

// owedBook is the book every test below refreshes: an id, a workdir and the boundary that owes it.
func owedBook(workdir string) pgstore.OwedBook {
	return pgstore.OwedBook{ID: "bk_1", Workdir: workdir, OwedAt: boundary}
}

var boundary = time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)

func tree() ingest.Manifest {
	return ingest.Manifest{
		Version: "tm-manifest-v2", Key: "k1", ChaptersTotal: 2, UnitsTotal: 3,
		Chapters: []ingest.ManifestChapter{
			{ID: "c1", Number: 1, UnitsTotal: 2, Units: []ingest.ManifestUnit{
				{ID: "c1:cut:0", FirstChunkIdx: 0}, {ID: "c1:cut:4", FirstChunkIdx: 4},
			}},
			{ID: "c2", Number: 2, UnitsTotal: 1, Units: []ingest.ManifestUnit{{ID: "c2:cut:0", FirstChunkIdx: 0}}},
		},
	}
}

// The two documents are joined by the key the engine publishes for exactly this — the chapter's
// ordinal and the unit's LEADER chunk index — and a pair the export says nothing about stays
// `pending` with no text rather than inheriting a neighbour's.
//
// Mutation caught: joining by position in the array instead of by (chapter, first_chunk_idx).
func TestTheTextIsJoinedOntoTheTreeByTheEnginesOwnKey(t *testing.T) {
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{
		manifest: tree(),
		export: ingest.Export{TotalUnits: 3, Units: []ingest.UnitText{
			{Chapter: 1, Unit: 4, Source: "第二节", Target: "Второй", State: ingest.StateTranslated},
			{Chapter: 2, Unit: 0, Source: "第三节", State: ingest.StateWithheld},
		}},
	}}
	// ⚠ The BANK half fails here — this workdir has no engine artifacts — and the tree lands anyway:
	// each channel is a different question about the same book, and answering two of three is
	// strictly better than answering none.
	if err := svc.Refresh(t.Context(), owedBook(t.TempDir())); err == nil {
		t.Error("a workdir with no bank read-out reported no failure at all")
	}
	if !store.saved || len(store.structure.Chapters) != 2 {
		t.Fatalf("structure: %+v", store.structure)
	}
	first := store.structure.Chapters[0]
	if first.Units[0].Ordinal != 0 || first.Units[0].State != ingest.StatePending || first.Units[0].Target != "" {
		t.Errorf("a pair the export did not mention: %+v", first.Units[0])
	}
	if first.Units[1].Target != "Второй" || first.Units[1].State != ingest.StateTranslated {
		t.Errorf("the pair at leader index 4: %+v", first.Units[1])
	}
	second := store.structure.Chapters[1]
	if second.Units[0].Source != "第三节" || second.Units[0].State != ingest.StateWithheld {
		t.Errorf("the pair of the second chapter: %+v", second.Units[0])
	}
	if store.structure.ManifestKey != "k1" {
		t.Errorf("the validity key did not travel: %q", store.structure.ManifestKey)
	}
}

// A book that has never been translated has a tree and no translations, and so does a book whose
// export this build could not read. The TREE lands either way: a reader screen that stayed empty
// because the text channel failed would hide a book the user can already navigate.
//
// Mutation caught: returning early when Export fails; claiming to KNOW the text that failed to read.
func TestAFailedTextReadStillLandsTheTree(t *testing.T) {
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{
		manifest: tree(), exportErr: errors.New("the engine refused"),
	}}
	_ = svc.Refresh(t.Context(), owedBook(t.TempDir()))
	if !store.saved {
		t.Fatal("a failed text read cost the whole tree")
	}
	for _, c := range store.structure.Chapters {
		for _, u := range c.Units {
			if u.State != ingest.StatePending || u.Target != "" {
				t.Errorf("a pair claims a translation nothing produced: %+v", u)
			}
			// …and it says so, which is what stops the store overwriting the stored text.
			if u.TextKnown {
				t.Errorf("a pair claims its text is known after the export failed: %+v", u)
			}
		}
	}
}

// The two engine calls are two re-cuts of the same source, and they can disagree: a pair in the
// manifest that the export did not carry is a SKEW, not an empty pair, and must not be written as
// one over text already materialized.
//
// Mutation caught: setting TextKnown for every pair of a successful export.
func TestAPairTheExportDidNotCarryDoesNotSpeakAboutItsText(t *testing.T) {
	// The manifest has three pairs; the export carries ONE of them, as a cut that moved between the
	// two calls would leave it.
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{
		manifest: tree(),
		export: ingest.Export{TotalUnits: 3, Units: []ingest.UnitText{
			{Chapter: 1, Unit: 0, Source: "第一节", Target: "Первый", State: ingest.StateTranslated},
		}},
	}}
	// The bank half fails on an empty workdir, which is not what this pins.
	_ = svc.Refresh(t.Context(), owedBook(t.TempDir()))
	var known, unknown int
	for _, c := range store.structure.Chapters {
		for _, u := range c.Units {
			if u.TextKnown {
				known++
			} else {
				unknown++
			}
		}
	}
	if known != 1 || unknown == 0 {
		t.Errorf("%d pairs claim known text and %d do not, want only the exported one", known, unknown)
	}
}

// A book with no bank read-out has never produced terms, and that is not an empty bank: saving one
// would erase a bank the engine simply has not written yet.
//
// Mutation caught: treating ErrNoBank as an empty read-out.
func TestAMissingBankReadOutSavesNothing(t *testing.T) {
	// ⚠ The book IS configured and simply has no bank sidecar yet — which is the branch this pins.
	// Without the configuration the read failed one step earlier, on the missing book.yaml, and the
	// declared property was never reached: "a read-out that has not been made must not erase one that
	// was" was asserted by a test that could not tell the two failures apart.
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, "book.yaml"),
		[]byte("book_id: bk_1\nproject_db: bk_1.db\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{manifest: tree()}}
	if err := svc.Refresh(t.Context(), owedBook(dir)); err != nil {
		t.Fatalf("a book that has never produced terms is not a failure: %v", err)
	}
	if store.bankSaved {
		t.Error("an absent bank read-out was saved as an empty bank")
	}
	if !store.saved {
		t.Error("the tree was not materialized, though only the bank was missing")
	}
}

// A workdir with no configuration at all is a different fact, and it IS a failure: nothing about
// this book can be located, so the refresh says so instead of reporting an empty book.
func TestAWorkdirWithNoConfigurationIsAFailureRatherThanAnEmptyBank(t *testing.T) {
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{manifest: tree()}}
	if err := svc.Refresh(t.Context(), owedBook(t.TempDir())); err == nil {
		t.Fatal("a workdir with no configuration was read as a book with no bank")
	}
	if store.bankSaved {
		t.Error("an unreadable configuration saved a bank anyway")
	}
}

// configured is a workdir the bank channel can answer about: the book exists and has simply never
// produced terms, so the read-out is absent rather than unreadable.
func configured(t *testing.T) string {
	t.Helper()
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, "book.yaml"),
		[]byte("book_id: bk_1\nproject_db: bk_1.db\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	return dir
}

// The debt is what brings the materializer back to a book, so only a materialization that answered
// EVERY channel may discharge it. A tree with no text is not a materialized book: before this the
// text channel's failure was logged and the caller was told the book was done.
//
// Mutation caught: returning nil from refreshStructure when Export fails.
func TestAPartialMaterializationDoesNotDischargeTheDebt(t *testing.T) {
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{
		manifest: tree(), exportErr: errors.New("the engine refused"),
	}}
	if err := svc.Refresh(t.Context(), owedBook(configured(t))); err == nil {
		t.Fatal("a materialization with no text reported success")
	}
	if !store.saved {
		t.Fatal("this fixture never reached the write, so it cannot observe the property")
	}
	if len(store.cleared) != 0 {
		t.Errorf("a partial materialization discharged the debt: %+v", store.cleared)
	}
}

// And a complete one discharges THAT boundary, carried through unchanged. Whether the STORE then
// refuses a debt stamped since is pgstore's own property and is pinned there, on a live database —
// here what is checked is that the boundary reaches it at all.
//
// Mutation caught: discharging with the current time instead of the boundary the caller was given.
func TestACompleteMaterializationDischargesTheBoundaryItRead(t *testing.T) {
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{
		manifest: tree(),
		export: ingest.Export{TotalUnits: 3, Units: []ingest.UnitText{
			{Chapter: 1, Unit: 0, Source: "第一节", Target: "Первый", State: ingest.StateTranslated},
		}},
	}}
	if err := svc.Refresh(t.Context(), owedBook(configured(t))); err != nil {
		t.Fatal(err)
	}
	if len(store.cleared) != 1 || store.cleared[0].ID != "bk_1" || !store.cleared[0].OwedAt.Equal(boundary) {
		t.Errorf("the debt discharged: %+v, want the boundary this read answered", store.cleared)
	}
}

// A materialization that HAPPENED is written down even when the context that paid for the reads is
// already spent. The reads legitimately consume the whole budget on a large book, and a discharge
// that then fails leaves the debt standing — so the next pass re-does two full re-chunks of the same
// source, and every pass after it.
//
// Mutation caught: discharging on the caller's own context.
func TestTheDischargeSurvivesAContextTheReadsUsedUp(t *testing.T) {
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{
		manifest: tree(),
		export: ingest.Export{TotalUnits: 3, Units: []ingest.UnitText{
			{Chapter: 1, Unit: 0, Source: "第一节", Target: "Первый", State: ingest.StateTranslated},
		}},
	}}
	ctx, cancel := context.WithCancel(t.Context())
	store.beforeClear = cancel // the budget runs out exactly between the last read and the record
	if err := svc.Refresh(ctx, owedBook(configured(t))); err != nil {
		t.Fatal(err)
	}
	if len(store.cleared) != 1 {
		t.Fatalf("the debt was not discharged at all: %+v", store.cleared)
	}
	if store.clearCtxErr != nil {
		t.Errorf("the discharge ran on a context that was already over: %v", store.clearCtxErr)
	}
}

// The drain is the only retry there is, so it must reach every owed book and must not let one book's
// failure end the pass: these are independent books of independent accounts.
//
// Mutation caught: returning on the first error.
func TestTheDrainMaterializesEveryOwedBookAndKeepsWhatFailed(t *testing.T) {
	store := &fakeStore{owed: []pgstore.OwedBook{
		{ID: "bk_1", Workdir: t.TempDir(), OwedAt: boundary}, // no configuration: the bank read fails
		{ID: "bk_2", Workdir: configured(t), OwedAt: boundary},
	}}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{manifest: tree()}}
	if err := svc.Drain(t.Context()); err != nil {
		t.Fatal(err)
	}
	if len(store.cleared) != 1 || store.cleared[0].ID != "bk_2" {
		t.Errorf("discharged %+v, want only the book that was materialized whole", store.cleared)
	}
	// …and the one that failed goes to the BACK of the queue. The list is oldest-first, so a book the
	// engine can never answer about would hold the front of it forever and the books behind it would
	// never be materialized at all.
	if len(store.deferred) != 1 || store.deferred[0].ID != "bk_1" {
		t.Errorf("deferred %+v, want the book whose materialization failed", store.deferred)
	}
}

// A deployment with no engine binary is a read replica, not a broken one: a refresh is a no-op and
// the surface it serves is whatever was materialized before.
func TestWithoutAnEngineARefreshIsANoOp(t *testing.T) {
	store := &fakeStore{}
	svc := &Service{Store: store, Engine: &fakeEngine{manifest: tree()}}
	if err := svc.Refresh(t.Context(), owedBook("/srv/books/bk_1")); err != nil {
		t.Fatal(err)
	}
	if store.saved || store.bankSaved {
		t.Error("a deployment with no engine binary materialized something")
	}
	// …and it does not discharge the debt either: the work belongs to an instance that has an engine.
	if len(store.cleared) != 0 {
		t.Errorf("a replica cleared a debt it did not pay: %+v", store.cleared)
	}
}

// The engine's manifest carries a `heading` and the contract forbids a deployment to project it —
// it is a rendered ordinal («Глава N»), not a label out of the book's data. The allowlist is
// enforced by the DECODER: there is nowhere for the field to land.
//
// Mutation caught: adding a Heading field to ingest.ManifestChapter.
func TestTheManifestsRenderedHeadingHasNowhereToLand(t *testing.T) {
	doc := `{"manifest_version":"tm-manifest-v2","key":"k","chapters":[
		{"id":"c1","number":1,"heading":"Глава 1","units_total":1,
		 "units":[{"id":"c1:cut:0","first_chunk_idx":0}]}]}`
	m, err := ingest.DecodeManifest([]byte(doc))
	if err != nil {
		t.Fatal(err)
	}
	if len(m.Chapters) != 1 || m.Chapters[0].ID != "c1" {
		t.Fatalf("manifest: %+v", m)
	}
	// Whatever the engine renders, this side has no field holding it: the check is over the decoded
	// STRUCT, so a field added later fails here rather than on a reader's screen.
	if rendered := fmt.Sprintf("%#v", m.Chapters[0]); strings.Contains(rendered, "Глава") {
		t.Errorf("the engine's rendered ordinal survived the decode: %s", rendered)
	}
}

// Two materializations of one book must not run at once. The intake pays its own debt inline while
// the sweep drains the rest every few seconds, so without a claim every upload slower than one sweep
// interval was read by both at the same time — two full re-chunks of the source, which is the cost
// this package removed elsewhere.
//
// Mutation caught: draining without claiming; ignoring a claim somebody else holds.
func TestTheDrainSkipsABookSomebodyElseIsAlreadyMaterializing(t *testing.T) {
	store := &fakeStore{owed: []pgstore.OwedBook{{ID: "bk_1", Workdir: configured(t), OwedAt: boundary}}}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{manifest: tree()}}
	store.unclaimable = true
	if err := svc.Drain(t.Context()); err != nil {
		t.Fatal(err)
	}
	if store.saved {
		t.Error("the drain read the engine for a book it does not hold the debt of")
	}
	// …and it DOES take a debt nobody holds — the claim is a lease, not a refusal.
	store.unclaimable = false
	if err := svc.Drain(t.Context()); err != nil {
		t.Fatal(err)
	}
	if len(store.claimed) != 1 || !store.saved {
		t.Errorf("claimed %+v, saved=%v — want the free debt taken and paid", store.claimed, store.saved)
	}
	// The discharge presents the stamp the CLAIM produced, not the one the queue was listed with: the
	// claim moved it, and a discharge holding the old value would clear nothing.
	if len(store.cleared) != 1 || !store.cleared[0].OwedAt.Equal(boundary.Add(MaterializeBudget)) {
		t.Errorf("discharged %+v, want the stamp the claim produced", store.cleared)
	}
}

// A manifest whose lists are shorter than its own counts NEVER reaches the store.
//
// The tree is written by replacement — `SaveStructure` deletes every chapter outside the list it is
// handed and the cascade takes the text with it — so an empty list is the deletion of the whole
// book. No engine produces one (`buildManifest` fills the chapters and the file is written
// atomically), which is exactly why the danger is silent: the case this refuses is not a broken
// engine but a document THIS build read wrongly, and the counts printed beside the lists are the
// only witness it has of that.
//
// The asymmetry that made it a defect: the intake refuses precisely this document and has since P6,
// because there "no chapters" is the verdict that deletes a user's upload. The materializer's write
// is as destructive and checked nothing.
//
// Mutation caught: dropping either floor from refreshStructure — the self-description check or the
// "no chapters at all" one.
func TestAManifestThatDoesNotDescribeItselfNeverReachesTheTree(t *testing.T) {
	for name, broken := range map[string]func(*ingest.Manifest){
		"a chapter list this build could not read": func(m *ingest.Manifest) { m.Chapters = nil },
		"one chapter short":                        func(m *ingest.Manifest) { m.Chapters = m.Chapters[:1] },
		"a unit list this build could not read":    func(m *ingest.Manifest) { m.Chapters[0].Units = nil },
		"counts no chapters at all": func(m *ingest.Manifest) {
			m.Chapters, m.ChaptersTotal, m.UnitsTotal = nil, 0, 0
		},
		// The BOOK-level pair count disagreeing with the sum over chapters. Its own case because the
		// per-chapter check cannot see it: every chapter can describe itself correctly while the
		// document as a whole says a different number of pairs — which is what a renamed field does.
		"counts pairs the chapters do not carry": func(m *ingest.Manifest) { m.UnitsTotal = 99 },
		// …and a chapter numbered from zero, which chapter numbering cannot produce.
		"a chapter numbered zero": func(m *ingest.Manifest) { m.Chapters[0].Number = 0 },
		// A per-chapter count that is wrong while the BOOK's total still adds up. Its own case because
		// nothing else reaches it: every other broken shape here also breaks the book-level sum, so the
		// per-chapter equality could be deleted and the battery would not notice.
		"chapter counts that swap while the book's total holds": func(m *ingest.Manifest) {
			m.Chapters[0].UnitsTotal, m.Chapters[1].UnitsTotal = 1, 2
		},
		// The identities, which no counter stands beside: a renamed key decodes to "" for every row and
		// the replacement write then keeps one of them.
		"a chapter with no id": func(m *ingest.Manifest) { m.Chapters[0].ID = "" },
		"a pair with no id":    func(m *ingest.Manifest) { m.Chapters[0].Units[1].ID = "" },
		"no identities at all, as a renamed key decodes": func(m *ingest.Manifest) {
			for i := range m.Chapters {
				m.Chapters[i].ID = ""
				for j := range m.Chapters[i].Units {
					m.Chapters[i].Units[j].ID = ""
				}
			}
		},
	} {
		t.Run(name, func(t *testing.T) {
			m := tree()
			broken(&m)
			store := &fakeStore{}
			svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{manifest: m}}
			err := svc.Refresh(t.Context(), owedBook(t.TempDir()))
			if err == nil {
				t.Fatal("the refresh reported success")
			}
			if store.saved {
				t.Errorf("the tree was rewritten from a document that does not describe itself: %+v", store.structure)
			}
			// …and the debt is NOT discharged: the reader keeps the tree it has and the book comes back.
			if len(store.cleared) != 0 {
				t.Errorf("the debt was discharged by a materialization that wrote nothing: %+v", store.cleared)
			}
		})
	}
}

// …and the same document reaching through the INTAKE's door is refused too. `RefreshCut` exists to
// save a third re-chunk per upload by passing the manifest the intake already decoded, and a floor
// on only one of the two entrances is a floor on neither.
func TestTheIntakesOwnCutIsHeldToTheSameFloor(t *testing.T) {
	m := tree()
	m.Chapters = nil
	store := &fakeStore{}
	svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{manifest: tree()}}
	if err := svc.RefreshCut(t.Context(), owedBook(t.TempDir()), m); err == nil {
		t.Fatal("the cut handed in by the intake was written without being checked")
	}
	if store.saved {
		t.Errorf("the tree was rewritten from the intake's own broken cut: %+v", store.structure)
	}
}

// A host that cannot run the engine does not write off every book on it.
//
// The attempt budget answers "this book cannot be read". An engine binary that is missing, locked
// out by another process or refusing an unmigrated schema says nothing about any book and applies to
// all of them at once — so counting it would abandon the whole library within a few passes, each
// book keeping whatever stale surface it had. The intake holds the same rule (books.defer_).
//
// Mutation caught: spending an attempt on a deployment fault; not deferring one at all.
func TestABrokenDeploymentDoesNotSpendABooksAttempts(t *testing.T) {
	for _, c := range []struct {
		name string
		err  error
	}{
		{"the binary cannot be run", fmt.Errorf("readmodel: read the manifest: %w", exec.ErrNotFound)},
		{"the project is locked", exitError(t, ingest.ExitProjectLocked)},
		{"the schema is not migrated", exitError(t, ingest.ExitSchemaMismatch)},
	} {
		t.Run(c.name, func(t *testing.T) {
			store := &fakeStore{owed: []pgstore.OwedBook{
				// One failure short of the write-off: if this pass counts, the book is given up on.
				{ID: "bk_1", Workdir: t.TempDir(), OwedAt: boundary, Attempts: maxAttempts - 1},
			}}
			svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{manifestErr: c.err}}
			if err := svc.Drain(t.Context()); err != nil {
				t.Fatal(err)
			}
			if len(store.abandoned) != 0 {
				t.Errorf("a book was given up on over a fault of the host: every book on it would be, within a few passes")
			}
			if len(store.deferred) != 1 {
				t.Fatalf("the debt was not put back: %+v", store.deferred)
			}
			if store.deferCost[0] != pgstore.CostsNoAttempt {
				t.Errorf("the deployment's fault was charged to the book's budget")
			}
		})
	}
}

// exitError is what an engine that ANSWERED with a refusal looks like from the runner: the platform
// tells that from "could not be run" by the type, so a fixture has to produce a real one.
func exitError(t *testing.T, code int) error {
	t.Helper()
	err := exec.CommandContext(t.Context(), "sh", "-c", "exit "+strconv.Itoa(code)).Run()
	var exit *exec.ExitError
	if !errors.As(err, &exit) || exit.ExitCode() != code {
		t.Fatalf("the fixture could not produce exit %d: %v", code, err)
	}
	return fmt.Errorf("readmodel: read the manifest: %w", err)
}

// A materialization that failed waits a REAL interval, and after enough of them the debt is given up
// on rather than retried for the life of the deployment.
//
// What was there before is the whole defect: claim, fail, `DeferReadModelDebt` writing `now()`, and
// the sweep running again fifteen seconds later — so "the back of the queue" was a queue of one and
// the book was re-read by the engine four times a minute, forever. Three things followed and every
// one of them was ongoing: up to five minutes of engine processes per pass; a book whose event
// stream can NEVER end, because `AtRest` requires the debt to be null and a browser reconnects to it
// for good; and, where the manifest reads and the export does not, a committed write on every pass —
// a revision bump and a frame every fifteen seconds to say nothing changed.
//
// Giving up is safe HERE in a way it never is for a run: the money of the boundary that stamped this
// debt has already settled, so what is lost is freshness of text. It is also not final — the next
// boundary of real work stamps a fresh debt (owesAReadingSurface resets the budget).
//
// Mutation caught: deferring to `now()`; not counting the attempt; never abandoning; abandoning on
// the first failure.
func TestAnUnpayableDebtBacksOffAndIsEventuallyGivenUpOn(t *testing.T) {
	// A workdir with no engine configuration: the bank read fails, so the materialization never
	// completes — the shape of a project directory an operator moved.
	broken := t.TempDir()
	for attempts := range maxAttempts {
		store := &fakeStore{owed: []pgstore.OwedBook{
			{ID: "bk_1", Workdir: broken, OwedAt: boundary, Attempts: attempts},
		}}
		svc := &Service{Store: store, Binary: "tmctl", Engine: &fakeEngine{manifest: tree()}}
		before := time.Now()
		if err := svc.Drain(t.Context()); err != nil {
			t.Fatal(err)
		}
		if len(store.cleared) != 0 {
			t.Fatalf("attempt %d: a failed materialization discharged the debt: %+v", attempts, store.cleared)
		}
		last := attempts+1 >= maxAttempts
		switch {
		case last && len(store.abandoned) != 1:
			t.Errorf("after %d failures the debt is still being retried; nothing ever ends this loop", attempts+1)
		case last && store.abandonReason == "":
			t.Error("the debt was written off with no reason recorded: an operator cannot tell why")
		case !last && len(store.abandoned) != 0:
			t.Errorf("the debt was given up on after %d failures, before its budget was spent", attempts+1)
		case !last && len(store.deferredUntil) != 1:
			t.Fatalf("attempt %d: deferred %d times, want once", attempts, len(store.deferredUntil))
		case !last && !store.deferredUntil[0].After(before.Add(30*time.Second)):
			// The number that matters: a deferral shorter than the sweep interval is not a deferral.
			t.Errorf("attempt %d: the debt was pushed to %v, barely past %v — the next pass picks it straight back up",
				attempts, store.deferredUntil[0], before)
		case !last && !store.deferredUntil[0].After(before.Add(retryIn(attempts+1)-time.Second)):
			// …and it GROWS with the count. Asserting the function alone leaves the CALL free to hand it
			// a constant, which is a flat one-minute retry wearing the shape of a backoff.
			t.Errorf("attempt %d: deferred to %v, which is not the %v this many failures have earned",
				attempts, store.deferredUntil[0], retryIn(attempts+1))
		}
	}
}

// …and the delay grows and stops growing. Without a ceiling a book that healed would wait days;
// without growth the first retry is the only one that costs little.
func TestTheMaterializationBackoffGrowsAndIsCapped(t *testing.T) {
	if retryIn(1) != time.Minute {
		t.Errorf("the first retry is %v, want a minute", retryIn(1))
	}
	if retryIn(3) <= retryIn(1) {
		t.Errorf("the retry does not grow: %v then %v", retryIn(1), retryIn(3))
	}
	if got := retryIn(1000); got != 30*time.Minute {
		t.Errorf("the retry of a long-dead book is %v, want the cap", got)
	}
}