textmachine/platform/internal/runs/resumeorder_test.go

package runs

import (
	"context"
	"errors"
	"strings"
	"testing"
	"time"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/money"
	"textmachine/platform/internal/pgstore"
	"textmachine/platform/internal/runner"
)

// THE ANSWER A DOUBLE RESUME GIVES MUST NOT DEPEND ON WHICH CALL GOT THERE FIRST (D39.246 п.6).
//
// Two callers reach the same instant from opposite sides — one reads the run while it is still
// `stopped` and loses the re-open to the unique index, the other reads it already `translating` —
// and a client cannot tell which of the two it was. Both are pinned, each by a fixture in which its
// own order is the only one reachable, and both assert through `bothResumesAnswered` so that the
// assertion itself cannot drift between them.

// bothResumesAnswered is the whole guarantee, in the form both orders must satisfy.
//
// ⛔ THE HOLD IS COUNTED, NOT INFERRED FROM THE ABSENCE OF A SECOND. "No second hold was taken" is
// satisfied by a fixture in which a second hold was impossible for any reason at all; a COUNT of the
// open reservations of this run is satisfied only by there being exactly one, and it names the
// figure it carries, so a hold of the wrong size cannot pass either.
func bothResumesAnswered(t *testing.T, f *fixture, runID string, runs []pgstore.Run, errs []error, hold money.MicroUSD) {
	t.Helper()
	for i, err := range errs {
		if err != nil {
			t.Fatalf("resume %d: %v — the run is continuing, and a refusal makes the answer depend on "+
				"which call reached the database first", i, err)
		}
		if runs[i].ID != runID || runs[i].Status != "translating" {
			t.Errorf("resume %d answered run %q in %q, want %q translating", i, runs[i].ID, runs[i].Status, runID)
		}
		if runs[i].StopRequested {
			t.Errorf("resume %d answered a run marked stop_requested while no stop was asked for", i)
		}
	}
	var holds int
	var held int64
	if err := f.store.Pool().QueryRow(f.ctx, `
		select count(*), coalesce(sum(amount_micro_usd), 0) from reservations
		 where engine_run_id like $1 || '#%' and state = 'open'`, runID).Scan(&holds, &held); err != nil {
		t.Fatal(err)
	}
	if holds != 1 || money.MicroUSD(held) != hold {
		t.Fatalf("%d open holds of %s after two resumes, want exactly one of %s",
			holds, money.MicroUSD(held).USD(), hold.USD())
	}
	var attempts int
	if err := f.store.Pool().QueryRow(f.ctx,
		`select count(*) from run_attempts where run_id = $1`, runID).Scan(&attempts); err != nil {
		t.Fatal(err)
	}
	if attempts != 2 {
		t.Fatalf("%d attempts after two resumes, want 2: the run was re-opened twice", attempts)
	}
	acct := f.account(t)
	if acct.Reserved != hold {
		t.Errorf("the account reserves %s, want %s", acct.Reserved.USD(), hold.USD())
	}
	if acct.Balance != acct.LedgerSum {
		t.Fatalf("the cached balance and the ledger disagree: %s vs %s", acct.Balance.USD(), acct.LedgerSum.USD())
	}
}

// otherInstance is a SECOND service over the same database — a second daemon of one deployment.
//
// It is what makes the contended order reachable at all: the book lock (bank.go) is IN-PROCESS, so
// two resumes on one service are serialized before either reaches the database and the loser's
// snapshot is whatever the scheduler gave it. Two services share no such lock, which is the shape
// this test's own header always described.
func (f *fixture) otherInstance(t *testing.T) *Service {
	t.Helper()
	s := service(t, f.runner, f.engine, f.now)
	s.Store = f.store
	s.Pricing = f.svc.Pricing
	return s
}

func stoppedByAFailure(f *fixture) runner.Marker {
	return runner.Marker{Result: "exit-code", Code: "exited", Status: "1", At: f.now.Add(time.Second)}
}

// THE SECOND CALL READS THE RUN ALREADY GOING — the order the tree used to refuse, and the one that
// needs no race at all: two clicks a second apart produce it every time.
//
// Measured before the branch below existed: the second call answered
// `runs: the run cannot be continued: it is translating`, five times out of five on an idle machine,
// while the concurrent fixture next door took the other path five times out of five. One defect, two
// answers, chosen by timing (PD-448).
func TestAResumeOfARunAlreadyGoingAnswersWithThatRun(t *testing.T) {
	f := newFixture(t, "10", 500)
	spent := money.MicroUSD(500_000)
	runID := f.stopped(t, 100, spent, stoppedByAFailure(f))

	first, err := f.svc.Resume(f.ctx, "u1", runID)
	if err != nil {
		t.Fatal(err)
	}
	// THE BOUNDARY THIS FIXTURE HAS TO CROSS, asserted rather than assumed: the run is committed as
	// `translating` before the second call starts, so that call decides on a snapshot that has the
	// winner's work in it. Without this the test would pass over two calls that both read `stopped`,
	// which is the OTHER order and is pinned by its own test.
	between, err := f.store.ReadRun(f.ctx, "u1", runID)
	if err != nil {
		t.Fatal(err)
	}
	if between.Status != "translating" {
		t.Fatalf("the run is %q between the two calls, so the second one never met the state this test is "+
			"about", between.Status)
	}
	second, err2 := f.svc.Resume(f.ctx, "u1", runID)
	bothResumesAnswered(t, f, runID, []pgstore.Run{first, second}, []error{err, err2}, fixtureHold(100)-spent)
}

// A LIVE RE-PASS IS CONTINUING TOO, and this is the half that makes the fix whole rather than half.
//
// The guard that refuses to resume a re-pass stands BELOW the status decision on purpose: it guards
// the RE-OPEN — a re-pass is bought again rather than continued — and a re-pass that is running has
// nothing to buy again. With the guard above, the defect would have been closed for one kind of
// purchase and left open for the other, which is the same user pressing the same button.
func TestALiveRePassAnswersWithTheRunAndOnlyAFinishedOneIsBoughtAgain(t *testing.T) {
	f := newFixture(t, "10", 5)
	f.stoppedRun(t, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(3)}, 0,
		runner.Marker{Result: "exit-code", Code: "exited", Status: "3", At: f.now.Add(time.Second)})
	fake := &fakeBankApplier{
		out: runner.BankApplyOutcome{ExitCode: 0, Exited: true, Report: okReport("apply"), Decoded: true},
	}
	fake.out.Report.BookID = f.bookID(t)
	f.svc.Bank = fake
	if _, err := f.svc.ApplyBankCorrections(f.ctx, BankCorrectionsInput{
		UserID: "u1", BookID: f.bookID(t), Preview: false,
		Decisions: []ingest.BankDecision{{Action: "decline", ID: "tm_1"}},
	}); err != nil {
		t.Fatal(err)
	}
	run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), RePass: true})
	if err != nil {
		t.Fatal(err)
	}
	if run.Status != "translating" || run.OrderedChapters == nil || *run.OrderedChapters != 0 {
		t.Fatalf("the fixture's re-pass is %q with ordered chapters %v, want a live run that bought no "+
			"chapters — which is what puts it on the guard's side of the door", run.Status, run.OrderedChapters)
	}
	before := f.account(t)
	got, err := f.svc.Resume(f.ctx, "u1", run.ID)
	if err != nil {
		t.Fatalf("resuming a re-pass that is RUNNING answered %v, want the run: it is continuing, and "+
			"«bought again» is the answer for one that has ended", err)
	}
	if got.ID != run.ID || got.Status != "translating" {
		t.Errorf("the answer is run %q in %q, want %q translating", got.ID, got.Status, run.ID)
	}
	if after := f.account(t); after.Reserved != before.Reserved || after.Balance != before.Balance {
		t.Errorf("an answer about a running re-pass moved money: %+v -> %+v", before, after)
	}
	// The control half, and without it the assertion above would be satisfied by a door that stopped
	// refusing re-passes altogether: once the run has ENDED, the purchase is available again and this
	// call is not how it is made.
	//
	// ⛔ THE FIXTURE HAS TO LEAVE THE GUARD AS THE ONLY THING THAT CAN REFUSE, and it did not before
	// the adversarial pass of 11.09 found it: with the guard DELETED this half still passed 8 times
	// out of 8, satisfied by two refusals standing either side of it. Both are closed here rather
	// than asserted around. The run's previous attempt keeps an OPEN reservation, so `reopen` answers
	// «its previous attempt is still being settled» before the guard is ever consulted; and the
	// fixture's clock is frozen, so this run and the book's earlier one carry the SAME `started_at`
	// and `newestRun` breaks the tie by id — a coin that decides whether the door says «a newer run
	// of this book exists» instead. `errors.Is(…, ErrNotResumable)` cannot tell any of the three
	// apart, which is why the guard's own words are asserted below as well.
	if _, err := f.store.Pool().Exec(f.ctx, `
		update runs set status = 'stopped', finished_at = $2, settled_at = $2, started_at = $2
		 where id = $1`, run.ID, f.now.Add(2*time.Second)); err != nil {
		t.Fatal(err)
	}
	if _, err := f.store.Pool().Exec(f.ctx, `
		update reservations set state = 'released', closed_at = $2
		 where engine_run_id like $1 || '#%' and state = 'open'`, run.ID, f.now.Add(2*time.Second)); err != nil {
		t.Fatal(err)
	}
	_, err = f.svc.Resume(f.ctx, "u1", run.ID)
	if !errors.Is(err, ErrNotResumable) {
		t.Fatalf("resuming an ENDED re-pass answered %v, want ErrNotResumable (bought again)", err)
	}
	if !strings.Contains(err.Error(), "bought again") {
		t.Fatalf("the refusal is %q, and this half is about ONE of them: the re-pass guard. Any other "+
			"refusal of the door satisfies errors.Is and leaves the guard unmeasured", err)
	}
}

// THE CUT IN THE IDEMPOTENT ANSWER: a run that has been asked to stop is not "continuing".
//
// D39.240 keeps the stop HARD, and a 202 that says a stopping run is continuing is the half-state
// that invariant forbids — the user would read their own stop as undone. The cause is its own word
// because the remedy is the one thing the other refusals of this code rule out: waiting works.
func TestAResumeOfALiveRunAlreadyAskedToStopIsRefusedWithItsOwnCause(t *testing.T) {
	f := newFixture(t, "10", 500)
	run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)})
	if err != nil {
		t.Fatal(err)
	}
	if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
		t.Fatal(err)
	}
	// The control half FIRST, on the very same run: with no stop asked for, this call answers with
	// the run. Without it the refusal below would be satisfied by a fixture in which the resume was
	// refused for some entirely different reason.
	going, err := f.svc.Resume(f.ctx, "u1", run.ID)
	if err != nil {
		t.Fatalf("a live run with no stop asked for answered %v, want the run", err)
	}
	if going.StopRequested {
		t.Fatal("the control run already carries stop_requested: the fixture cannot tell the two halves apart")
	}
	if _, err := f.svc.Stop(f.ctx, "u1", run.ID); err != nil {
		t.Fatal(err)
	}
	before := f.account(t)
	_, err = f.svc.Resume(f.ctx, "u1", run.ID)
	if !errors.Is(err, ErrStopRequested) {
		t.Fatalf("resuming a run under a stop answered %v, want ErrStopRequested: answering 202 would read "+
			"as the stop having been undone", err)
	}
	if !errors.Is(err, ErrNotResumable) {
		t.Errorf("the refusal is not a kind of ErrNotResumable, so the wire would not answer run_not_resumable: %v", err)
	}
	if after := f.account(t); after.Reserved != before.Reserved || after.Balance != before.Balance {
		t.Errorf("the refused resume moved money: %+v -> %+v", before, after)
	}
}

// ⛔ THE ANSWER IS CHECKED AGAINST THE ROW IT IS MADE OF — the guarantee `continuing` exists for, and
// the one nothing measured until the acceptance of 11.09 planted two mutations in it and both lived.
//
// Both callers of it decide on a snapshot taken BEFORE the book lock. Between that snapshot and the
// answer the run can end — reach its ceiling, be stopped, fail — and a `202` over a halted run tells a
// client to render "continuing" over work that has stopped, which is worse than the refusal it
// replaced. The contract's own sentence («a `202` MEANS THE RUN IS CONTINUING») is true only because
// of this check.
//
// It is pinned DIRECTLY rather than through a race, because the window it closes is two statements
// wide and no fixture can stand inside it. Here the state is simply arranged and the function asked.
func TestTheAnswerToAContinuingRunIsCheckedAgainstTheRowItIsMadeOf(t *testing.T) {
	for _, tc := range []struct {
		status string
		want   error
		why    string
	}{
		{"translating", nil, "the run is going, so the answer is the run"},
		{"paused", ErrCeilingReached, "it reached its ceiling while this call was deciding, and the remedy is a NEW run"},
		{"stopped", ErrNotResumable, "it ended while this call was deciding"},
		{"failed", ErrNotResumable, "it ended while this call was deciding"},
	} {
		t.Run(tc.status, func(t *testing.T) {
			f := newFixture(t, "10", 500)
			run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
			if err != nil {
				t.Fatal(err)
			}
			// The row moves under the caller, which is exactly what the snapshot cannot see.
			if tc.status != "translating" {
				if _, err := f.store.Pool().Exec(f.ctx,
					`update runs set status = $2, finished_at = $3 where id = $1`,
					run.ID, tc.status, f.now.Add(time.Second)); err != nil {
					t.Fatal(err)
				}
			}
			got, err := f.svc.continuing(f.ctx, "u1", run.ID)
			if tc.want == nil {
				if err != nil {
					t.Fatalf("a run standing at %q answered %v, want the run: %s", tc.status, err, tc.why)
				}
				if got.Status != "translating" {
					t.Errorf("the answer carries status %q", got.Status)
				}
				return
			}
			if !errors.Is(err, tc.want) {
				t.Fatalf("a run standing at %q answered %v, want %v: %s", tc.status, err, tc.want, tc.why)
			}
			if got.ID != "" {
				t.Errorf("a refusal carried a run: %+v", got)
			}
		})
	}
	// ⚠ AND THE CAUSE IS THE TABLE'S, not a bare 409: without this the `paused` row above would be
	// satisfied by any ErrNotResumable, and the client would be sent to press the same button again.
	f := newFixture(t, "10", 500)
	run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := f.store.Pool().Exec(f.ctx,
		`update runs set status = 'paused', finished_at = $2 where id = $1`, run.ID, f.now.Add(time.Second)); err != nil {
		t.Fatal(err)
	}
	_, err = f.svc.continuing(f.ctx, "u1", run.ID)
	if !errors.Is(err, ErrCeilingReached) {
		t.Fatalf("a run that halted at its ceiling answered %v, want ErrCeilingReached", err)
	}
	if strings.Contains(err.Error(), "ended while this call was deciding") {
		t.Errorf("the ceiling answered with the generic sentence: %v", err)
	}
}

// THE CONTENDED ORDER, made inevitable instead of hoped for.
//
// Both callers are held inside their own transactions by a row lock this test owns, so NEITHER can
// commit and BOTH judge a snapshot that says `stopped` — the interleaving that used to be produced
// only by a loaded machine (the row was measured at `load average 42–48`, and on an idle one the
// concurrent fixture took this path five times out of five while never reaching the other). The wait
// is ASSERTED: a caller that did not block never met the contention, and "both answered" would then
// be true of a test that measured nothing.
//
// It runs on TWO services because one service serializes its own callers before the database: the
// book lock is in-process, which is why this test's own header — "both calls pass the state check
// together" — describes a deployment of two instances and never described one.
func TestTwoResumesOfOneRunTakeOneHoldAndBothAnswer(t *testing.T) {
	const held = 400 * time.Millisecond
	f := newFixture(t, "10", 500)
	spent := money.MicroUSD(500_000)
	runID := f.stopped(t, 100, spent, stoppedByAFailure(f))
	instances := []*Service{f.svc, f.otherInstance(t)}

	tx, err := f.store.Pool().Begin(f.ctx)
	if err != nil {
		t.Fatal(err)
	}
	defer func() { _ = tx.Rollback(f.ctx) }()
	if _, err := tx.Exec(f.ctx, `select id from runs where id = $1 for update`, runID); err != nil {
		t.Fatal(err)
	}

	type answer struct {
		run    pgstore.Run
		waited time.Duration
		err    error
	}
	done := make(chan answer, len(instances))
	for _, svc := range instances {
		go func() {
			// A context of its own: these calls must block on the database, not on the test's clock.
			start := time.Now()
			run, err := svc.Resume(context.Background(), "u1", runID)
			done <- answer{run, time.Since(start), err}
		}()
	}
	time.Sleep(held)
	// The boundary, asserted while both callers are inside: nothing has committed, so the state they
	// are judging is `stopped` for BOTH of them.
	var committed string
	if err := f.store.Pool().QueryRow(f.ctx, `select status from runs where id = $1`, runID).Scan(&committed); err != nil {
		t.Fatal(err)
	}
	if committed != "stopped" {
		t.Fatalf("the run is committed as %q while both callers are still inside: one of them judged a "+
			"fresh snapshot and this fixture is measuring the other order", committed)
	}
	if err := tx.Rollback(f.ctx); err != nil {
		t.Fatal(err)
	}

	var answers []pgstore.Run
	var errs []error
	for range instances {
		got := <-done
		if got.waited < held {
			t.Fatalf("a resume returned after %.2fs while the row was locked for %s: it never waited on the "+
				"lock, so this fixture did not reproduce the contention and proves nothing", got.waited.Seconds(), held)
		}
		answers = append(answers, got.run)
		errs = append(errs, got.err)
	}
	bothResumesAnswered(t, f, runID, answers, errs, fixtureHold(100)-spent)
}

// ⛔ THE LOSER'S OWN BRANCH, AND THE WINDOW I WRONGLY CALLED UNREACHABLE.
//
// The re-open race has two losers, and the other one is pinned next door. This is the one that loses
// on the unique index: it read the run as `stopped`, walked all the way to `reopen`, and found that
// somebody else had opened attempt N+1 first. Its answer is the run — through `continuing`, which
// checks the row it is made of, because between that caller's snapshot and its answer the run can
// have ENDED.
//
// ⚠ I REPORTED THAT WINDOW AS UNREACHABLE, AND IT IS NOT. My probe counted how often my own fixtures
// arrived at the branch with a run in some other state — never — and I wrote that down as "no fixture
// can reach it". That answers «do my tests get there», not «is it reachable»; it is the same defect
// as a pin whose background condition is frozen, one level up, in the reasoning. The acceptance of
// 11.09 built this fixture and the mutation it catches.
//
// The window is not raced for, it is ARRANGED: the book row is held from a transaction of this test,
// which stops the resume INSIDE RestartRun — after its snapshot, before its insert — and the winner's
// work plus the ending are written into that gap.
func TestTheLoserOfTheIndexRaceIsAnsweredFromTheRowAndNotFromItsSnapshot(t *testing.T) {
	f := newFixture(t, "10", 500)
	runID := f.stopped(t, 100, money.MicroUSD(500_000), stoppedByAFailure(f))
	book := f.bookID(t)

	tx, err := f.store.Pool().Begin(f.ctx)
	if err != nil {
		t.Fatal(err)
	}
	defer func() { _ = tx.Rollback(f.ctx) }()
	if _, err := tx.Exec(f.ctx, `select id from books where id = $1 for update`, book); err != nil {
		t.Fatal(err)
	}

	type answer struct {
		run pgstore.Run
		err error
	}
	done := make(chan answer, 1)
	go func() {
		run, err := f.svc.Resume(context.Background(), "u1", runID)
		done <- answer{run, err}
	}()

	// WAIT FOR THE CALLER TO BE INSIDE, asked of the database rather than slept for: a sleep long
	// enough to be safe is a sleep that hides the day it stops being long enough.
	waitForLockWaiter(t, f, book)

	// The gap. The winner's attempt N+1 — which is what will refuse this caller's insert — and then
	// the ending this caller's snapshot cannot know about.
	var attemptNo int
	if err := tx.QueryRow(f.ctx, `
		insert into run_attempts (run_id, attempt_no, started_at, last_offset, engine_binary, engine_run_id)
		select $1, max(attempt_no) + 1, $2, 0, '', $1 || '#w' from run_attempts where run_id = $1
		returning attempt_no`, runID, f.now.Add(time.Second)).Scan(&attemptNo); err != nil {
		t.Fatal(err)
	}
	if _, err := tx.Exec(f.ctx,
		`update runs set status = 'paused', paused_reason = 'credit_exhausted', finished_at = $2 where id = $1`,
		runID, f.now.Add(2*time.Second)); err != nil {
		t.Fatal(err)
	}
	if err := tx.Commit(f.ctx); err != nil {
		t.Fatal(err)
	}

	got := <-done
	t.Logf("the loser lost attempt %d and answered: status=%q err=%v", attemptNo, got.run.Status, got.err)
	if got.err == nil {
		t.Fatalf("the loser was handed a %q run with no error: a 202 over a run that has HALTED tells a "+
			"client to render «continuing» over work that stopped, which is the whole of what continuing "+
			"exists to prevent", got.run.Status)
	}
	if !errors.Is(got.err, ErrCeilingReached) {
		t.Fatalf("the loser answered %v, want ceiling_reached: the run stands at `paused`, and the canon "+
			"gives that status its own cause", got.err)
	}
	if got.run.ID != "" {
		t.Errorf("a refusal carried a run: %+v", got.run)
	}
}

// waitForLockWaiter blocks until somebody is waiting on a lock in this database — the observable form
// of "the caller is inside and has not committed". Asked of `pg_stat_activity` rather than slept for.
func waitForLockWaiter(t *testing.T, f *fixture, book string) {
	t.Helper()
	deadline := time.Now().Add(20 * time.Second)
	for {
		var waiting int
		if err := f.store.Pool().QueryRow(f.ctx, `
			select count(*) from pg_stat_activity
			 where datname = current_database() and wait_event_type = 'Lock' and state = 'active'`).Scan(&waiting); err != nil {
			t.Fatal(err)
		}
		if waiting > 0 {
			return
		}
		if time.Now().After(deadline) {
			t.Fatalf("nobody ever waited on a lock in this database: the resume never reached RestartRun, "+
				"so the gap this fixture writes into was never open and nothing was measured (book %s)", book)
		}
		time.Sleep(20 * time.Millisecond)
	}
}

// THE OTHER CALLER OF `continuing` — the one whose snapshot said the run was going — and the same
// question asked of it: is the answer made of the row, or of the snapshot?
//
// ⚠ The acceptance could not build this one and said so; it is reachable through the BOOK LOCK THIS
// SERVICE HOLDS IN PROCESS (bank.go), which sits between the snapshot and the branch. The test takes
// that lock, lets a resume pile up behind it, ends the run in the gap, and releases.
//
// The stale path is ASSERTED and not assumed: a caller whose snapshot had already seen `failed` takes
// the switch's `default` and says «it is failed», while this one says «it ended while this call was
// deciding». Two different sentences, and only the second one means the fixture measured what it is
// about — so a gap too short to matter fails loudly instead of passing for the wrong reason.
func TestTheGoingRunsAnswerIsMadeOfTheRowAndNotOfTheSnapshot(t *testing.T) {
	f := newFixture(t, "10", 500)
	run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
	if err != nil {
		t.Fatal(err)
	}
	if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
		t.Fatal(err)
	}
	unlock, err := f.svc.lockBook(f.ctx, f.bookID(t))
	if err != nil {
		t.Fatal(err)
	}
	type answer struct {
		run pgstore.Run
		err error
	}
	done := make(chan answer, 1)
	go func() {
		got, err := f.svc.Resume(context.Background(), "u1", run.ID)
		done <- answer{got, err}
	}()
	// Long enough for the caller to take its snapshot and pile up on the lock this test holds. If it
	// is not, the assertion below says so rather than passing.
	time.Sleep(300 * time.Millisecond)
	if _, err := f.store.Pool().Exec(f.ctx,
		`update runs set status = 'failed', finished_at = $2 where id = $1`,
		run.ID, f.now.Add(time.Second)); err != nil {
		t.Fatal(err)
	}
	unlock()

	got := <-done
	t.Logf("the going run's caller answered: status=%q err=%v", got.run.Status, got.err)
	if got.err == nil {
		t.Fatalf("a caller whose snapshot said «translating» was handed a %q run with no error: the "+
			"contract's own sentence is that a 202 means the run is continuing", got.run.Status)
	}
	if !strings.Contains(got.err.Error(), "ended while this call was deciding") {
		t.Fatalf("the answer is %q, and this fixture is about the caller that decided on a STALE snapshot: "+
			"a caller that had already seen `failed` says «it is failed» instead, so the gap was not open "+
			"and nothing was measured", got.err)
	}
	if got.run.ID != "" {
		t.Errorf("a refusal carried a run: %+v", got.run)
	}
}