textmachine/backend/internal/pipeline/wavepanic_test.go

158 lines
6.6 KiB
Go

package pipeline
import (
"context"
"errors"
"fmt"
"strings"
"sync/atomic"
"testing"
"textmachine/backend/internal/obs"
)
// wavepanic_test.go: a panic of a WAVE WORKER must leave the engine as a failing error, never as the
// process-level exit 2 the Go runtime gives an unrecovered panic — which is the code the shell contract
// reserves for "completed with flags", so a run that died mid-book was recorded as `ready` (row 176).
//
// The wave seam is where this has to be tested: the paid path runs inside runWave's goroutine pool
// (waverun.go), and a recover on the main goroutine alone would leave every real crash uncovered.
func TestWaveWorkerPanicSurfacesAsAFailingError(t *testing.T) {
const workers = 4
var cancelled atomic.Int32
inFlight := make(chan struct{}, workers-1)
r := &Runner{}
// The siblings do what real stage work does — wait on the wave context — so this also pins the
// masking trap: the panic cancels the wave, every other worker then fails with context.Canceled, and
// if THAT became the wave's error the run would leave as exit 5, a graceful stop. The panicking
// worker waits until the others are in flight, so the outcome does not depend on the scheduler.
err := r.runWave(context.Background(), workers, workers, func(ctx context.Context, i int) error {
if i == 0 {
for k := 0; k < workers-1; k++ {
<-inFlight
}
panic("worker exploded mid-chunk")
}
inFlight <- struct{}{}
<-ctx.Done()
cancelled.Add(1)
return ctx.Err()
})
if err == nil {
t.Fatal("a panicking worker must fail the wave; a nil error here is the run reported as success")
}
var panicked *obs.PanicError
if !errors.As(err, &panicked) {
t.Fatalf("the wave error must be a *obs.PanicError, got %T: %v", err, err)
}
if panicked.Where != "wave worker" {
t.Errorf("the panic must name the goroutine it died in, got %q", panicked.Where)
}
// The stack has to survive to whoever prints the error: on the paid path the only guaranteed reader
// is `tmctl: <err>` on stderr, and diagnosis of a crash mid-book is worth the verbosity.
msg := err.Error()
for _, want := range []string{"worker exploded mid-chunk", "goroutine", "runWave"} {
if !strings.Contains(msg, want) {
t.Errorf("the panic error must carry %q so the stack reaches stderr; got: %s", want, msg)
}
}
if errors.Is(err, context.Canceled) {
t.Error("the panic must not be masked by the cancellation it caused (that reads as exit 5, a graceful stop)")
}
// The crash stops the wave rather than letting it grind on: every sibling it cancelled did return.
if got := cancelled.Load(); got != workers-1 {
t.Errorf("a panicking worker must cancel the wave: %d of %d siblings saw the cancellation", got, workers-1)
}
}
// TestWaveWorkerPanicOutranksAnEarlierWaveError is the race an adversarial review of this pack found:
// the first error to arrive cancels the wave, and a panic in code nobody expected to run under
// cancellation arrives SECOND. Routed through first-wins it was discarded with its stack, and the run
// departed as the sibling's error — a ceiling halt (exit 4, which the platform records as `paused`) or a
// cancellation (exit 5). A process that crashed is neither.
func TestWaveWorkerPanicOutranksAnEarlierWaveError(t *testing.T) {
for _, tc := range []struct {
name string
first error
}{
{"ceiling halt (exit 4 = paused)", &CeilingHalt{Scope: "book", err: errors.New("book USD ceiling reached")}},
{"cancellation (exit 5 = graceful stop)", context.Canceled},
{"an ordinary infra error", errors.New("provider unreachable")},
} {
t.Run(tc.name, func(t *testing.T) {
const workers = 2
inFlight := make(chan struct{})
r := &Runner{}
// Item 0 is dispatched first, so the panicking worker is parked and holding its item before
// the sibling fails — otherwise the wave's cancellation would drop item 1 and no panic would
// happen at all. It then waits for ctx.Done(), which runWave fires only AFTER recording the
// sibling's error, so the panic is strictly second without a sleep.
err := r.runWave(context.Background(), workers, workers, func(ctx context.Context, i int) error {
if i == 0 {
close(inFlight)
<-ctx.Done()
panic("worker exploded under cancellation")
}
<-inFlight
return tc.first
})
var panicked *obs.PanicError
if !errors.As(err, &panicked) {
t.Fatalf("the crash must outrank the earlier wave error; wave returned %T: %v", err, err)
}
if !strings.Contains(err.Error(), "goroutine") {
t.Errorf("the stack must survive the race; got: %s", err.Error())
}
})
}
}
// TestPanicErrorMatchesNoExitSentinel is the trap the fix is one line away from: an error born of
// recover that matched a dictionary sentinel would leave through a NON-failing exit code with the
// dictionary formally untouched — 2 flags, 3 signature stop, 4 ceiling (the platform reads `paused`),
// 5 graceful stop, 10-19 refusal ("nothing was spent"). The panicked VALUE is deliberately one of those
// types in each case, because that is the shape that would slip through an Unwrap.
func TestPanicErrorMatchesNoExitSentinel(t *testing.T) {
for _, tc := range []struct {
name string
value any
}{
{"plain string", "boom"},
{"runtime error", fmt.Errorf("nil map write")},
{"context.Canceled", context.Canceled},
{"CompletedWithFlags", &CompletedWithFlags{Flagged: 1, Total: 2}},
{"WaveSignatureStop", &WaveSignatureStop{Terms: 3}},
{"CeilingHalt", &CeilingHalt{Scope: "book", err: errors.New("ceiling")}},
{"Refusal", refuse(RefusalBadConfig, errors.New("bad config"))},
} {
t.Run(tc.name, func(t *testing.T) {
// Wrapped, because that is how it reaches main: through the wave, the driver and translate().
err := fmt.Errorf("pipeline: wave draft: %w", obs.NewPanicError("wave worker", tc.value))
var flagged *CompletedWithFlags
var sigStop *WaveSignatureStop
var ceiling *CeilingHalt
var refusal *Refusal
if errors.As(err, &flagged) {
t.Error("a panic must not read as completed-with-flags (exit 2)")
}
if errors.As(err, &sigStop) {
t.Error("a panic must not read as a signature stop (exit 3)")
}
if errors.As(err, &ceiling) {
t.Error("a panic must not read as a ceiling halt (exit 4 — the platform records `paused`)")
}
if errors.As(err, &refusal) {
t.Error("a panic must not read as a refusal (10-19 — the band promises nothing was spent)")
}
if errors.Is(err, context.Canceled) {
t.Error("a panic must not read as a graceful stop (exit 5)")
}
var panicked *obs.PanicError
if !errors.As(err, &panicked) {
t.Error("the panic must stay recognisable through the wrap")
}
})
}
}