108 lines
5.2 KiB
Go
108 lines
5.2 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"testing"
|
|
|
|
"textmachine/backend/internal/obs"
|
|
"textmachine/backend/internal/runevents"
|
|
"textmachine/backend/internal/store"
|
|
)
|
|
|
|
// volumepanic_test.go: fix-list ФЧ-7 — a worker CRASH landing on top of a ceiling a sibling already
|
|
// caught. PD-113 was closed with two channels that fail independently (the stream's `ceiling` event and
|
|
// exit code 4), and one panic used to take both: the wave returned the crash alone, `errors.As(err,
|
|
// &CeilingHalt)` went false, and the run departed as a bare failure with no ceiling event at all — so the
|
|
// platform, whose reconcile lets a ceiling event survive ANY exit, recorded a ceiling stop as `failed`.
|
|
|
|
// TestACrashOverACaughtCeilingKeepsBothFacts is the seam itself: the wave must carry the crash AND the
|
|
// ceiling, with the crash still RANKING (the consumers read it first, and a crashed process is not
|
|
// paused). Only the money fact is rescued, not the verdict.
|
|
func TestACrashOverACaughtCeilingKeepsBothFacts(t *testing.T) {
|
|
r := &Runner{}
|
|
halt := &CeilingHalt{Scope: runevents.ScopeBook, err: fmt.Errorf("pipeline: book USD ceiling reached: %w", errReserveCeiling)}
|
|
// ⚠ THE TRIGGER MOVED, AND THE REASON IS A BEHAVIOUR CHANGE THIS PACK ORDERED (backlog row 277). This
|
|
// test used to park the crashing worker on `<-ctx.Done()` and let the CEILING fire it: a refused
|
|
// reservation cancelled the wave, and the crash happened under that cancellation. A spend ceiling no
|
|
// longer cancels anything — «do not start anything new» instead of «kill what is running» is the whole
|
|
// of the pack — so parked there the worker would wait forever and the test would hang, which it did
|
|
// when the change first landed.
|
|
//
|
|
// What the test asserts is unchanged and is not weakened: the crash and the ceiling are recorded in
|
|
// SEPARATE slots and both leave the wave. What is gone is the ordering ritual, and it is gone because
|
|
// it no longer decides anything — with two slots there is no first-wins race between these two facts
|
|
// to lose. The ordering that still matters (a crash against an infra error, which DOES cancel) is
|
|
// pinned by wavepanic_test.go, where ctx.Done() still fires.
|
|
// The crashing worker must be HOLDING ITS ITEM before the halt is returned — under the latch the
|
|
// feeder stops handing out indices, so a sibling that returns first can leave item 1 undispatched and
|
|
// no panic happens at all (it did, on the first attempt at this rewrite).
|
|
started := make(chan struct{})
|
|
sibling := make(chan struct{})
|
|
err := r.runWave(context.Background(), 2, 2, func(ctx context.Context, i int) error {
|
|
if i == 1 {
|
|
close(started)
|
|
<-sibling
|
|
panic("a worker died while its sibling's ceiling stop was on the record")
|
|
}
|
|
<-started
|
|
defer close(sibling)
|
|
return halt
|
|
})
|
|
var panicked *obs.PanicError
|
|
if !errors.As(err, &panicked) {
|
|
t.Fatalf("the crash must still rank — a process that died is not a pause: %v", err)
|
|
}
|
|
var got *CeilingHalt
|
|
if !errors.As(err, &got) {
|
|
t.Fatalf("the ceiling was caught before the crash and must survive it: %v", err)
|
|
}
|
|
if got.Scope != runevents.ScopeBook {
|
|
t.Fatalf("the surviving halt lost its scope: %+v", got)
|
|
}
|
|
if !errors.Is(err, errReserveCeiling) {
|
|
t.Fatal("every degrade path keys on the sentinel; it must still match")
|
|
}
|
|
}
|
|
|
|
// TestACrashOverACaughtCeilingStillAnnouncesTheCeiling is the STREAM half. The outcome mirrors the exit
|
|
// code — exit 1, `failed`, because the process really did crash — while the `ceiling` event still goes,
|
|
// because it states that the money ran out and that did not stop being true. The platform reads the event
|
|
// before any exit code and lands on `paused`.
|
|
func TestACrashOverACaughtCeilingStillAnnouncesTheCeiling(t *testing.T) {
|
|
dir := t.TempDir()
|
|
st, err := store.Open(dir + "/events.db")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
|
e, err := openEmitter(st, dir, obs.NewTraceID(), "book", log)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = e.close() }()
|
|
|
|
halt := &CeilingHalt{Scope: runevents.ScopeDay, err: fmt.Errorf("pipeline: daily USD ceiling reached: %w", errReserveCeiling)}
|
|
e.terminal(nil, errors.Join(obs.NewPanicError("wave worker", "boom"), halt))
|
|
|
|
envs, _ := readJournal(t, dir+"/book.yaml")
|
|
if len(envs) < 3 {
|
|
t.Fatalf("want hello, ceiling and the terminal line, got %d events", len(envs))
|
|
}
|
|
ceiling, finished := envs[len(envs)-2], envs[len(envs)-1]
|
|
if ceiling.Type != runevents.TypeCeiling {
|
|
t.Fatalf("the money fact was dropped because a worker crashed after it: got %s before the terminal line", ceiling.Type)
|
|
}
|
|
if c := payload[runevents.Ceiling](t, ceiling); !c.Halted || c.Scope != runevents.ScopeDay {
|
|
t.Fatalf("ceiling event %+v, want halted with the day scope", c)
|
|
}
|
|
// ⚠ AND THE OUTCOME IS STILL `failed`. The outcome vocabulary mirrors the shell contract, and a run
|
|
// that crashed exits 1; calling it `ceiling` here would make the two channels disagree about one run.
|
|
if got := payload[runevents.Finished](t, finished).Outcome; got != runevents.OutcomeFailed {
|
|
t.Fatalf("terminal outcome %q, want %q — the process crashed, and the outcome mirrors its exit code", got, runevents.OutcomeFailed)
|
|
}
|
|
}
|