279 lines
10 KiB
Go
279 lines
10 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"bytes"
|
||
"fmt"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"syscall"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/chunk/chunktest"
|
||
"textmachine/backend/internal/obs"
|
||
"textmachine/backend/internal/runevents"
|
||
)
|
||
|
||
// runevents_crash_test.go — the acceptance core of the outbox (row 103): a real SIGKILL in the middle of
|
||
// a wave must leave a journal a reader can still consume, and a journal that says nothing the engine's
|
||
// SQLite does not hold. It is the same helper-process shape as store/kill9_test.go, and for the same
|
||
// reason: an emulated crash proves the emulation.
|
||
|
||
const (
|
||
crashBookEnv = "TM_EVENTS_CRASH_BOOK"
|
||
crashModeEnv = "TM_EVENTS_CRASH_MODE"
|
||
)
|
||
|
||
// TestHelperEventsRun is the child process. It runs the book against the PARENT's provider (an
|
||
// httptest server on 127.0.0.1, which a subprocess can dial) and, in `exit` mode, leaves through
|
||
// os.Exit — which skips every defer, including Runner.Close.
|
||
func TestHelperEventsRun(t *testing.T) {
|
||
bookPath := os.Getenv(crashBookEnv)
|
||
if bookPath == "" {
|
||
t.Skip("helper process only")
|
||
}
|
||
r, err := NewRunner(bookPath, obs.NewLogger())
|
||
if err != nil {
|
||
fmt.Fprintln(os.Stderr, "helper runner:", err)
|
||
os.Exit(1)
|
||
}
|
||
if _, err := r.TranslateBook(tracedCtx()); err != nil {
|
||
fmt.Fprintln(os.Stderr, "helper translate:", err)
|
||
os.Exit(1)
|
||
}
|
||
if os.Getenv(crashModeEnv) == "exit" {
|
||
// PD-61(а): os.Exit skips defers, so anything the emitter had been holding would be lost here —
|
||
// which is why it holds nothing. Runner.Close is deliberately NOT called.
|
||
os.Exit(0)
|
||
}
|
||
r.Close()
|
||
}
|
||
|
||
// crashFixture writes a book big enough that a kill lands mid-wave, with a provider slow enough to make
|
||
// the window wide. Returns the book path.
|
||
func crashFixture(t *testing.T, chapters int) string {
|
||
t.Helper()
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
time.Sleep(4 * time.Millisecond) // widen the window the SIGKILL has to land in
|
||
return draftEdit(body)
|
||
})
|
||
t.Cleanup(srv.Close)
|
||
|
||
var eps []chunktest.Chapter
|
||
var spine []string
|
||
for i := 1; i <= chapters; i++ {
|
||
id := fmt.Sprintf("c%d", i)
|
||
eps = append(eps, chunktest.Chapter{ID: id, Href: id + ".xhtml", Body: fmt.Sprintf("<p>静かな図書館の朝%d。</p>", i)})
|
||
spine = append(spine, id)
|
||
}
|
||
return setupProjectOpts(t, srv.URL, projectOpts{epub: eps, spine: spine, waveWorkers: 4})
|
||
}
|
||
|
||
// waitForJournalLines blocks until the book's journal holds at least n COMPLETE lines.
|
||
func waitForJournalLines(t *testing.T, bookPath string, n int) {
|
||
t.Helper()
|
||
path := filepath.Join(filepath.Dir(bookPath), runevents.JournalFile)
|
||
deadline := time.Now().Add(30 * time.Second)
|
||
for {
|
||
if body, err := os.ReadFile(path); err == nil && bytes.Count(body, []byte("\n")) >= n {
|
||
return
|
||
}
|
||
if time.Now().After(deadline) {
|
||
t.Fatalf("the child wrote fewer than %d journal lines in 30s", n)
|
||
}
|
||
time.Sleep(2 * time.Millisecond)
|
||
}
|
||
}
|
||
|
||
func runHelper(t *testing.T, bookPath, mode string) *exec.Cmd {
|
||
t.Helper()
|
||
exe, err := os.Executable()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cmd := exec.Command(exe, "-test.run", "^TestHelperEventsRun$")
|
||
cmd.Env = append(os.Environ(), crashBookEnv+"="+bookPath, crashModeEnv+"="+mode)
|
||
cmd.Stderr = os.Stderr
|
||
return cmd
|
||
}
|
||
|
||
func TestASigkillMidWaveLeavesAReadableJournalThatMatchesTheDatabase(t *testing.T) {
|
||
if testing.Short() {
|
||
t.Skip("subprocess test")
|
||
}
|
||
bookPath := crashFixture(t, 40)
|
||
|
||
cmd := runHelper(t, bookPath, "kill")
|
||
if err := cmd.Start(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Kill on the journal's OWN progress rather than on a stopwatch: a sleep long enough to be safe on a
|
||
// loaded machine is long enough to let the child finish on an idle one, and then the test proves
|
||
// nothing. Waiting for lines means the signal always lands mid-wave.
|
||
waitForJournalLines(t, bookPath, 8)
|
||
if err := cmd.Process.Signal(syscall.SIGKILL); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
_ = cmd.Wait()
|
||
|
||
envs, raw := readJournal(t, bookPath) // also asserts the file does not end mid-line
|
||
assertOneStream(t, envs)
|
||
units := 0
|
||
for _, ev := range envs {
|
||
if ev.Type == runevents.TypeUnitDone {
|
||
units++
|
||
}
|
||
}
|
||
if units == 0 {
|
||
t.Fatal("the killed process made no announced progress — the window missed the wave entirely")
|
||
}
|
||
// The kill has to land MID-wave for this test to be about anything: a stream that reached its own
|
||
// terminal line finished before the signal, and then nothing here was exercised.
|
||
if envs[len(envs)-1].Type == runevents.TypeFinished {
|
||
t.Fatal("the child finished before the SIGKILL — widen the book or the provider delay")
|
||
}
|
||
|
||
// Everything the file claims, the database holds — in the outbox byte for byte, and as the
|
||
// dispositions the announcements describe. This is the direction that matters: a journal running
|
||
// AHEAD of the ledger is what would show a user work that was never done.
|
||
r := newRunner(t, bookPath)
|
||
runID := payload[runevents.Hello](t, envs[0]).EngineRunID
|
||
stored, err := r.Store.PendingEvents(runID, 0, 1_000_000)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(stored) < len(raw) {
|
||
t.Fatalf("the file carries %d lines the outbox does not have (outbox %d)", len(raw)-len(stored), len(stored))
|
||
}
|
||
for i := range raw {
|
||
if string(stored[i].Line) != string(raw[i]) {
|
||
t.Fatalf("line %d is not the stored row:\n file %s\noutbox %s", i+1, raw[i], stored[i].Line)
|
||
}
|
||
}
|
||
rowsAt := map[chunkKey]map[string]bool{}
|
||
for _, cs := range storedRows(t, r) {
|
||
k := chunkKey{cs.Chapter, cs.ChunkIdx}
|
||
if rowsAt[k] == nil {
|
||
rowsAt[k] = map[string]bool{}
|
||
}
|
||
rowsAt[k][cs.Stage] = true
|
||
}
|
||
for _, ev := range envs {
|
||
if ev.Type != runevents.TypeUnitDone {
|
||
continue
|
||
}
|
||
u := payload[runevents.UnitDone](t, ev)
|
||
stage := "draft"
|
||
if u.Wave == runevents.WaveEdit {
|
||
stage = "edit"
|
||
}
|
||
if !rowsAt[chunkKey{u.Chapter, u.Unit}][stage] {
|
||
t.Fatalf("the journal announced ch%d unit%d (%s) but the database has no such row", u.Chapter, u.Unit, u.Wave)
|
||
}
|
||
}
|
||
|
||
// …and the run catches up honestly: a second process opens its OWN stream in the same file, finishes
|
||
// the book, and announces only the units IT resolved.
|
||
if _, err := r.TranslateBook(tracedCtx()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r.Close()
|
||
|
||
envs2, _ := readJournal(t, bookPath)
|
||
rs := regions(envs2)
|
||
if len(rs) != 2 {
|
||
t.Fatalf("want the killed stream plus the resumed one, got %d regions", len(rs))
|
||
}
|
||
for _, region := range rs {
|
||
assertOneStream(t, region)
|
||
assertProgressSane(t, region)
|
||
}
|
||
resumedUnits := 0
|
||
for _, ev := range rs[1] {
|
||
if ev.Type == runevents.TypeUnitDone {
|
||
resumedUnits++
|
||
}
|
||
}
|
||
if resumedUnits == 0 {
|
||
t.Fatal("the resumed process announced nothing — the crash left no work to catch up on")
|
||
}
|
||
// COVERAGE is the guarantee, and it is the one the crash used to break: every unit×wave of the book is
|
||
// announced at least once across the processes it took. Losing one is not a cosmetic miss — a reader
|
||
// folds these into a chapter's counters, and a chapter one short of its own total never reads finished,
|
||
// which is a book that can never complete and a chapter the platform keeps offering to translate.
|
||
//
|
||
// DUPLICATION across processes is legal and deliberate. Between a line landing on the file and the
|
||
// ledger row that records the delivery there is no transaction — nothing can put a file write and a
|
||
// database commit in one — so a process killed in that window makes the next one announce that unit
|
||
// again. Delivery is ratified as at-least-once (D39.119 п.3) exactly because this boundary exists; the
|
||
// engine therefore errs to the side that loses nothing, and a duplicate is the consumer's to absorb.
|
||
// Within ONE process there is no such window and no such duplicate, which is asserted separately.
|
||
counted := map[string]int{}
|
||
for _, ev := range envs2 {
|
||
if ev.Type != runevents.TypeUnitDone {
|
||
continue
|
||
}
|
||
u := payload[runevents.UnitDone](t, ev)
|
||
counted[fmt.Sprintf("%s/%d/%d", u.Wave, u.Chapter, u.Unit)]++
|
||
}
|
||
for _, region := range rs {
|
||
perRegion := map[string]int{}
|
||
for _, ev := range region {
|
||
if ev.Type != runevents.TypeUnitDone {
|
||
continue
|
||
}
|
||
u := payload[runevents.UnitDone](t, ev)
|
||
key := fmt.Sprintf("%s/%d/%d", u.Wave, u.Chapter, u.Unit)
|
||
if perRegion[key]++; perRegion[key] > 1 {
|
||
t.Fatalf("one process announced unit %s twice — within a stream this is a plain bug", key)
|
||
}
|
||
}
|
||
}
|
||
last := envs2[len(envs2)-1]
|
||
if last.Type != runevents.TypeFinished {
|
||
t.Fatalf("the resumed run must terminate its stream, got %s", last.Type)
|
||
}
|
||
var final runevents.Progress
|
||
for _, ev := range envs2 {
|
||
if ev.Type == runevents.TypeProgress {
|
||
final = payload[runevents.Progress](t, ev)
|
||
}
|
||
}
|
||
if final.Draft.Done != final.Draft.Total || final.Edit.Done != final.Edit.Total || final.Draft.Total == 0 {
|
||
t.Fatalf("the caught-up counters are %+v — a finished book must reach its own denominator", final)
|
||
}
|
||
if len(counted) != final.Draft.Total+final.Edit.Total {
|
||
t.Fatalf("%d unit-waves announced, the book has %d+%d: the crash lost an announcement for good",
|
||
len(counted), final.Draft.Total, final.Edit.Total)
|
||
}
|
||
}
|
||
|
||
func TestTheTerminalEventSurvivesAnExitThatSkipsEveryDefer(t *testing.T) {
|
||
// PD-61(а), the property this pack had to build rather than promise: `os.Exit` and `log.Fatal` skip
|
||
// defers, so a buffered writer around the journal would lose exactly the lines that say the run
|
||
// ended. Nothing is buffered — a line and its newline go to the kernel in one write — so there is
|
||
// nothing left to flush. Wrapping the journal in a bufio.Writer fails this test.
|
||
if testing.Short() {
|
||
t.Skip("subprocess test")
|
||
}
|
||
bookPath := crashFixture(t, 3)
|
||
|
||
cmd := runHelper(t, bookPath, "exit")
|
||
if err := cmd.Run(); err != nil {
|
||
t.Fatalf("helper: %v", err)
|
||
}
|
||
envs, _ := readJournal(t, bookPath)
|
||
assertOneStream(t, envs)
|
||
last := envs[len(envs)-1]
|
||
if last.Type != runevents.TypeFinished {
|
||
t.Fatalf("the last line after an os.Exit is %s — the terminal event was lost", last.Type)
|
||
}
|
||
if got := payload[runevents.Finished](t, last).Outcome; got != runevents.OutcomeClean {
|
||
t.Fatalf("outcome %q, want %q", got, runevents.OutcomeClean)
|
||
}
|
||
if _, err := os.Stat(filepath.Join(filepath.Dir(bookPath), runevents.JournalFile)); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|