115 lines
3.5 KiB
Go
115 lines
3.5 KiB
Go
package store
|
||
|
||
import (
|
||
"fmt"
|
||
"math"
|
||
"math/rand"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"syscall"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// kill9_test.go — Р6/Phase 0 acceptance test: kill -9 mid-work loses
|
||
// at most one in-flight call and NEVER desyncs the money from the
|
||
// checkpoints. A real SIGKILL against a subprocess (the standard library's
|
||
// helper-process pattern), not emulation.
|
||
//
|
||
// Invariant after any kill: committed_usd == SUM(checkpoints.cost_usd)
|
||
// — because settle and the checkpoint are written in ONE transaction. Orphaned
|
||
// reserved_usd are cleared by the recovery pass on open.
|
||
|
||
const killLoopEnv = "TM_STORE_KILL_LOOP_DB"
|
||
|
||
func TestHelperKillLoop(t *testing.T) {
|
||
path := os.Getenv(killLoopEnv)
|
||
if path == "" {
|
||
t.Skip("helper process only")
|
||
}
|
||
s, err := Open(path)
|
||
if err != nil {
|
||
fmt.Fprintln(os.Stderr, "helper open:", err)
|
||
os.Exit(1)
|
||
}
|
||
if err := s.UpsertSnapshot("snap", "brief", `{}`); err != nil {
|
||
os.Exit(1)
|
||
}
|
||
job, err := s.EnsureJob("book", 1, "draft", "snap")
|
||
if err != nil {
|
||
os.Exit(1)
|
||
}
|
||
// Infinite reserve → settle+checkpoint loop; the parent will SIGKILL us
|
||
// at a random moment. No Release — recovery will clear the "crashed" reservations.
|
||
for i := 0; ; i++ {
|
||
res, verdict, err := s.Reserve("book", 0.002, Ceilings{BookUSD: 10_000})
|
||
if err != nil || verdict != ReserveOK {
|
||
os.Exit(1)
|
||
}
|
||
cp := Checkpoint{
|
||
RequestHash: fmt.Sprintf("hash-%d-%d", os.Getpid(), i),
|
||
JobID: job.ID, ChunkIdx: i, Stage: "draft", Role: "translator",
|
||
ModelRequested: "m", ModelActual: "m",
|
||
ResponseText: "ответ", UsageJSON: "{}", CostUSD: 0.001,
|
||
}
|
||
if err := s.SettleWithCheckpoint(res, 0.001, cp, nil); err != nil {
|
||
os.Exit(1)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestKillMinus9LosesAtMostOneCall(t *testing.T) {
|
||
if testing.Short() {
|
||
t.Skip("subprocess test")
|
||
}
|
||
exe, err := os.Executable()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
path := filepath.Join(t.TempDir(), "kill9.db")
|
||
|
||
for round := 0; round < 5; round++ { // D23.4: 3→5 rounds — more SIGKILL windows per run
|
||
cmd := exec.Command(exe, "-test.run", "^TestHelperKillLoop$", "-test.v")
|
||
cmd.Env = append(os.Environ(), killLoopEnv+"="+path)
|
||
if err := cmd.Start(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Let it run for a random amount of time, then SIGKILL — nothing graceful.
|
||
time.Sleep(time.Duration(150+rand.Intn(400)) * time.Millisecond)
|
||
if err := cmd.Process.Signal(syscall.SIGKILL); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
_ = cmd.Wait()
|
||
|
||
s, err := Open(path) // the recovery pass clears the stuck reservations
|
||
if err != nil {
|
||
t.Fatalf("round %d: reopen after kill: %v", round, err)
|
||
}
|
||
|
||
var checkpointSum float64
|
||
var checkpointCount int
|
||
if err := s.r.QueryRow(`SELECT COALESCE(SUM(cost_usd),0), COUNT(*) FROM checkpoints`).Scan(&checkpointSum, &checkpointCount); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
committed, reserved, err := s.SpentUSD("book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
s.Close()
|
||
|
||
if reserved != 0 {
|
||
t.Fatalf("round %d: recovery must clear reservations, reserved=%v", round, reserved)
|
||
}
|
||
// Acceptance core: money and checkpoints are atomic — a mismatch would mean
|
||
// a paid-but-lost (or free-but-recorded) call.
|
||
if math.Abs(committed-checkpointSum) > 1e-9 {
|
||
t.Fatalf("round %d: committed=%v != sum(checkpoints)=%v over %d checkpoints — settle+checkpoint not atomic",
|
||
round, committed, checkpointSum, checkpointCount)
|
||
}
|
||
if checkpointCount == 0 && round > 0 {
|
||
t.Fatalf("round %d: helper made no progress", round)
|
||
}
|
||
t.Logf("round %d: %d checkpoints, committed=$%.6f — consistent after SIGKILL", round, checkpointCount, committed)
|
||
}
|
||
}
|