156 lines
6.7 KiB
Go
156 lines
6.7 KiB
Go
package runs
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"textmachine/platform/internal/ingest"
|
|
"textmachine/platform/internal/runner"
|
|
)
|
|
|
|
// ⛔ «THE COLUMN IS FILLED» AND «A PERSON SEES IT» ARE DIFFERENT CLAIMS, and this test makes the
|
|
// second one.
|
|
//
|
|
// A run this platform killed by its own grace is OUR POLICY, not a fault of the deployment, and the
|
|
// pack that asked for it to be «told apart» is answered on three surfaces at once — which is why they
|
|
// are asserted together:
|
|
//
|
|
// - the WIRE keeps its retry axis (`interrupted`: retrying IS the remedy, and it is, because nobody
|
|
// is stopping the next spawn) — a fourth value there would answer the same question in a fourth
|
|
// phrase, which the canon's own constraint comment forbids;
|
|
// - the ATTEMPT keeps the machine's word `timeout`, which is what tells our deadline from a kill
|
|
// nobody chose (`oom-kill` and friends);
|
|
// - and a PERSON is told at the moment it happens, because a column no surface reads is not a
|
|
// signal (this zone's own rule, written at the parked attempt's line).
|
|
func TestAKillByOurOwnGraceIsRecordedWhereAPersonCanSeeIt(t *testing.T) {
|
|
f := newFixture(t, "10", 500)
|
|
var buf bytes.Buffer
|
|
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
live := f.live(t)
|
|
// What systemd writes when TimeoutStopSec — the grace THIS platform sets — expires: measured on a
|
|
// unit of this shape, and told from a stop the engine caught (`exit-code/exited/5`) by what the
|
|
// machine saw rather than by inference.
|
|
if err := runner.WriteMarker(f.svc.markerPath(live.RunID, live.AttemptNo),
|
|
runner.Marker{Unit: live.UnitName, Result: "timeout", Code: "killed", Status: "KILL",
|
|
At: f.now.Add(time.Second)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.StatusReport{TotalUnits: 10, Done: 2, Spend: usd(0), Reserved: usd(0)}, nil)
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := f.store.ReadRun(f.ctx, "u1", run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Status != "failed" || got.FailureReason != "interrupted" {
|
|
t.Errorf("a run we killed by our own deadline reads %s/%s, want failed/interrupted: "+
|
|
"`service_error` would tell the user their deployment is broken, and a fourth word would "+
|
|
"answer the retry question a fourth way", got.Status, got.FailureReason)
|
|
}
|
|
var result string
|
|
if err := f.store.Pool().QueryRow(f.ctx,
|
|
`select coalesce(exit_result, '') from run_attempts where run_id = $1`, run.ID).Scan(&result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result != "timeout" {
|
|
t.Errorf("the attempt's exit_result is %q, want the machine's own %q: without it our policy is "+
|
|
"byte-identical to a kill nobody chose", result, "timeout")
|
|
}
|
|
// The human-facing half: the ending is ANNOUNCED with that word, so an operator does not have to
|
|
// query the database to tell a policy from a fault.
|
|
var said bool
|
|
for _, l := range strings.Split(buf.String(), "\n") {
|
|
if !strings.Contains(l, `"run finished"`) {
|
|
continue
|
|
}
|
|
var line map[string]any
|
|
if err := json.Unmarshal([]byte(l), &line); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if line["result"] == "timeout" {
|
|
said = true
|
|
}
|
|
}
|
|
if !said {
|
|
t.Errorf("no line told anybody HOW this run ended; the fact would live in a column only: %s", buf.String())
|
|
}
|
|
}
|
|
|
|
// ⛔ THE FIXTURE THE PACK ASKED FOR: the «stranger» in the journal IS THIS VERY RUN.
|
|
//
|
|
// The platform names a stream before the unit starts and keys the name on (run, attempt), and the
|
|
// engine mints a FRESH one when the given id has already written for this book — so a respawn of this
|
|
// same run appears in its own journal under an id the cursor will never match
|
|
// (reconcile.drainJournal, `may be this very run, alive and writing`). Everything the buyer is told
|
|
// has to be true in THAT case, which is the one where a wording built on «somebody else touched your
|
|
// book» lies.
|
|
//
|
|
// What is asserted is therefore not only the word but the ACTION: the run stays live and translating,
|
|
// nothing is quarantined, and the only thing said is that the figures are behind — which is true.
|
|
func TestWhenTheStrangerInTheJournalIsThisVeryRunNothingWeSayIsFalse(t *testing.T) {
|
|
f := newFixture(t, "10", 500)
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), Chapters: order(100)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.runner.alive = true
|
|
// Our own stream, some progress, and then a handshake under an id the ENGINE minted for this same
|
|
// book because ours had already written. It is indistinguishable from a foreign writer here, which
|
|
// is exactly the point.
|
|
respawned := `{"seq":1,"type":"hello","data":{"stream_version":"1.1","engine_run_id":"` +
|
|
f.live(t).EngineRunID + `-minted-again","book_id":"b"}}` + "\n"
|
|
journal := filepath.Join(f.workdir, ingest.JournalFile)
|
|
if err := os.WriteFile(journal, []byte(hello(t, f)+
|
|
`{"seq":2,"type":"progress","data":{"draft":{"done":3,"total":20},"eta_seconds":99}}`+"\n"+
|
|
respawned), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.StatusReport{TotalUnits: 20, Done: 7, ETASeconds: 42, Spend: usd(0), Reserved: usd(0)}, nil)
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := f.store.ReadRun(f.ctx, "u1", run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// TRUE, and the only claim made: the figures are behind. The run is not accused of anything and
|
|
// nobody is named.
|
|
if !got.ProgressLagging {
|
|
t.Error("the buyer is told nothing while the projection has stopped at a handshake: the bar " +
|
|
"stands, the estimate keeps moving, and the run goes on spending")
|
|
}
|
|
if got.Status != "translating" {
|
|
t.Errorf("the run reads %q: a stream this platform cannot follow is not a run that ended, and "+
|
|
"treating it as one would end a paying run over its own respawn", got.Status)
|
|
}
|
|
if got.FailureReason != "" || got.PausedReason != "" {
|
|
t.Errorf("the run carries reason failure=%q paused=%q: a park is neither", got.FailureReason, got.PausedReason)
|
|
}
|
|
row := stalledRow(t, f, run.ID)
|
|
if row.QuarantineReason != "" {
|
|
t.Errorf("the park was recorded as a quarantine (%q), which offers an operator a lift that would "+
|
|
"refuse — and says the stream is unreadable when it is merely somebody else's", row.QuarantineReason)
|
|
}
|
|
acct := f.account(t)
|
|
if acct.Reserved != fixtureHold(100) {
|
|
t.Errorf("the hold moved to %s while the projection was parked: the run is still going and its "+
|
|
"money must not be touched by our inability to follow it", acct.Reserved.USD())
|
|
}
|
|
}
|