package runs // P8-REVIEW axis 3 probe. READ-ONLY over the zone: this file adds no production code and is not a // proposed pin — it is the reproduction of two claims about the SETTLEMENT half of the P8-FIX // treatment, run against the same fixture the zone's own tests use. // // Claim A: a settlement that fails CHEAPLY (the engine answers, and its report carries no committed // figure — "absent is not zero", PD-40) is retried on EVERY pass forever. Each retry is a real // `tmctl status` against the book: seconds of engine CPU every 15 s, for as long as the row exists. // Nothing counts it, so `reconcile_failures` stays 0 and StalledAfter is unreachable. // // Claim B: even a settlement that IS counted — deferred to the threshold — is invisible to every // operator surface the pack built: `StalledRuns` and the `tm_platform_runs_stalled` gauge both join // `a.ended_at is null and r.finished_at is null`, and `AbandonRun` refuses a finished run outright. import ( "errors" "testing" "time" "textmachine/platform/internal/ingest" "textmachine/platform/internal/pgstore" "textmachine/platform/internal/runner" ) func TestProbeACheapSettlementFailureIsRetriedForeverAndNeverCounted(t *testing.T) { f := newFixture(t, "20", 500) run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), CeilingChapters: 10}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } a := f.lastAttempt(t, run.ID) if err := writeExitMarker(t, f, run.ID, a); err != nil { t.Fatal(err) } // The engine ANSWERS, quickly, and its report carries no committed figure. This is the state the // zone's own fixture uses for "the money cannot be closed" (stalled_test.go:74, :613). f.engine.set(ingest.StatusReport{TotalUnits: 10, Done: 10}, nil) if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } open, err := f.store.UnsettledRuns(f.ctx, f.svc.now()) if err != nil { t.Fatal(err) } if len(open) != 1 { t.Fatalf("%d unsettled runs, want the one this probe is about", len(open)) } // Five more passes, the clock moving so nothing is skipped for the wrong reason. before := f.engine.called() for i := range 5 { later := f.now.Add(time.Duration(i+1) * time.Minute) f.svc.Now = func() time.Time { return later } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } } t.Logf("engine calls over five passes of one unsettleable run: %d", f.engine.called()-before) var failures int var after *time.Time if err := f.store.Pool().QueryRow(f.ctx, `select reconcile_failures, reconcile_after from run_attempts where id = $1`, open[0].AttemptID). Scan(&failures, &after); err != nil { t.Fatal(err) } t.Logf("reconcile_failures=%d reconcile_after=%v after five passes", failures, after) stalled, err := f.store.StalledRuns(f.ctx, StalledAfter) if err != nil { t.Fatal(err) } o, err := f.store.Observe(f.ctx, StalledAfter) if err != nil { t.Fatalf("observe: %v", err) } t.Logf("StalledRuns(%d)=%d gauge=%d oldest_hold_seconds=%.0f", StalledAfter, len(stalled), o.StalledRuns, o.OldestHoldSeconds) if failures != 0 { t.Logf("NOT REPRODUCED: the cheap failure IS counted") } } func TestProbeAStalledSettlementIsInvisibleToEveryOperatorSurface(t *testing.T) { f := newFixture(t, "20", 500) run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), CeilingChapters: 10}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } a := f.lastAttempt(t, run.ID) if err := writeExitMarker(t, f, run.ID, a); err != nil { t.Fatal(err) } f.engine.set(ingest.StatusReport{TotalUnits: 10, Done: 10}, nil) if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } open, err := f.store.UnsettledRuns(f.ctx, f.svc.now()) if err != nil { t.Fatal(err) } if len(open) != 1 { t.Fatalf("%d unsettled runs", len(open)) } // Now count it to the threshold the way an EXPENSIVE settlement wedge would: the settlement phase // and the reconciliation phase share `deferItem`, so this is the same write settleOne performs. for i := 1; i <= StalledAfter; i++ { got, err := f.store.DeferRun(f.ctx, open[0].AttemptID, f.svc.now().Add(backoff(i)), "the engine did not answer") if err != nil { t.Fatal(err) } if got != i { t.Fatalf("after %d deferrals the row counts %d", i, got) } } // The three surfaces the pack built for exactly this state. stalled, err := f.store.StalledRuns(f.ctx, StalledAfter) if err != nil { t.Fatal(err) } all, err := f.store.StalledRuns(f.ctx, 0) if err != nil { t.Fatal(err) } o, err := f.store.Observe(f.ctx, StalledAfter) if err != nil { t.Fatal(err) } abandon := f.store.AbandonRun(f.ctx, run.ID, "the engine will never answer", true, f.now) t.Logf("attempt %d: reconcile_failures=%d", open[0].AttemptID, StalledAfter) t.Logf("tmplatformctl runs --stalled -> StalledRuns(%d) = %d rows", StalledAfter, len(stalled)) t.Logf("tmplatformctl runs -> StalledRuns(0) = %d rows", len(all)) t.Logf("tm_platform_runs_stalled gauge -> %d", o.StalledRuns) t.Logf("tm_platform_oldest_hold_seconds -> %.0f", o.OldestHoldSeconds) t.Logf("tmplatformctl run abandon -> %v (ErrNoRun=%v)", abandon, errors.Is(abandon, pgstore.ErrNoRun)) if len(stalled) != 0 || o.StalledRuns != 0 { t.Logf("NOT REPRODUCED: the stalled settlement IS visible") } } // writeExitMarker ends an attempt's unit cleanly, the way the zone's own tests do. func writeExitMarker(t *testing.T, f *fixture, runID string, a pgstore.LiveRun) error { t.Helper() return runner.WriteMarker(f.svc.markerPath(runID, a.AttemptNo), runner.Marker{Unit: a.UnitName, Result: "exit-code", Code: "exited", Status: "0"}) } // Claim C: `run abandon`'s SECOND guard is not readable in advance from the operator's own table. // AbandonRun refuses on `unit_name != ” OR spend_baseline is not null` (pgstore/runs.go:465). The // table prints the first (UNIT, and listRuns says so in as many words at runs.go:223-227) and never // the second: the baseline reaches the operator only inverted, as SPENT being a figure rather than // "?". So the row an operator reads as "no unit, safe to abandon" is refused with a message naming a // unit the table said did not exist. func TestProbeTheAbandonsSecondGuardIsNotReadableFromTheOperatorsTable(t *testing.T) { f := newFixture(t, "20", 500) run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), CeilingChapters: 10}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } // `ReleaseSpawnClaim` clears the name when the unit could NOT be created and deliberately keeps // the baseline as the tombstone (spawn.go:130, RecordSpawn). This is that row. if _, err := f.store.Pool().Exec(f.ctx, `update run_attempts set unit_name = null, reconcile_failures = $2 where run_id = $1`, run.ID, StalledAfter); err != nil { t.Fatal(err) } rows, err := f.store.StalledRuns(f.ctx, StalledAfter) if err != nil { t.Fatal(err) } if len(rows) != 1 { t.Fatalf("%d rows", len(rows)) } unit := rows[0].UnitName if unit == "" { unit = "(none)" // exactly what cmd/tmplatformctl/runs.go prints } spent := "?" if rows[0].SpentMicroUSD != nil { spent = "a figure" } err = f.store.AbandonRun(f.ctx, run.ID, "the host is gone", false, f.now) t.Logf("the operator's row: UNIT=%s SPENT=%s FAILS=%d", unit, spent, rows[0].Failures) t.Logf("tmplatformctl run abandon on that row -> %v", err) t.Logf("refused = %v", errors.Is(err, pgstore.ErrRunMayHaveAProcess)) }