textmachine/platform/docs/p8-review/axis3-queue/probe_settlement_surface_test.go.txt

package runs

// P8-REVIEW axis 3 probe. READ-ONLY over the zone: this file adds no production code and is not a
// proposed pin — it is the reproduction of two claims about the SETTLEMENT half of the P8-FIX
// treatment, run against the same fixture the zone's own tests use.
//
// Claim A: a settlement that fails CHEAPLY (the engine answers, and its report carries no committed
// figure — "absent is not zero", PD-40) is retried on EVERY pass forever. Each retry is a real
// `tmctl status` against the book: seconds of engine CPU every 15 s, for as long as the row exists.
// Nothing counts it, so `reconcile_failures` stays 0 and StalledAfter is unreachable.
//
// Claim B: even a settlement that IS counted — deferred to the threshold — is invisible to every
// operator surface the pack built: `StalledRuns` and the `tm_platform_runs_stalled` gauge both join
// `a.ended_at is null and r.finished_at is null`, and `AbandonRun` refuses a finished run outright.

import (
	"errors"
	"testing"
	"time"

	"textmachine/platform/internal/ingest"
	"textmachine/platform/internal/pgstore"
	"textmachine/platform/internal/runner"
)

func TestProbeACheapSettlementFailureIsRetriedForeverAndNeverCounted(t *testing.T) {
	f := newFixture(t, "20", 500)
	run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), CeilingChapters: 10})
	if err != nil {
		t.Fatal(err)
	}
	if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
		t.Fatal(err)
	}
	a := f.lastAttempt(t, run.ID)
	if err := writeExitMarker(t, f, run.ID, a); err != nil {
		t.Fatal(err)
	}
	// The engine ANSWERS, quickly, and its report carries no committed figure. This is the state the
	// zone's own fixture uses for "the money cannot be closed" (stalled_test.go:74, :613).
	f.engine.set(ingest.StatusReport{TotalUnits: 10, Done: 10}, nil)
	if err := f.svc.Sweep(f.ctx); err != nil {
		t.Fatal(err)
	}
	open, err := f.store.UnsettledRuns(f.ctx, f.svc.now())
	if err != nil {
		t.Fatal(err)
	}
	if len(open) != 1 {
		t.Fatalf("%d unsettled runs, want the one this probe is about", len(open))
	}
	// Five more passes, the clock moving so nothing is skipped for the wrong reason.
	before := f.engine.called()
	for i := range 5 {
		later := f.now.Add(time.Duration(i+1) * time.Minute)
		f.svc.Now = func() time.Time { return later }
		if err := f.svc.Sweep(f.ctx); err != nil {
			t.Fatal(err)
		}
	}
	t.Logf("engine calls over five passes of one unsettleable run: %d", f.engine.called()-before)
	var failures int
	var after *time.Time
	if err := f.store.Pool().QueryRow(f.ctx,
		`select reconcile_failures, reconcile_after from run_attempts where id = $1`, open[0].AttemptID).
		Scan(&failures, &after); err != nil {
		t.Fatal(err)
	}
	t.Logf("reconcile_failures=%d reconcile_after=%v after five passes", failures, after)
	stalled, err := f.store.StalledRuns(f.ctx, StalledAfter)
	if err != nil {
		t.Fatal(err)
	}
	o, err := f.store.Observe(f.ctx, StalledAfter)
	if err != nil {
		t.Fatalf("observe: %v", err)
	}
	t.Logf("StalledRuns(%d)=%d gauge=%d oldest_hold_seconds=%.0f",
		StalledAfter, len(stalled), o.StalledRuns, o.OldestHoldSeconds)
	if failures != 0 {
		t.Logf("NOT REPRODUCED: the cheap failure IS counted")
	}
}

func TestProbeAStalledSettlementIsInvisibleToEveryOperatorSurface(t *testing.T) {
	f := newFixture(t, "20", 500)
	run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), CeilingChapters: 10})
	if err != nil {
		t.Fatal(err)
	}
	if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
		t.Fatal(err)
	}
	a := f.lastAttempt(t, run.ID)
	if err := writeExitMarker(t, f, run.ID, a); err != nil {
		t.Fatal(err)
	}
	f.engine.set(ingest.StatusReport{TotalUnits: 10, Done: 10}, nil)
	if err := f.svc.Sweep(f.ctx); err != nil {
		t.Fatal(err)
	}
	open, err := f.store.UnsettledRuns(f.ctx, f.svc.now())
	if err != nil {
		t.Fatal(err)
	}
	if len(open) != 1 {
		t.Fatalf("%d unsettled runs", len(open))
	}
	// Now count it to the threshold the way an EXPENSIVE settlement wedge would: the settlement phase
	// and the reconciliation phase share `deferItem`, so this is the same write settleOne performs.
	for i := 1; i <= StalledAfter; i++ {
		got, err := f.store.DeferRun(f.ctx, open[0].AttemptID, f.svc.now().Add(backoff(i)), "the engine did not answer")
		if err != nil {
			t.Fatal(err)
		}
		if got != i {
			t.Fatalf("after %d deferrals the row counts %d", i, got)
		}
	}
	// The three surfaces the pack built for exactly this state.
	stalled, err := f.store.StalledRuns(f.ctx, StalledAfter)
	if err != nil {
		t.Fatal(err)
	}
	all, err := f.store.StalledRuns(f.ctx, 0)
	if err != nil {
		t.Fatal(err)
	}
	o, err := f.store.Observe(f.ctx, StalledAfter)
	if err != nil {
		t.Fatal(err)
	}
	abandon := f.store.AbandonRun(f.ctx, run.ID, "the engine will never answer", true, f.now)
	t.Logf("attempt %d: reconcile_failures=%d", open[0].AttemptID, StalledAfter)
	t.Logf("tmplatformctl runs --stalled  -> StalledRuns(%d) = %d rows", StalledAfter, len(stalled))
	t.Logf("tmplatformctl runs            -> StalledRuns(0)  = %d rows", len(all))
	t.Logf("tm_platform_runs_stalled gauge -> %d", o.StalledRuns)
	t.Logf("tm_platform_oldest_hold_seconds -> %.0f", o.OldestHoldSeconds)
	t.Logf("tmplatformctl run abandon      -> %v (ErrNoRun=%v)", abandon, errors.Is(abandon, pgstore.ErrNoRun))
	if len(stalled) != 0 || o.StalledRuns != 0 {
		t.Logf("NOT REPRODUCED: the stalled settlement IS visible")
	}
}

// writeExitMarker ends an attempt's unit cleanly, the way the zone's own tests do.
func writeExitMarker(t *testing.T, f *fixture, runID string, a pgstore.LiveRun) error {
	t.Helper()
	return runner.WriteMarker(f.svc.markerPath(runID, a.AttemptNo),
		runner.Marker{Unit: a.UnitName, Result: "exit-code", Code: "exited", Status: "0"})
}

// Claim C: `run abandon`'s SECOND guard is not readable in advance from the operator's own table.
// AbandonRun refuses on `unit_name != ” OR spend_baseline is not null` (pgstore/runs.go:465). The
// table prints the first (UNIT, and listRuns says so in as many words at runs.go:223-227) and never
// the second: the baseline reaches the operator only inverted, as SPENT being a figure rather than
// "?". So the row an operator reads as "no unit, safe to abandon" is refused with a message naming a
// unit the table said did not exist.
func TestProbeTheAbandonsSecondGuardIsNotReadableFromTheOperatorsTable(t *testing.T) {
	f := newFixture(t, "20", 500)
	run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: f.bookID(t), CeilingChapters: 10})
	if err != nil {
		t.Fatal(err)
	}
	if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
		t.Fatal(err)
	}
	// `ReleaseSpawnClaim` clears the name when the unit could NOT be created and deliberately keeps
	// the baseline as the tombstone (spawn.go:130, RecordSpawn). This is that row.
	if _, err := f.store.Pool().Exec(f.ctx,
		`update run_attempts set unit_name = null, reconcile_failures = $2 where run_id = $1`,
		run.ID, StalledAfter); err != nil {
		t.Fatal(err)
	}
	rows, err := f.store.StalledRuns(f.ctx, StalledAfter)
	if err != nil {
		t.Fatal(err)
	}
	if len(rows) != 1 {
		t.Fatalf("%d rows", len(rows))
	}
	unit := rows[0].UnitName
	if unit == "" {
		unit = "(none)" // exactly what cmd/tmplatformctl/runs.go prints
	}
	spent := "?"
	if rows[0].SpentMicroUSD != nil {
		spent = "a figure"
	}
	err = f.store.AbandonRun(f.ctx, run.ID, "the host is gone", false, f.now)
	t.Logf("the operator's row: UNIT=%s SPENT=%s FAILS=%d", unit, spent, rows[0].Failures)
	t.Logf("tmplatformctl run abandon on that row -> %v", err)
	t.Logf("refused = %v", errors.Is(err, pgstore.ErrRunMayHaveAProcess))
}