textmachine/platform/docs/p8-review/axis3-queue/live-stalled-settlement.sh

59 lines
4 KiB
Bash
Executable file

#!/usr/bin/env bash
# P8-REVIEW axis 3, finding F2 — the LIVE end-to-end. It builds the state a wedged SETTLEMENT leaves
# and then asks every operator surface about it.
#
# It never starts a translation: the book's directory is moved away first, so `spawnAttempt` refuses
# at `bookMeter` (a $0 `tmctl status`) BEFORE `Runner.Start` is ever reached (internal/runs/spawn.go).
#
# Environment expected (the stand of the P8-REVIEW pack, axis 3):
# ROOT=/home/ubuntu-26/tm-p8-review/a3-queue PGHOST=/tmp PGPORT=55433 PGUSER=postgres
set -euo pipefail
ROOT="${ROOT:?set ROOT to the stand root that holds platform/ and stand/}"
export PATH="$HOME/.local/pgsql/bin:$PATH" PGHOST="${PGHOST:-/tmp}" PGPORT="${PGPORT:-55433}" PGUSER="${PGUSER:-postgres}"
export TM_PLATFORM_DSN="postgres://postgres@/tmp8rev_a3_dev?host=/tmp&port=55433&sslmode=disable"
CTL="$ROOT/tmplatformctl"
dropdb --if-exists tmp8rev_a3_dev; createdb tmp8rev_a3_dev
cd "$ROOT/platform" && go build -o "$ROOT/tmplatformd" ./cmd/tmplatformd && go build -o "$CTL" ./cmd/tmplatformctl
mkdir -p "$ROOT/stand/books" "$ROOT/stand/state"
env TM_PLATFORM_MIGRATE=1 TM_PLATFORM_ADDR=127.0.0.1:8103 TM_PLATFORM_DEV_LOGIN=dev-owner \
TM_PLATFORM_INSECURE_COOKIES=1 TM_PLATFORM_BOOKS_DIR="$ROOT/stand/books" \
TM_PLATFORM_STATE_DIR="$ROOT/stand/state" TM_PLATFORM_ENGINE_BIN="$HOME/tmstand-work/tmctl" \
TM_PLATFORM_BOOK_TEMPLATE="$HOME/tmstand-work/book-template.yaml" TM_PLATFORM_CTL_BIN="$CTL" \
TM_PLATFORM_LANGUAGE_PAIRS="zh>ru,ja>ru:unavailable" TM_PLATFORM_METRICS_ADDR=127.0.0.1:9103 \
"$ROOT/tmplatformd" > "$ROOT/daemon.log" 2>&1 &
PID=$!
trap 'kill '"$PID"' 2>/dev/null || true' EXIT
until curl -s --noproxy '*' -o /dev/null http://127.0.0.1:8103/healthz; do sleep 1; done
TM_PLATFORM_DEV_LOGIN=dev-owner "$CTL" seed --url http://127.0.0.1:8103
BOOK=$(psql -d tmp8rev_a3_dev -At -c "select id from books limit 1")
mv "$ROOT/stand/books/$BOOK" "$ROOT/stand/books/$BOOK.moved" # PD-162 shape: every spawn refuses
J="$ROOT/jar.txt"
curl -s --noproxy '*' -c "$J" -X POST http://127.0.0.1:8103/auth/dev-login -o /dev/null
curl -s --noproxy '*' -b "$J" -X POST "http://127.0.0.1:8103/v0/books/$BOOK/runs" \
-H 'Content-Type: application/json' -H 'X-TM-Client: p8-review' \
-d '{"ceiling_chapters":3,"stop_for_signing":false}' | head -c 200; echo
until [ "$("$CTL" runs 2>&1 | grep -c run_)" -ge 1 ]; do sleep 3; done
echo "### the LIVE half of the treatment, which works:"; "$CTL" runs
RUN=$(psql -d tmp8rev_a3_dev -At -c "select id from runs limit 1")
kill "$PID"; wait "$PID" 2>/dev/null || true # the daemon must not settle it before we look
"$CTL" run abandon --run "$RUN" --reason "p8-review: the workdir is gone"
# Exactly what runs.deferItem writes on the FIFTH consecutive failure of the SETTLEMENT phase.
psql -d tmp8rev_a3_dev -At -c "update run_attempts set reconcile_failures = 5,
reconcile_after = now() + interval '16 minutes',
reconcile_error = 'the settlement took its whole budget without finishing'
where run_id = '$RUN';"
echo; echo "### the settlement is now stalled at the threshold, and the money is frozen:"
psql -d tmp8rev_a3_dev -c "select r.id, r.status, r.settled_at, a.reconcile_failures, res.state, res.amount_micro_usd
from runs r join run_attempts a on a.run_id=r.id
join reservations res on res.engine_run_id = r.id||'#'||a.attempt_no;"
echo "### tmplatformctl runs --stalled (the command the stalled log line names):"; "$CTL" runs --stalled
echo "### tmplatformctl runs:"; "$CTL" runs
echo "### tmplatformctl run abandon (the acting half):"; "$CTL" run abandon --run "$RUN" --reason "p8-review" || true
echo "### the gauge tm_platform_runs_stalled (pgstore.Observe's own predicate):"
psql -d tmp8rev_a3_dev -At -c "select count(*) from run_attempts a join runs r on r.id = a.run_id
where a.ended_at is null and r.finished_at is null and a.reconcile_failures >= 5;"
echo "### the only surface that hints at it:"; "$CTL" books