package main import ( "context" "os" "path/filepath" "strings" "testing" "time" "unicode" "unicode/utf8" "github.com/jackc/pgx/v5" "textmachine/platform/internal/runner" ) // seedALiveRun writes the smallest world the runs listing reads: an account, a book, a live run with // a live attempt, and the hold against it. Written as SQL because the subject is the LISTING, and // going through the service would drag the whole runner in to prove a table. func seedALiveRun(t *testing.T, dsn string) { t.Helper() now := time.Now().UTC() execSQL(t, dsn, `insert into users (id, email) values ('u1','u1@example.org')`) execSQL(t, dsn, `insert into books (id, owner_id, title, source_lang, target_lang, status, chapter_count, workdir, engine_book_id, added_at, revision) values ('bk1','u1','蛊真人','zh','ru','translating',10,'/srv/books/bk1','bk1',$1,1)`, now) execSQL(t, dsn, `insert into runs (id, book_id, status, verify_bank, ceiling_chapters, started_at, revision) values ('run_probe','bk1','translating',false,10,$1,1)`, now) execSQL(t, dsn, `insert into run_attempts (run_id, attempt_no, started_at, last_offset, engine_run_id) values ('run_probe',1,$1,0,'tm-stream-run_probe-1')`, now) } func execSQL(t *testing.T, dsn, sql string, args ...any) { t.Helper() ctx := context.Background() conn, err := pgx.Connect(ctx, dsn) if err != nil { t.Fatal(err) } defer conn.Close(ctx) if _, err := conn.Exec(ctx, sql, args...); err != nil { t.Fatalf("%s: %v", sql, err) } } // exit-marker runs INSIDE a run's transient unit, as ExecStopPost, where there is no database and no // credential to open one with. Reaching for the DSN there would fail every marker in production and // lose the only record of how the run ended — so the dispatch has to answer this command before it // reads a secret. Pinned here because the dispatch order is invisible at the call site. func TestTheExitMarkerCommandNeedsNoDatabase(t *testing.T) { t.Setenv("TM_PLATFORM_DSN", "") t.Setenv("TM_PLATFORM_DSN_FILE", "") t.Setenv("SERVICE_RESULT", "oom-kill") t.Setenv("EXIT_CODE", "killed") t.Setenv("EXIT_STATUS", "TERM") path := filepath.Join(t.TempDir(), "runs", "run_1-1.exit") if err := run([]string{"exit-marker", path, "tm-run-run_1-1"}, os.Stdout); err != nil { t.Fatalf("exit-marker with no DSN: %v", err) } m, err := runner.ReadMarker(path) if err != nil { t.Fatal(err) } if m.Result != "oom-kill" || m.Code != "killed" || m.Status != "TERM" || m.Unit != "tm-run-run_1-1" { t.Errorf("marker: %+v", m) } // Every other command still needs one, or it would silently do nothing. if err := run([]string{"balance", "--user", "u1"}, os.Stdout); err == nil { t.Error("a database command ran without a DSN") } } func TestTheExitMarkerCommandNeedsBothArguments(t *testing.T) { t.Setenv("TM_PLATFORM_DSN", "") for _, args := range [][]string{{"exit-marker"}, {"exit-marker", "only-a-path"}} { if err := run(append([]string{}, args...), os.Stdout); err == nil { t.Errorf("%v was accepted", args) } } } // The development intake is what puts a book in the read model until the contract's upload exists. // It refuses a directory that is not an engine project, because the alternative is a run that dies // at argument parsing inside a transient unit, where the only trace is "exit-code 1". func TestAddingABookRefusesADirectoryTheEngineCannotUse(t *testing.T) { dir := t.TempDir() args := []string{"--user", "u1", "--workdir", dir, "--title", "b", "--source-lang", "zh", "--target-lang", "ru", "--chapters", "10", "--characters", "23000000"} _, err := parseBookIntake(args) if err == nil || !strings.Contains(err.Error(), "engine project directory") { t.Fatalf("a directory without book.yaml gave %v", err) } if err := os.WriteFile(filepath.Join(dir, runner.ConfigFile), []byte("book_id: x\n"), 0o600); err != nil { t.Fatal(err) } got, err := parseBookIntake(args) if err != nil { t.Fatalf("a valid project directory was refused: %v", err) } if got.ChapterCount != 10 || got.CharacterCount != 23_000_000 || got.OwnerID != "u1" { t.Errorf("intake: %+v", got) } if !filepath.IsAbs(got.Workdir) { t.Errorf("the workdir was not made absolute: %q — the unit runs with its own working directory", got.Workdir) } } // Chapters are what bound the run-ceiling scale, so a book registered without them would offer a // scale of zero and no run could ever start on it. func TestAddingABookRequiresTheFactsTheScaleIsBuiltFrom(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, runner.ConfigFile), []byte("book_id: x\n"), 0o600); err != nil { t.Fatal(err) } full := []string{"--user", "u1", "--workdir", dir, "--title", "b", "--source-lang", "zh", "--target-lang", "ru", "--chapters", "10"} drop := func(flag string) []string { out := []string{} for i := 0; i < len(full); i += 2 { if full[i] != flag { out = append(out, full[i], full[i+1]) } } return out } for _, flag := range []string{"--user", "--workdir", "--title", "--source-lang", "--target-lang", "--chapters"} { if _, err := parseBookIntake(drop(flag)); err == nil { t.Errorf("a book without %s was accepted", flag) } } } // `tmplatformctl runs` shows LIVE runs, and `--stalled` narrows that to the ones the reconciler // keeps failing on. Both halves are what the usage line promises, and the first half was missing: // the floor was one failure, so on a healthy deployment the command answered "no run is failing to // reconcile" and never showed the runs that were going perfectly well — the same answer it gives // while a run IS wedged and the deferral has not counted it yet. // // Mutation caught: raising the plain form's floor above zero. func TestTheRunsListingShowsLiveRunsAndNarrowsToStalledOnes(t *testing.T) { dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedALiveRun(t, dsn) if got := capture(t, "runs"); !strings.Contains(got, "run_probe") { t.Errorf("the plain listing does not show a live run:\n%s", got) } if got := capture(t, "runs", "--stalled"); strings.Contains(got, "run_probe") { t.Errorf("a run with no failures is reported as stalled:\n%s", got) } // …and once it has failed enough, the narrowed form names it. execSQL(t, dsn, `update run_attempts set reconcile_failures = 99`) if got := capture(t, "runs", "--stalled"); !strings.Contains(got, "run_probe") { t.Errorf("a run past the threshold is missing from --stalled:\n%s", got) } } // The table stays a table, and the figures in it are the ones an operator may act on. // // Both columns it prints from the database are hostile: the engine's stderr arrives verbatim and // multi-byte (the product translates zh/ja), and the TITLE is what the uploader typed — the intake // strips control characters only from titles taken from a filename. A tab forges a column, a newline // forges a row, and a byte-slice through a three-byte rune leaves the tail unreadable. // // The SPENT column is the money half: `spend_micro_usd` holds the engine's BOOK-lifetime figure, so // the attempt's own cost is the difference from its baseline — and with no baseline there is no // honest figure at all, which is what `settle` itself concludes. // // Mutation caught: cutting by byte in oneLine; dropping the control-character map; printing // spend_micro_usd raw; printing zero for an attempt with no baseline. func TestTheOperatorsTableCannotBeForgedByABooksOwnText(t *testing.T) { dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedALiveRun(t, dsn) execSQL(t, dsn, `update books set title = 'A' || chr(9) || 'FORGED' || chr(10) || 'ROW'`) execSQL(t, dsn, `update run_attempts set reconcile_failures = 99, reconcile_error = $1, spend_micro_usd = 123456, spend_baseline_micro_usd = 100000`, "engine: "+strings.Repeat("蛊真人", 100)) got := capture(t, "runs", "--stalled") rows := 0 for _, line := range strings.Split(strings.TrimSpace(got), "\n") { if strings.Contains(line, "FORGED") { rows++ } } switch { case rows != 1: t.Errorf("the book's title spans %d lines of the table: a newline in it forges rows\n%s", rows, got) case strings.Contains(got, "A\tFORGED"): t.Errorf("the book's title kept its tab: it forges a column in a table aligned by them\n%s", got) case !utf8.ValidString(got): t.Errorf("the table is not valid UTF-8: the engine's error was cut through a rune\n%q", got) case !strings.Contains(got, "0.023456"): t.Errorf("SPENT is not this attempt's own cost (123456 - 100000):\n%s", got) } // With no baseline the settlement refuses to price the attempt, and so does the table. execSQL(t, dsn, `update run_attempts set spend_baseline_micro_usd = null`) if got := capture(t, "runs", "--stalled"); !strings.Contains(got, "?") { t.Errorf("an attempt the settlement will not price is given a figure anyway:\n%s", got) } // ⚠ AND THE CONTROL CHARACTERS `strings.Fields` DOES NOT TOUCH, asserted on the function because // the table above cannot show them: tabwriter consumes a tab into padding, so the forged column // is invisible in the output it produces. ESC is the one that matters — a title carrying terminal // escapes rewrites the screen of the operator reading the table. for _, r := range oneLine("A\x1b[2JB\x00C\x07D") { if unicode.IsControl(r) { t.Errorf("oneLine passes %q through to an operator's terminal", r) } } } // seedAStuckSettlement is the OTHER population the operator's table must show: a run that ENDED and // whose money never closed. Written with the same raw INSERTs as its live sibling above, and for the // same reason — this package tests the COMMAND, and the state's own provenance is pinned where the // state is produced (internal/runs, on state grown through the ordinary paths). func seedAStuckSettlement(t *testing.T, dsn string) { t.Helper() now := time.Now().UTC() execSQL(t, dsn, `insert into users (id, email) values ('u2','u2@example.org')`) execSQL(t, dsn, `insert into books (id, owner_id, title, source_lang, target_lang, status, chapter_count, workdir, engine_book_id, added_at, revision) values ('bk2','u2','стоящая книга','zh','ru','failed',10,'/srv/books/bk2','bk2',$1,1)`, now) execSQL(t, dsn, `insert into runs (id, book_id, status, verify_bank, ceiling_chapters, started_at, finished_at, revision) values ('run_stuck','bk2','failed',false,10,$1,$1,1)`, now) execSQL(t, dsn, `insert into run_attempts (run_id, attempt_no, started_at, ended_at, last_offset, engine_run_id, spend_baseline_micro_usd, reconcile_failures, reconcile_error) values ('run_stuck',1,$1,$1,0,'run_stuck#1',0,7,'the settlement could not be computed')`, now) execSQL(t, dsn, `insert into account_balances (user_id, balance_micro_usd, updated_at) values ('u2', 910000, $1)`, now) execSQL(t, dsn, `insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id, note, created_at) values ('u2','grant',1000000,'test','seed','',$1), ('u2','hold',-90000,'run','run_stuck#1','',$1)`, now) execSQL(t, dsn, `insert into reservations (engine_run_id, user_id, book_id, amount_micro_usd, ceiling_micro_usd, state, opened_at) values ('run_stuck#1','u2','bk2',90000,90000,'open',$1)`, now) } // The operator's table shows BOTH halves of a stall and says which is which. // // The ERROR the reconciler logs at the threshold names this command by name. Until PD-385 the // command answered "no run is failing to reconcile" over a run whose money was frozen, because the // store's query was the live phase's alone — so the one sentence that tells an operator to act sent // them to the one surface that could not show it. // // Mutations caught: dropping the PHASE column; the store narrowing back to live attempts; the empty // message going back to "no run is live", which is a claim about half the question. func TestTheRunsListingShowsAStuckSettlementAndNamesThePhase(t *testing.T) { dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedALiveRun(t, dsn) seedAStuckSettlement(t, dsn) plain := capture(t, "runs") for _, want := range []string{"run_probe", "run_stuck", "live", "settling", "PHASE"} { if !strings.Contains(plain, want) { t.Errorf("the plain listing is missing %q — both halves belong to it:\n%s", want, plain) } } stalled := capture(t, "runs", "--stalled") if !strings.Contains(stalled, "run_stuck") { t.Errorf("`runs --stalled` does not show the run whose settlement is stuck, which is exactly "+ "where the threshold's own ERROR sends an operator:\n%s", stalled) } if strings.Contains(stalled, "run_probe") { t.Errorf("a live run with no failures is reported as stalled:\n%s", stalled) } // The hold is the reason the row is worth printing at all. if !strings.Contains(stalled, "0.090000") { t.Errorf("the stuck settlement's HELD column does not show the frozen hold:\n%s", stalled) } } // A settlement that has NOT failed stays out of the plain table: every settlement is briefly open, // and a table that lists healthy work is one an operator stops reading. // // Mutation caught: the store's floor for the settling half going from `greatest($1, 1)` to `$1`. func TestASettlementInFlightIsNotInTheOperatorsTable(t *testing.T) { dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedAStuckSettlement(t, dsn) execSQL(t, dsn, `update run_attempts set reconcile_failures = 0, reconcile_error = null`) if got := capture(t, "runs"); strings.Contains(got, "run_stuck") { t.Errorf("a settlement that has not failed is in the operator's table:\n%s", got) } // ⚠ AND THE OTHER SIDE OF THE SAME FLOOR, which is what makes it a floor and not a ceiling: ONE // failure is where the row starts appearing. Without this the assertion above is satisfied by any // threshold at all — two, five, never — and the number in `greatest($1, 1)` is unexecuted. execSQL(t, dsn, `update run_attempts set reconcile_failures = 1`) if got := capture(t, "runs"); !strings.Contains(got, "run_stuck") { t.Errorf("a settlement that has failed ONCE is not in the plain table: one failure is the "+ "smallest number that means something went wrong, and it is where this row starts "+ "being worth printing\n%s", got) } } // The verdict on a stuck settlement says what it actually did, and a second one says the state // rather than "no such run". // // Both sentences matter: the first is the operator's only confirmation that the money moved, and the // second is the difference between "you were late" and "you were wrong about the id" — answering the // first with the second is how a frozen hold used to read as a typo. // // Mutation caught: printing the live branch's "its hold comes back whole on the next sweep" for a // population no sweep will ever come for; mapping ErrMoneyAlreadyClosed back onto "is not a live run". func TestAbandoningAStuckSettlementSaysWhatItDidAndThenSaysItIsDone(t *testing.T) { dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedAStuckSettlement(t, dsn) got := capture(t, "run", "abandon", "--run", "run_stuck", "--reason", "the engine build was removed") if !strings.Contains(got, "settlement") || !strings.Contains(got, "whole") { t.Errorf("the verdict does not say that the settlement was given up on and the hold returned:\n%s", got) } if strings.Contains(got, "on the next sweep") { t.Error("the verdict promises a sweep, and for this population there will never be one") } // Second time: the state, not "no such run". var out strings.Builder err := run([]string{"run", "abandon", "--run", "run_stuck", "--reason", "again"}, &out) if err == nil { t.Fatal("abandoning twice was accepted") } if !strings.Contains(err.Error(), "already closed") { t.Errorf("the second verdict says %q, want the state it is actually in", err) } } // The operator's hand on the quarantine (PD-426): the listing shows the reason before it is lifted, // the lift clears the column and says the reason back with the cursor the next sweep resumes from, // and the three ways to be wrong about the run answer differently — no such run, a run that is not // quarantined, a run with no live attempt — because the remedies differ. // // Mutation caught: clearing without `returning` the reason; answering ErrNoRun for every miss; // dropping the QUARANTINE column from the listing. func TestLiftingAQuarantineClearsItAndSaysWhatItWas(t *testing.T) { dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedALiveRun(t, dsn) execSQL(t, dsn, `update run_attempts set quarantine_reason = 'ingest: hello payload: json: cannot unmarshal string', last_offset = 4321, last_seq = 7 where run_id = 'run_probe'`) // A neighbour in the same state, so the lift is proven to clear ONE run's attempt and not the column. execSQL(t, dsn, `insert into books (id, owner_id, title, source_lang, target_lang, status, chapter_count, workdir, engine_book_id, added_at, revision) values ('bk2','u1','other','zh','ru','translating',10,'/srv/books/bk2','bk2',now(),1)`) execSQL(t, dsn, `insert into runs (id, book_id, status, verify_bank, ceiling_chapters, started_at, revision) values ('run_other','bk2','translating',false,10,now(),1)`) execSQL(t, dsn, `insert into run_attempts (run_id, attempt_no, started_at, last_offset, engine_run_id, quarantine_reason) values ('run_other',1,now(),0,'tm-stream-run_other-1','ingest: neighbour')`) if got := capture(t, "runs"); !strings.Contains(got, "QUARANTINE") || !strings.Contains(got, "cannot unmarshal string") { t.Fatalf("the listing does not show the quarantine the operator would lift:\n%s", got) } // A reason longer than a table column: the confirmation is the last place the platform holds it, // so it comes out whole. Mutation caught: printing it through oneLine. long := "ingest: journal /srv/textmachine/books/bk1/events.jsonl shrank from 1234567 to 654321 bytes: it is not append-only (the figures are the diagnosis and they sit past a table column's width)" execSQL(t, dsn, `update run_attempts set quarantine_reason = $1 where run_id = 'run_probe'`, long) if got := capture(t, "run", "unquarantine", "--run", "run_probe"); !strings.Contains(got, long) { t.Fatalf("the lift cut the reason it has just erased (%d chars):\n%s", len(long), got) } execSQL(t, dsn, `update run_attempts set quarantine_reason = 'ingest: hello payload: json: cannot unmarshal string' where run_id = 'run_probe'`) got := capture(t, "run", "unquarantine", "--run", "run_probe") for _, want := range []string{"run_probe attempt 1", "quarantine lifted", "offset 4321", "seq 7", "cannot unmarshal string"} { if !strings.Contains(got, want) { t.Errorf("the lift did not say %q:\n%s", want, got) } } var reason *string conn, err := pgx.Connect(t.Context(), dsn) if err != nil { t.Fatal(err) } defer conn.Close(t.Context()) if err := conn.QueryRow(t.Context(), `select quarantine_reason from run_attempts where run_id = 'run_probe'`).Scan(&reason); err != nil { t.Fatal(err) } if reason != nil { t.Fatalf("the column still says %q after the lift", *reason) } var other *string if err := conn.QueryRow(t.Context(), `select quarantine_reason from run_attempts where run_id = 'run_other'`).Scan(&other); err != nil { t.Fatal(err) } if other == nil || *other != "ingest: neighbour" { t.Fatalf("lifting run_probe touched run_other's attempt: %v", other) } if got := capture(t, "runs"); strings.Contains(got, "cannot unmarshal string") { t.Fatalf("the listing still shows a quarantine after the lift:\n%s", got) } var out strings.Builder if err := run([]string{"run", "unquarantine", "--run", "run_probe"}, &out); err == nil || !strings.Contains(err.Error(), "not quarantined") { t.Fatalf("a second lift answered %v (output %q), want a refusal saying it is not quarantined", err, out.String()) } if err := run([]string{"run", "unquarantine", "--run", "run_nope"}, &out); err == nil || !strings.Contains(err.Error(), "there is no run run_nope") { t.Fatalf("a lift on a run that does not exist answered %v", err) } execSQL(t, dsn, `update run_attempts set ended_at = now(), quarantine_reason = 'stale' where run_id = 'run_probe'`) if err := run([]string{"run", "unquarantine", "--run", "run_probe"}, &out); err == nil || !strings.Contains(err.Error(), "no live attempt") { t.Fatalf("a lift on a run whose attempt has ended answered %v", err) } if out.String() != "" { t.Fatalf("a refused lift printed an outcome: %q", out.String()) } } // A settling row's quarantine is history, not an invitation: the lift works on LIVE attempts and the // gauge counts the same set, so the listing shows the reason on live rows only, and a lift asked for // on the finished run says which refusal it is. // // Mutation caught: printing the reason on settling rows too. func TestAQuarantineOnASettlingRowIsHistoryAndNotOfferedForLifting(t *testing.T) { dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedAStuckSettlement(t, dsn) execSQL(t, dsn, `update run_attempts set quarantine_reason = 'ingest: history of a finished run' where run_id = 'run_stuck'`) got := capture(t, "runs") if !strings.Contains(got, "run_stuck") || !strings.Contains(got, "settling") { t.Fatalf("the settling row is missing:\n%s", got) } if strings.Contains(got, "history of a finished run") { t.Fatalf("a settling row offers its old quarantine as if it could be lifted:\n%s", got) } var out strings.Builder if err := run([]string{"run", "unquarantine", "--run", "run_stuck"}, &out); err == nil || !strings.Contains(err.Error(), "no live attempt") { t.Fatalf("a lift on a finished run answered %v, want the no-live-attempt refusal", err) } } // Two facts the table must carry beside the figures, both about states an operator would otherwise // misread as health. // // SPENT is a FLOOR: the engine's committed meter records a call it cancelled in flight at zero // (PD-441), so an exact-looking amount invites the one decision it cannot support — writing a hold off // against it as if the work were priced. // // PARKED is the projection's other stop. It has its own column rather than a word in QUARANTINE // because the remedies are opposite: a quarantine waits for `run unquarantine`, a park clears itself // (PD-438). Before the column, such a run showed an EMPTY quarantine cell and numbers that had simply // stopped moving. func TestTheOperatorsTableSaysSpentIsAFloorAndNamesAParkedProjection(t *testing.T) { dsn := freshDB(t) t.Setenv("TM_PLATFORM_DSN", dsn) seedAStuckSettlement(t, dsn) execSQL(t, dsn, `update run_attempts set spend_micro_usd = 63404 where run_id = 'run_stuck'`) got := capture(t, "runs") if !strings.Contains(got, "≥0.063404") { t.Errorf("SPENT prints an exact amount; the engine's meter is a lower bound:\n%s", got) } if !strings.Contains(got, "SPENT is a FLOOR") { t.Errorf("the table carries no legend saying what the sign means:\n%s", got) } if !strings.Contains(got, "PARKED") { t.Errorf("the table has no PARKED column, so a parked projection is indistinguishable from health:\n%s", got) } // A LIVE attempt, because the park is a live attempt's state and the column is printed for one. seedALiveRun(t, dsn) execSQL(t, dsn, `update run_attempts set parked_at = now() - interval '3 minutes' where run_id = 'run_probe'`) if got := capture(t, "runs"); !strings.Contains(got, "3m0s") { t.Errorf("a parked attempt shows no elapsed time in PARKED, so «how long has it been quiet» "+ "has no answer:\n%s", got) } }