package pgstore import ( "errors" "fmt" "strings" "sync" "testing" "time" "textmachine/platform/internal/money" ) // The balance cache and the ledger are two representations of the same fact, and money is the one // place where "usually consistent" is not a property. This asserts they agree after EVERY step. // Mutation caught: updating account_balances outside the ledger's transaction, or skipping it. func TestCreditLifecycleKeepsTheCacheEqualToTheLedger(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','蛊真人','zh','ru','not_started','/srv/books/bk1','gzr')`) now := time.Now().UTC() check := func(step string, want money.MicroUSD) { t.Helper() a, err := s.ReadAccount(ctx, "u1") if err != nil { t.Fatalf("%s: %v", step, err) } if a.Balance != a.LedgerSum { t.Fatalf("%s: cached balance %d disagrees with the ledger %d", step, a.Balance, a.LedgerSum) } if a.Balance != want { t.Fatalf("%s: balance = %s, want %s", step, a.Balance.USD(), want.USD()) } } if _, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "g1", "free tier", now); err != nil { t.Fatal(err) } check("after the grant", 5*money.PerUSD) if err := s.Hold(ctx, "u1", "bk1", "run-1", 2*money.PerUSD, now); err != nil { t.Fatal(err) } check("while a run is held", 3*money.PerUSD) // The attempt cost less than it reserved: the difference comes back. if err := s.Settle(ctx, "run-1", 1_200_000, now); err != nil { t.Fatal(err) } check("after settlement", 5*money.PerUSD-1_200_000) // A second settlement of the same attempt changes nothing: the reservation is no longer open. if err := s.Settle(ctx, "run-1", 1_200_000, now); !errors.Is(err, ErrNoReservation) { t.Fatalf("a repeated settlement must be refused, got %v", err) } check("after a repeated settlement", 5*money.PerUSD-1_200_000) // A run that never spent gives its whole reservation back. if err := s.Hold(ctx, "u1", "bk1", "run-2", 1*money.PerUSD, now); err != nil { t.Fatal(err) } check("while the second run is held", 5*money.PerUSD-1_200_000-1*money.PerUSD) if err := s.Release(ctx, "run-2", now); err != nil { t.Fatal(err) } check("after release", 5*money.PerUSD-1_200_000) } // Idempotency is what makes a retried worker safe. Mutation caught: dropping the ON CONFLICT clause // (the insert then fails) or moving the balance update outside the "actually inserted" branch (the // balance then doubles while the ledger does not). func TestGrantIsIdempotentBySource(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") now := time.Now().UTC() for i := range 3 { applied, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "same-key", "free tier", now) if err != nil { t.Fatal(err) } // The caller must be able to tell "credited" from "already spent": a CLI that prints // success on the second call tells an operator money moved when it did not. if applied != (i == 0) { t.Fatalf("call %d reported applied=%v", i, applied) } } got, err := s.Balance(ctx, "u1") if err != nil { t.Fatal(err) } if got != 5*money.PerUSD { t.Fatalf("three identical grants credited %s", got.USD()) } var rows int if err := s.pool.QueryRow(ctx, `select count(*) from credit_ledger where user_id='u1'`).Scan(&rows); err != nil { t.Fatal(err) } if rows != 1 { t.Fatalf("ledger has %d rows for one grant", rows) } } // The hold is the enforcement half of the design: credit that is reserved is not available to the // next run. Mutation caught: removing the balance check. The FOR UPDATE that serialises it is a // CONCURRENCY property and no sequential test can see it — that one is pinned below. func TestHoldRefusesMoreThanTheBalance(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','蛊真人','zh','ru','not_started','/srv/books/bk1','gzr')`) now := time.Now().UTC() if _, err := s.Grant(ctx, "u1", 1*money.PerUSD, "admin", "g1", "free tier", now); err != nil { t.Fatal(err) } if err := s.Hold(ctx, "u1", "bk1", "run-1", 2*money.PerUSD, now); !errors.Is(err, ErrInsufficientCredit) { t.Fatalf("a hold beyond the balance must be refused, got %v", err) } // And the refusal left nothing behind. got, err := s.Balance(ctx, "u1") if err != nil { t.Fatal(err) } if got != 1*money.PerUSD { t.Fatalf("balance moved on a refused hold: %s", got.USD()) } var reservations int if err := s.pool.QueryRow(ctx, `select count(*) from reservations`).Scan(&reservations); err != nil { t.Fatal(err) } if reservations != 0 { t.Fatalf("a refused hold left %d reservations", reservations) } // An account with no credit at all is refused the same way, not crashed. seedUser(t, s, ctx, "u2") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk2','u2','x','zh','ru','not_started','/srv/books/bk2','x')`) if err := s.Hold(ctx, "u2", "bk2", "run-2", 1, now); !errors.Is(err, ErrInsufficientCredit) { t.Fatalf("an account with no ledger must be refused, got %v", err) } } // The sign rules are DDL, so a sign error in the code that writes money fails at the write instead // of quietly topping an account up. func TestLedgerRefusesWrongSigns(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") t.Run("a grant is never a debit", func(t *testing.T) { assertViolation(t, s, ctx, "credit_ledger_sign", `insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id) values ('u1','grant',-1,'x','1')`) }) t.Run("a hold is never a credit", func(t *testing.T) { assertViolation(t, s, ctx, "credit_ledger_sign", `insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id) values ('u1','hold',1,'x','2')`) }) t.Run("a settlement never credits", func(t *testing.T) { assertViolation(t, s, ctx, "credit_ledger_sign", `insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id) values ('u1','settlement',1,'x','3')`) }) t.Run("an adjustment carries a reason", func(t *testing.T) { assertViolation(t, s, ctx, "credit_ledger_adjustment_has_note", `insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id) values ('u1','adjustment',5,'x','4')`) }) t.Run("a closed reservation has a closing time", func(t *testing.T) { exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`) assertViolation(t, s, ctx, "reservations_closed_has_time", `insert into reservations (engine_run_id, user_id, book_id, amount_micro_usd, ceiling_micro_usd, state) values ('r1','u1','bk1',1,1,'settled')`) }) } // The payer must own the book. The database refuses it, not a check the next caller has to // remember: charging one account for another's translation is the money shape of API1 BOLA. // Mutation caught: inserting the reservation without the owner condition, or dropping the // composite foreign key. func TestHoldRefusesAnotherAccountsBook(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") seedUser(t, s, ctx, "u2") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`) now := time.Now().UTC() if _, err := s.Grant(ctx, "u2", 5*money.PerUSD, "admin", "g", "", now); err != nil { t.Fatal(err) } if err := s.Hold(ctx, "u2", "bk1", "run-1", money.PerUSD, now); !errors.Is(err, ErrNotOwner) { t.Fatalf("holding against another account's book returned %v", err) } a, err := s.ReadAccount(ctx, "u2") if err != nil { t.Fatal(err) } if a.Balance != 5*money.PerUSD { t.Fatalf("the refused hold moved money: %s", a.Balance.USD()) } } // A hold that debits nothing reserves nothing. If the ledger already holds this attempt id, the // insert is a no-op and reporting success would spawn a run against credit nobody set aside. func TestSecondHoldOnOneAttemptIsRefused(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`) now := time.Now().UTC() if _, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "g", "", now); err != nil { t.Fatal(err) } if err := s.Hold(ctx, "u1", "bk1", "run-1", money.PerUSD, now); err != nil { t.Fatal(err) } if err := s.Release(ctx, "run-1", now); err != nil { t.Fatal(err) } // Same attempt id again: the reservation row is gone from `open`, but the ledger key is spent. if err := s.Hold(ctx, "u1", "bk1", "run-1", money.PerUSD, now); err == nil { t.Fatal("a second hold on one attempt id was accepted; it debited nothing") } } // Spending more than was reserved means the engine's ceiling did not hold. The account is not the // place to absorb that: the settlement is capped and the row says so. // Mutation caught: settling the reported amount unchecked (the balance then goes negative). func TestSettlementIsCappedAtTheHold(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`) now := time.Now().UTC() if _, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "g", "", now); err != nil { t.Fatal(err) } if err := s.Hold(ctx, "u1", "bk1", "run-1", 2*money.PerUSD, now); err != nil { t.Fatal(err) } if err := s.Settle(ctx, "run-1", 500*money.PerUSD, now); err != nil { t.Fatal(err) } a, err := s.ReadAccount(ctx, "u1") if err != nil { t.Fatal(err) } if a.Balance != 3*money.PerUSD { t.Fatalf("balance = %s, want the hold and nothing more taken", a.Balance.USD()) } var note string if err := s.pool.QueryRow(ctx, `select note from credit_ledger where kind='settlement'`).Scan(¬e); err != nil { t.Fatal(err) } if note == "" { t.Fatal("a capped settlement must say so in the row") } } // A book with money reserved against it cannot be deleted. Cascading here would leave the `hold` // row in the ledger with nothing left to release it. func TestBookWithAnOpenHoldCannotBeDeleted(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`) now := time.Now().UTC() if _, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "g", "", now); err != nil { t.Fatal(err) } if err := s.Hold(ctx, "u1", "bk1", "run-1", money.PerUSD, now); err != nil { t.Fatal(err) } if err := s.DeleteBook(ctx, "bk1"); err == nil { t.Fatal("a book with an open hold was deleted; its ledger debit is now unreleasable") } if err := s.Release(ctx, "run-1", now); err != nil { t.Fatal(err) } if err := s.DeleteBook(ctx, "bk1"); err != nil { t.Fatalf("a book with no open hold must be deletable: %v", err) } // The money history stays: the ledger is the financial record, the reservation was bookkeeping. a, err := s.ReadAccount(ctx, "u1") if err != nil { t.Fatal(err) } if a.Balance != 5*money.PerUSD || a.Balance != a.LedgerSum { t.Fatalf("deleting the book moved money: %s", a.Balance.USD()) } } // PD-26/PD-52. Every money operation takes the balance row lock FIRST, and the rule is only worth // having if a test notices its removal. The cycle needs a settlement and a hold that touch the same // reservation row: with the lock taken first in both, Settle waits for the balance before it touches // the row, so Hold never waits on a row while holding what Settle wants. Take it out of // closeReservation and the two acquire in opposite orders. // // Written by acceptance; re-measured here on PostgreSQL 18.4: with the lock order inverted it fails // 5 runs out of 5, at 5-10 deadlocks per 150 rounds, and with the fix it is green. Probabilistic in // the failing direction, which is why the round count stays high. // Mutation caught: deleting the lockBalance call from closeReservation. func TestHoldAndSettleOnTheSameAttemptDoNotDeadlock(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`) now := time.Now().UTC() if _, err := s.Grant(ctx, "u1", 100000*money.PerUSD, "admin", "g", "", now); err != nil { t.Fatal(err) } var mu sync.Mutex var deadlocks int note := func(err error) { if err != nil && strings.Contains(err.Error(), "deadlock") { mu.Lock() deadlocks++ mu.Unlock() } } for i := range 150 { id := fmt.Sprintf("run-%d", i) if err := s.Hold(ctx, "u1", "bk1", id, money.PerUSD, now); err != nil { t.Fatal(err) } var wg sync.WaitGroup wg.Go(func() { note(s.Settle(ctx, id, money.PerUSD/2, now)) }) wg.Go(func() { note(s.Hold(ctx, "u1", "bk1", id, money.PerUSD, now)) }) wg.Wait() } a, err := s.ReadAccount(ctx, "u1") if err != nil { t.Fatal(err) } if a.Balance != a.LedgerSum { t.Fatalf("the cache drifted from the ledger: %s against %s", a.Balance.USD(), a.LedgerSum.USD()) } if deadlocks > 0 { t.Fatalf("%d deadlocks in 150 rounds: the lock order is not the same in both paths", deadlocks) } } // PD-56. A typo in an account id is the commonest operator error, and every money entry point must // name it the same way. Before this, Balance said "no such account" while Grant and Adjust returned // the Postgres constraint name — which reads as a broken database, not a mistyped id. // Mutation caught: dropping the constraint check in appendLedger. func TestMoneyOperationsAgreeOnAMissingAccount(t *testing.T) { s, ctx := testDB(t) now := time.Now().UTC() grant := func() error { _, err := s.Grant(ctx, "no-such-user", money.PerUSD, "admin", "k1", "", now); return err } adjust := func() error { _, err := s.Adjust(ctx, "no-such-user", money.PerUSD, "admin", "k2", "why", now) return err } balance := func() error { _, err := s.Balance(ctx, "no-such-user"); return err } read := func() error { _, err := s.ReadAccount(ctx, "no-such-user"); return err } for name, call := range map[string]func() error{ "grant": grant, "adjust": adjust, "balance": balance, "read account": read, } { t.Run(name, func(t *testing.T) { if err := call(); !errors.Is(err, ErrNoAccount) { t.Fatalf("got %v, want ErrNoAccount", err) } }) } } // The row lock, not the comparison, is what stops two runs from spending the same credit. Each hold // here is affordable on its own and they are not affordable together, so without FOR UPDATE both // read the same balance, both pass the check, and the account goes negative — the cache and the // ledger drifting together, which is why the invariant assertions elsewhere cannot see it either. // Found by review: the sequential test above claimed this and could not deliver it. // Mutation caught: dropping `for update` from lockBalance. func TestConcurrentHoldsCannotOvercommitAnAccount(t *testing.T) { s, ctx := testDB(t) seedUser(t, s, ctx, "u1") exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`) now := time.Now().UTC() const rounds = 60 for i := range rounds { user := fmt.Sprintf("u-%d", i) book := fmt.Sprintf("bk-%d", i) seedUser(t, s, ctx, user) exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id) values ($1,$2,'x','zh','ru','not_started','/srv/books/x','x')`, book, user) // Ten dollars, and two runs that each want six. if _, err := s.Grant(ctx, user, 10*money.PerUSD, "admin", "g", "", now); err != nil { t.Fatal(err) } var wg sync.WaitGroup var mu sync.Mutex var granted int for j := range 2 { wg.Go(func() { err := s.Hold(ctx, user, book, fmt.Sprintf("run-%d-%d", i, j), 6*money.PerUSD, now) switch { case err == nil: mu.Lock() granted++ mu.Unlock() case errors.Is(err, ErrInsufficientCredit): default: mu.Lock() t.Errorf("round %d: unexpected hold error: %v", i, err) mu.Unlock() } }) } wg.Wait() a, err := s.ReadAccount(ctx, user) if err != nil { t.Fatal(err) } if granted != 1 { t.Fatalf("round %d: %d of two competing holds were granted from one balance; balance is now %s", i, granted, a.Balance.USD()) } if a.Balance < 0 { t.Fatalf("round %d: balance went negative (%s): two runs spent the same credit", i, a.Balance.USD()) } if a.Balance != a.LedgerSum { t.Fatalf("round %d: cache %s disagrees with the ledger %s", i, a.Balance.USD(), a.LedgerSum.USD()) } } }