textmachine/platform/internal/pgstore/credits_test.go

package pgstore

import (
	"errors"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"

	"textmachine/platform/internal/money"
)

// The balance cache and the ledger are two representations of the same fact, and money is the one
// place where "usually consistent" is not a property. This asserts they agree after EVERY step.
// Mutation caught: updating account_balances outside the ledger's transaction, or skipping it.
func TestCreditLifecycleKeepsTheCacheEqualToTheLedger(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk1','u1','蛊真人','zh','ru','not_started','/srv/books/bk1','gzr')`)
	now := time.Now().UTC()

	check := func(step string, want money.MicroUSD) {
		t.Helper()
		a, err := s.ReadAccount(ctx, "u1")
		if err != nil {
			t.Fatalf("%s: %v", step, err)
		}
		if a.Balance != a.LedgerSum {
			t.Fatalf("%s: cached balance %d disagrees with the ledger %d", step, a.Balance, a.LedgerSum)
		}
		if a.Balance != want {
			t.Fatalf("%s: balance = %s, want %s", step, a.Balance.USD(), want.USD())
		}
	}

	if _, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "g1", "free tier", now); err != nil {
		t.Fatal(err)
	}
	check("after the grant", 5*money.PerUSD)

	if err := s.Hold(ctx, "u1", "bk1", "run-1", 2*money.PerUSD, now); err != nil {
		t.Fatal(err)
	}
	check("while a run is held", 3*money.PerUSD)

	// The attempt cost less than it reserved: the difference comes back.
	if err := s.Settle(ctx, "run-1", 1_200_000, now); err != nil {
		t.Fatal(err)
	}
	check("after settlement", 5*money.PerUSD-1_200_000)

	// A second settlement of the same attempt changes nothing: the reservation is no longer open.
	if err := s.Settle(ctx, "run-1", 1_200_000, now); !errors.Is(err, ErrNoReservation) {
		t.Fatalf("a repeated settlement must be refused, got %v", err)
	}
	check("after a repeated settlement", 5*money.PerUSD-1_200_000)

	// A run that never spent gives its whole reservation back.
	if err := s.Hold(ctx, "u1", "bk1", "run-2", 1*money.PerUSD, now); err != nil {
		t.Fatal(err)
	}
	check("while the second run is held", 5*money.PerUSD-1_200_000-1*money.PerUSD)
	if err := s.Release(ctx, "run-2", now); err != nil {
		t.Fatal(err)
	}
	check("after release", 5*money.PerUSD-1_200_000)
}

// Idempotency is what makes a retried worker safe. Mutation caught: dropping the ON CONFLICT clause
// (the insert then fails) or moving the balance update outside the "actually inserted" branch (the
// balance then doubles while the ledger does not).
func TestGrantIsIdempotentBySource(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	now := time.Now().UTC()

	for i := range 3 {
		applied, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "same-key", "free tier", now)
		if err != nil {
			t.Fatal(err)
		}
		// The caller must be able to tell "credited" from "already spent": a CLI that prints
		// success on the second call tells an operator money moved when it did not.
		if applied != (i == 0) {
			t.Fatalf("call %d reported applied=%v", i, applied)
		}
	}
	got, err := s.Balance(ctx, "u1")
	if err != nil {
		t.Fatal(err)
	}
	if got != 5*money.PerUSD {
		t.Fatalf("three identical grants credited %s", got.USD())
	}
	var rows int
	if err := s.pool.QueryRow(ctx, `select count(*) from credit_ledger where user_id='u1'`).Scan(&rows); err != nil {
		t.Fatal(err)
	}
	if rows != 1 {
		t.Fatalf("ledger has %d rows for one grant", rows)
	}
}

// The hold is the enforcement half of the design: credit that is reserved is not available to the
// next run. Mutation caught: removing the balance check. The FOR UPDATE that serialises it is a
// CONCURRENCY property and no sequential test can see it — that one is pinned below.
func TestHoldRefusesMoreThanTheBalance(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk1','u1','蛊真人','zh','ru','not_started','/srv/books/bk1','gzr')`)
	now := time.Now().UTC()
	if _, err := s.Grant(ctx, "u1", 1*money.PerUSD, "admin", "g1", "free tier", now); err != nil {
		t.Fatal(err)
	}

	if err := s.Hold(ctx, "u1", "bk1", "run-1", 2*money.PerUSD, now); !errors.Is(err, ErrInsufficientCredit) {
		t.Fatalf("a hold beyond the balance must be refused, got %v", err)
	}
	// And the refusal left nothing behind.
	got, err := s.Balance(ctx, "u1")
	if err != nil {
		t.Fatal(err)
	}
	if got != 1*money.PerUSD {
		t.Fatalf("balance moved on a refused hold: %s", got.USD())
	}
	var reservations int
	if err := s.pool.QueryRow(ctx, `select count(*) from reservations`).Scan(&reservations); err != nil {
		t.Fatal(err)
	}
	if reservations != 0 {
		t.Fatalf("a refused hold left %d reservations", reservations)
	}

	// An account with no credit at all is refused the same way, not crashed.
	seedUser(t, s, ctx, "u2")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk2','u2','x','zh','ru','not_started','/srv/books/bk2','x')`)
	if err := s.Hold(ctx, "u2", "bk2", "run-2", 1, now); !errors.Is(err, ErrInsufficientCredit) {
		t.Fatalf("an account with no ledger must be refused, got %v", err)
	}
}

// The sign rules are DDL, so a sign error in the code that writes money fails at the write instead
// of quietly topping an account up.
func TestLedgerRefusesWrongSigns(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	t.Run("a grant is never a debit", func(t *testing.T) {
		assertViolation(t, s, ctx, "credit_ledger_sign",
			`insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id)
			 values ('u1','grant',-1,'x','1')`)
	})
	t.Run("a hold is never a credit", func(t *testing.T) {
		assertViolation(t, s, ctx, "credit_ledger_sign",
			`insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id)
			 values ('u1','hold',1,'x','2')`)
	})
	t.Run("a settlement never credits", func(t *testing.T) {
		assertViolation(t, s, ctx, "credit_ledger_sign",
			`insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id)
			 values ('u1','settlement',1,'x','3')`)
	})
	t.Run("an adjustment carries a reason", func(t *testing.T) {
		assertViolation(t, s, ctx, "credit_ledger_adjustment_has_note",
			`insert into credit_ledger (user_id, kind, amount_micro_usd, source, source_id)
			 values ('u1','adjustment',5,'x','4')`)
	})
	t.Run("a closed reservation has a closing time", func(t *testing.T) {
		exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
		                 values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`)
		assertViolation(t, s, ctx, "reservations_closed_has_time",
			`insert into reservations (engine_run_id, user_id, book_id, amount_micro_usd, ceiling_micro_usd, state)
			 values ('r1','u1','bk1',1,1,'settled')`)
	})
}

// The payer must own the book. The database refuses it, not a check the next caller has to
// remember: charging one account for another's translation is the money shape of API1 BOLA.
// Mutation caught: inserting the reservation without the owner condition, or dropping the
// composite foreign key.
func TestHoldRefusesAnotherAccountsBook(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	seedUser(t, s, ctx, "u2")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`)
	now := time.Now().UTC()
	if _, err := s.Grant(ctx, "u2", 5*money.PerUSD, "admin", "g", "", now); err != nil {
		t.Fatal(err)
	}
	if err := s.Hold(ctx, "u2", "bk1", "run-1", money.PerUSD, now); !errors.Is(err, ErrNotOwner) {
		t.Fatalf("holding against another account's book returned %v", err)
	}
	a, err := s.ReadAccount(ctx, "u2")
	if err != nil {
		t.Fatal(err)
	}
	if a.Balance != 5*money.PerUSD {
		t.Fatalf("the refused hold moved money: %s", a.Balance.USD())
	}
}

// A hold that debits nothing reserves nothing. If the ledger already holds this attempt id, the
// insert is a no-op and reporting success would spawn a run against credit nobody set aside.
func TestSecondHoldOnOneAttemptIsRefused(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`)
	now := time.Now().UTC()
	if _, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "g", "", now); err != nil {
		t.Fatal(err)
	}
	if err := s.Hold(ctx, "u1", "bk1", "run-1", money.PerUSD, now); err != nil {
		t.Fatal(err)
	}
	if err := s.Release(ctx, "run-1", now); err != nil {
		t.Fatal(err)
	}
	// Same attempt id again: the reservation row is gone from `open`, but the ledger key is spent.
	if err := s.Hold(ctx, "u1", "bk1", "run-1", money.PerUSD, now); err == nil {
		t.Fatal("a second hold on one attempt id was accepted; it debited nothing")
	}
}

// Spending more than was reserved means the engine's ceiling did not hold. The account is not the
// place to absorb that: the settlement is capped and the row says so.
// Mutation caught: settling the reported amount unchecked (the balance then goes negative).
func TestSettlementIsCappedAtTheHold(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`)
	now := time.Now().UTC()
	if _, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "g", "", now); err != nil {
		t.Fatal(err)
	}
	if err := s.Hold(ctx, "u1", "bk1", "run-1", 2*money.PerUSD, now); err != nil {
		t.Fatal(err)
	}
	if err := s.Settle(ctx, "run-1", 500*money.PerUSD, now); err != nil {
		t.Fatal(err)
	}
	a, err := s.ReadAccount(ctx, "u1")
	if err != nil {
		t.Fatal(err)
	}
	if a.Balance != 3*money.PerUSD {
		t.Fatalf("balance = %s, want the hold and nothing more taken", a.Balance.USD())
	}
	var note string
	if err := s.pool.QueryRow(ctx,
		`select note from credit_ledger where kind='settlement'`).Scan(&note); err != nil {
		t.Fatal(err)
	}
	if note == "" {
		t.Fatal("a capped settlement must say so in the row")
	}
}

// A book with money reserved against it cannot be deleted. Cascading here would leave the `hold`
// row in the ledger with nothing left to release it.
func TestBookWithAnOpenHoldCannotBeDeleted(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`)
	now := time.Now().UTC()
	if _, err := s.Grant(ctx, "u1", 5*money.PerUSD, "admin", "g", "", now); err != nil {
		t.Fatal(err)
	}
	if err := s.Hold(ctx, "u1", "bk1", "run-1", money.PerUSD, now); err != nil {
		t.Fatal(err)
	}
	if err := s.DeleteBook(ctx, "bk1"); err == nil {
		t.Fatal("a book with an open hold was deleted; its ledger debit is now unreleasable")
	}
	if err := s.Release(ctx, "run-1", now); err != nil {
		t.Fatal(err)
	}
	if err := s.DeleteBook(ctx, "bk1"); err != nil {
		t.Fatalf("a book with no open hold must be deletable: %v", err)
	}
	// The money history stays: the ledger is the financial record, the reservation was bookkeeping.
	a, err := s.ReadAccount(ctx, "u1")
	if err != nil {
		t.Fatal(err)
	}
	if a.Balance != 5*money.PerUSD || a.Balance != a.LedgerSum {
		t.Fatalf("deleting the book moved money: %s", a.Balance.USD())
	}
}

// PD-26/PD-52. Every money operation takes the balance row lock FIRST, and the rule is only worth
// having if a test notices its removal. The cycle needs a settlement and a hold that touch the same
// reservation row: with the lock taken first in both, Settle waits for the balance before it touches
// the row, so Hold never waits on a row while holding what Settle wants. Take it out of
// closeReservation and the two acquire in opposite orders.
//
// Written by acceptance; re-measured here on PostgreSQL 18.4: with the lock order inverted it fails
// 5 runs out of 5, at 5-10 deadlocks per 150 rounds, and with the fix it is green. Probabilistic in
// the failing direction, which is why the round count stays high.
// Mutation caught: deleting the lockBalance call from closeReservation.
func TestHoldAndSettleOnTheSameAttemptDoNotDeadlock(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`)
	now := time.Now().UTC()
	if _, err := s.Grant(ctx, "u1", 100000*money.PerUSD, "admin", "g", "", now); err != nil {
		t.Fatal(err)
	}

	var mu sync.Mutex
	var deadlocks int
	note := func(err error) {
		if err != nil && strings.Contains(err.Error(), "deadlock") {
			mu.Lock()
			deadlocks++
			mu.Unlock()
		}
	}
	for i := range 150 {
		id := fmt.Sprintf("run-%d", i)
		if err := s.Hold(ctx, "u1", "bk1", id, money.PerUSD, now); err != nil {
			t.Fatal(err)
		}
		var wg sync.WaitGroup
		wg.Go(func() { note(s.Settle(ctx, id, money.PerUSD/2, now)) })
		wg.Go(func() { note(s.Hold(ctx, "u1", "bk1", id, money.PerUSD, now)) })
		wg.Wait()
	}

	a, err := s.ReadAccount(ctx, "u1")
	if err != nil {
		t.Fatal(err)
	}
	if a.Balance != a.LedgerSum {
		t.Fatalf("the cache drifted from the ledger: %s against %s", a.Balance.USD(), a.LedgerSum.USD())
	}
	if deadlocks > 0 {
		t.Fatalf("%d deadlocks in 150 rounds: the lock order is not the same in both paths", deadlocks)
	}
}

// PD-56. A typo in an account id is the commonest operator error, and every money entry point must
// name it the same way. Before this, Balance said "no such account" while Grant and Adjust returned
// the Postgres constraint name — which reads as a broken database, not a mistyped id.
// Mutation caught: dropping the constraint check in appendLedger.
func TestMoneyOperationsAgreeOnAMissingAccount(t *testing.T) {
	s, ctx := testDB(t)
	now := time.Now().UTC()
	grant := func() error { _, err := s.Grant(ctx, "no-such-user", money.PerUSD, "admin", "k1", "", now); return err }
	adjust := func() error {
		_, err := s.Adjust(ctx, "no-such-user", money.PerUSD, "admin", "k2", "why", now)
		return err
	}
	balance := func() error { _, err := s.Balance(ctx, "no-such-user"); return err }
	read := func() error { _, err := s.ReadAccount(ctx, "no-such-user"); return err }
	for name, call := range map[string]func() error{
		"grant": grant, "adjust": adjust, "balance": balance, "read account": read,
	} {
		t.Run(name, func(t *testing.T) {
			if err := call(); !errors.Is(err, ErrNoAccount) {
				t.Fatalf("got %v, want ErrNoAccount", err)
			}
		})
	}
}

// The row lock, not the comparison, is what stops two runs from spending the same credit. Each hold
// here is affordable on its own and they are not affordable together, so without FOR UPDATE both
// read the same balance, both pass the check, and the account goes negative — the cache and the
// ledger drifting together, which is why the invariant assertions elsewhere cannot see it either.
// Found by review: the sequential test above claimed this and could not deliver it.
// Mutation caught: dropping `for update` from lockBalance.
func TestConcurrentHoldsCannotOvercommitAnAccount(t *testing.T) {
	s, ctx := testDB(t)
	seedUser(t, s, ctx, "u1")
	exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
	                 values ('bk1','u1','x','zh','ru','not_started','/srv/books/bk1','x')`)
	now := time.Now().UTC()

	const rounds = 60
	for i := range rounds {
		user := fmt.Sprintf("u-%d", i)
		book := fmt.Sprintf("bk-%d", i)
		seedUser(t, s, ctx, user)
		exec(t, s, ctx, `insert into books (id, owner_id, title, source_lang, target_lang, status, workdir, engine_book_id)
		                 values ($1,$2,'x','zh','ru','not_started','/srv/books/x','x')`, book, user)
		// Ten dollars, and two runs that each want six.
		if _, err := s.Grant(ctx, user, 10*money.PerUSD, "admin", "g", "", now); err != nil {
			t.Fatal(err)
		}
		var wg sync.WaitGroup
		var mu sync.Mutex
		var granted int
		for j := range 2 {
			wg.Go(func() {
				err := s.Hold(ctx, user, book, fmt.Sprintf("run-%d-%d", i, j), 6*money.PerUSD, now)
				switch {
				case err == nil:
					mu.Lock()
					granted++
					mu.Unlock()
				case errors.Is(err, ErrInsufficientCredit):
				default:
					mu.Lock()
					t.Errorf("round %d: unexpected hold error: %v", i, err)
					mu.Unlock()
				}
			})
		}
		wg.Wait()

		a, err := s.ReadAccount(ctx, user)
		if err != nil {
			t.Fatal(err)
		}
		if granted != 1 {
			t.Fatalf("round %d: %d of two competing holds were granted from one balance; balance is now %s",
				i, granted, a.Balance.USD())
		}
		if a.Balance < 0 {
			t.Fatalf("round %d: balance went negative (%s): two runs spent the same credit", i, a.Balance.USD())
		}
		if a.Balance != a.LedgerSum {
			t.Fatalf("round %d: cache %s disagrees with the ledger %s", i, a.Balance.USD(), a.LedgerSum.USD())
		}
	}
}