textmachine/backend/internal/store/ledger.go

package store

import (
	"database/sql"
	"errors"
	"fmt"
	"time"
)

// ledger.go: reserve-before-call / settle-after money discipline (a port of
// the vojo Reserve/Settle/ReleaseReservation semantics onto SQLite). Ceilings are
// per-book (in total) and per-day (Р7); checked against committed + reserved under
// the write pool's immediate transaction, so that concurrent calls do not
// overshoot the ceiling by more than one maximum reservation.

// ReserveResult is the outcome of a pre-call admission check.
type ReserveResult int

const (
	ReserveOK         ReserveResult = iota
	ReserveDeniedBook               // per-book USD ceiling hit
	ReserveDeniedDay                // daily USD ceiling hit
)

// Reservation is the handle Settle/Release need to undo the admission.
type Reservation struct {
	BookID   string
	Date     string // UTC day the reservation was booked under
	Estimate float64
}

// Ceilings are the admission limits. Zero means "no limit" for that axis —
// fail-fast in config forbids an all-zero pair for real runs.
type Ceilings struct {
	BookUSD float64
	DayUSD  float64
}

func todayUTC() string { return time.Now().UTC().Format("2006-01-02") }

// Reserve books estimate USD against the ceilings BEFORE the call. On success
// the estimate is added to reserved_usd; Settle converts it to committed
// spend, Release returns it. Denials are results, not errors.
func (s *Store) Reserve(bookID string, estimate float64, c Ceilings) (Reservation, ReserveResult, error) {
	ctx, cancel := opContext()
	defer cancel()
	day := todayUTC()
	res := Reservation{BookID: bookID, Date: day, Estimate: estimate}

	tx, err := s.w.BeginTx(ctx, nil) // write pool: immediate tx, serialized
	if err != nil {
		return res, ReserveOK, err
	}
	defer tx.Rollback()

	var bookTotal, dayTotal float64
	if err := tx.QueryRowContext(ctx,
		`SELECT COALESCE(SUM(committed_usd + reserved_usd), 0) FROM spend WHERE book_id = ?`, bookID,
	).Scan(&bookTotal); err != nil {
		return res, ReserveOK, err
	}
	if err := tx.QueryRowContext(ctx,
		`SELECT COALESCE(SUM(committed_usd + reserved_usd), 0) FROM spend WHERE date = ?`, day,
	).Scan(&dayTotal); err != nil {
		return res, ReserveOK, err
	}
	if c.BookUSD > 0 && bookTotal+estimate > c.BookUSD {
		return res, ReserveDeniedBook, nil
	}
	if c.DayUSD > 0 && dayTotal+estimate > c.DayUSD {
		return res, ReserveDeniedDay, nil
	}

	if _, err := tx.ExecContext(ctx,
		`INSERT INTO spend (book_id, date, reserved_usd) VALUES (?, ?, ?)
		 ON CONFLICT (book_id, date) DO UPDATE SET reserved_usd = reserved_usd + excluded.reserved_usd`,
		bookID, day, estimate); err != nil {
		return res, ReserveOK, err
	}
	if err := tx.Commit(); err != nil {
		return res, ReserveOK, err
	}
	return res, ReserveOK, nil
}

// Release frees a reservation whose call produced no billable spend (transport
// exhaustion, terminal 4xx before a 2xx). MAX(0, …) guards a double-release.
func (s *Store) Release(res Reservation) error {
	ctx, cancel := opContext()
	defer cancel()
	_, err := s.w.ExecContext(ctx,
		`UPDATE spend SET reserved_usd = MAX(0, reserved_usd - ?) WHERE book_id = ? AND date = ?`,
		res.Estimate, res.BookID, res.Date)
	return err
}

// RepairSpentUSD sums the settled cost of a book's REPAIR calls (pack-16) — the money figure the runner
// gates against gates.repair.budget_usd. It keys on the checkpoint ROLE rather than on a new column: a
// repair call is addressed under its own synthetic role, so the existing schema already separates this
// spend class (no migration). Derived checkpoints are written with cost 0, so they add nothing to the sum.
func (s *Store) RepairSpentUSD(bookID string) (float64, error) {
	return s.RoleSpentUSD(bookID, "repair")
}

// RoleSpentUSD sums the settled cost of a book's calls made under one synthetic ROLE — the budget figure
// every sub-step class gates against (repair, terminology). It keys on the checkpoint role rather than on
// a per-class column, so a new call class needs no migration to become countable; derived checkpoints are
// written with cost 0 and therefore add nothing.
func (s *Store) RoleSpentUSD(bookID, role string) (float64, error) {
	ctx, cancel := opContext()
	defer cancel()
	var sum float64
	err := s.r.QueryRowContext(ctx, `
		SELECT COALESCE(SUM(c.cost_usd), 0)
		FROM checkpoints c JOIN jobs j ON c.job_id = j.id
		WHERE j.book_id = ? AND c.role = ?`, bookID, role).Scan(&sum)
	return sum, err
}

// RepairStats counts the book's repair OUTCOMES from durable artifacts alone — no counter column exists and
// none is needed (pack-16 §15.2 B): a per-run counter on retrieval_state would be zeroed by the draft wave's
// unconditional rewrite on every resumed run, whereas the checkpoints and the derived-export namespace
// survive. Calls = repair-role checkpoints; Declined = those whose reply is the no-change sentinel; Applied =
// units whose final_hash points at a repair export. Deterministic and self-healing on resume.
func (s *Store) RepairStats(bookID, noChangeSentinel, derivedPrefix string) (calls, declined, applied int, err error) {
	ctx, cancel := opContext()
	defer cancel()
	err = s.r.QueryRowContext(ctx, `
		SELECT COUNT(*), COALESCE(SUM(CASE WHEN TRIM(c.response_text) = ? THEN 1 ELSE 0 END), 0)
		FROM checkpoints c JOIN jobs j ON c.job_id = j.id
		WHERE j.book_id = ? AND c.role = ?`, noChangeSentinel, bookID, "repair").Scan(&calls, &declined)
	if err != nil {
		return 0, 0, 0, err
	}
	err = s.r.QueryRowContext(ctx, `
		SELECT COUNT(*) FROM chunk_status WHERE book_id = ? AND final_hash LIKE ?`,
		bookID, derivedPrefix+"%").Scan(&applied)
	return calls, declined, applied, err
}

// Checkpoint is one persisted raw LLM response (see migrate.go v1).
type Checkpoint struct {
	RequestHash       string
	JobID             int64
	ChunkIdx          int
	Attempt           int
	Stage             string
	Role              string
	ModelRequested    string
	ModelActual       string
	ResponseText      string
	UsageJSON         string
	CostUSD           float64
	FinishReason      string
	ProviderRequestID string
	// Escalation marks a single-hop fallback-draft checkpoint (D12): summed against
	// escalation.budget_usd, independent of the primary translation spend.
	Escalation bool
}

// SettleWithCheckpoint atomically (ONE transaction, one file) converts the
// reservation into committed spend AND persists the raw response. This
// closes the hole «settle went through — kill -9 — checkpoint not written =
// double payment» (implementation-notes §3.3): after a restart there is either both the
// debit and the checkpoint (the call won't repeat), or neither (recovery
// releases the reservation, the call repeats and is paid for exactly once). The only
// unrecoverable loss is the in-flight call, which is precisely the honest «≤1 call» acceptance.
//
// Idempotent per request_hash: a duplicate settle for the same hash books
// nothing and keeps the original checkpoint.
//
// `spend` is the run-event seam (row 103) joining this transaction: it renders the cumulative-spend
// journal line from the sequence number the outbox assigns here and the book's new committed total, read
// after the debit and INSIDE this transaction. That is the ratified formula — the event is an outbox
// projection of a committed row, not a second write — and this is the one place it is load-bearing: a
// journal line claiming money the ledger never booked is a divergence about money, not about a progress
// bar. nil disables it (every read-only path and every $0 test).
func (s *Store) SettleWithCheckpoint(res Reservation, cost float64, cp Checkpoint, spend *SpendLine) error {
	ctx, cancel := opContext()
	defer cancel()

	tx, err := s.w.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback()

	ins, err := tx.ExecContext(ctx, `
		INSERT INTO checkpoints (
			request_hash, job_id, chunk_idx, attempt, stage, role,
			model_requested, model_actual, response_text, usage_json,
			cost_usd, finish_reason, provider_request_id, escalation
		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
		ON CONFLICT (request_hash) DO NOTHING`,
		cp.RequestHash, cp.JobID, cp.ChunkIdx, cp.Attempt, cp.Stage, cp.Role,
		cp.ModelRequested, cp.ModelActual, cp.ResponseText, cp.UsageJSON,
		cp.CostUSD, cp.FinishReason, cp.ProviderRequestID, cp.Escalation)
	if err != nil {
		return fmt.Errorf("store: checkpoint insert: %w", err)
	}
	inserted, _ := ins.RowsAffected()
	if inserted == 0 {
		// Duplicate settle (retried caller): release the reservation, book no
		// new spend — the original settle already did.
		if _, err := tx.ExecContext(ctx,
			`UPDATE spend SET reserved_usd = MAX(0, reserved_usd - ?) WHERE book_id = ? AND date = ?`,
			res.Estimate, res.BookID, res.Date); err != nil {
			return err
		}
		return tx.Commit()
	}

	if _, err := tx.ExecContext(ctx, `
		INSERT INTO spend (book_id, date, committed_usd, reserved_usd) VALUES (?, ?, ?, 0)
		ON CONFLICT (book_id, date) DO UPDATE SET
			committed_usd = committed_usd + excluded.committed_usd,
			reserved_usd  = MAX(0, reserved_usd - ?)`,
		res.BookID, res.Date, cost, res.Estimate); err != nil {
		return fmt.Errorf("store: settle spend: %w", err)
	}
	if spend != nil {
		var committed float64
		if err := tx.QueryRowContext(ctx,
			`SELECT COALESCE(SUM(committed_usd), 0) FROM spend WHERE book_id = ?`, res.BookID).Scan(&committed); err != nil {
			return fmt.Errorf("store: read committed for the spend event: %w", err)
		}
		if _, err := enqueueEvent(ctx, tx, spend.RunID, func(seq int64) ([]byte, error) {
			return spend.Line(seq, committed)
		}); err != nil {
			return err
		}
	}
	return tx.Commit()
}

// SpendLine is how the driver puts its cumulative-spend event into the settle transaction: RunID names
// the outbox stream (the process's engine run id) and Line renders the journal line from the sequence
// number the outbox assigned and the book's new committed total. The store stays dumb storage — it
// never learns what an event means, exactly as it never learns what a `disposition` string means.
type SpendLine struct {
	RunID string
	Line  func(seq int64, committedUSD float64) ([]byte, error)
}

// PutDerivedCheckpoint persists a $0 DERIVED checkpoint — a deterministic post-processing
// artifact, NOT a billed provider response (D38 infra-pack): the output-sanitizer's cosmetic
// strip commits the cleaned final text here so the standard final_hash→checkpoint.response_text
// export contract (records.json / exp12_extract) yields the cleaned text for a flagged chunk that
// would otherwise export empty (D35.4a). It touches NO spend/reservation (cost must be 0), keyed
// by its own derived request_hash (namespaced, collision-free with real attempt hashes), and is
// idempotent (ON CONFLICT DO NOTHING) so a re-run re-derives the identical row for free. Escalation
// is deliberately false so it never counts toward escalation.budget_usd. It is deleted with its
// stage's real checkpoints on `redrive` (ResetChunkStages joins by job/chunk/stage), never orphaned.
func (s *Store) PutDerivedCheckpoint(cp Checkpoint) error {
	ctx, cancel := opContext()
	defer cancel()
	_, err := s.w.ExecContext(ctx, `
		INSERT INTO checkpoints (
			request_hash, job_id, chunk_idx, attempt, stage, role,
			model_requested, model_actual, response_text, usage_json,
			cost_usd, finish_reason, provider_request_id, escalation
		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, '', 0)
		ON CONFLICT (request_hash) DO NOTHING`,
		cp.RequestHash, cp.JobID, cp.ChunkIdx, cp.Attempt, cp.Stage, cp.Role,
		cp.ModelRequested, cp.ModelActual, cp.ResponseText, cp.UsageJSON, cp.FinishReason)
	if err != nil {
		return fmt.Errorf("store: derived checkpoint insert: %w", err)
	}
	return nil
}

// GetCheckpoint returns the persisted response for a request hash, if any —
// the resume path: a hit means the call is NOT repeated or re-billed.
func (s *Store) GetCheckpoint(requestHash string) (*Checkpoint, error) {
	ctx, cancel := opContext()
	defer cancel()
	cp := Checkpoint{RequestHash: requestHash}
	err := s.r.QueryRowContext(ctx, `
		SELECT job_id, chunk_idx, attempt, stage, role, model_requested, model_actual,
		       response_text, usage_json, cost_usd, finish_reason, provider_request_id, escalation
		FROM checkpoints WHERE request_hash = ?`, requestHash).Scan(
		&cp.JobID, &cp.ChunkIdx, &cp.Attempt, &cp.Stage, &cp.Role, &cp.ModelRequested, &cp.ModelActual,
		&cp.ResponseText, &cp.UsageJSON, &cp.CostUSD, &cp.FinishReason, &cp.ProviderRequestID, &cp.Escalation)
	if errors.Is(err, sql.ErrNoRows) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	return &cp, nil
}

// RoleResponse is one persisted model answer with the chunk coordinates it was produced for — enough to
// re-derive anything the raw text carried, and nothing more (no usage, no money, no hashes).
type RoleResponse struct {
	Chapter      int
	ChunkIdx     int
	Attempt      int
	ResponseText string
	FinishReason string
}

// RoleResponsesForBook returns EVERY persisted answer a role ever produced for a book, oldest job first —
// including the attempts a later run superseded.
//
// It exists for the banknote union. The per-chunk telemetry row is keyed (book, chapter, chunk) and
// UPSERTED, so when a chunk is genuinely re-drafted the new proposals overwrite the old ones — and that
// channel was measured to propose a substantially different set on every sampling, so the overwrite loses
// real coverage. Checkpoints keep every answer the book ever paid for, addressed by its own request hash,
// so the union is DERIVED rather than stored: no new table, no migration, self-healing like the row it
// supplements.
//
// It returns the $0 DERIVED rows too (the stripped export, the sanitized export), deliberately: they carry
// a non-provider finish_reason, so the caller's ordinary "trust a complete generation only" gate already
// excludes them, and filtering them here would put a second, silently drifting copy of that rule in SQL.
//
// mustContain is an optional VOLUME filter on the response text (empty = no filter). A book's whole
// answer history is megabytes and grows with every re-purchase, so a caller that only cares about answers
// carrying a marker passes it here instead of reading everything into memory. It is never a semantic
// rule — the caller still parses whatever it gets — and the marker stays a Go constant, passed as a
// value, so SQL holds no second copy of it.
//
// Ordering is (job, chunk, attempt) so a fold over the result is deterministic without sorting in Go.
func (s *Store) RoleResponsesForBook(bookID, role, mustContain string) ([]RoleResponse, error) {
	q := `SELECT j.chapter, c.chunk_idx, c.attempt, c.response_text, c.finish_reason
	      FROM checkpoints c JOIN jobs j ON j.id = c.job_id
	      WHERE j.book_id = ? AND c.role = ?`
	args := []any{bookID, role}
	if mustContain != "" {
		q += ` AND instr(c.response_text, ?) > 0`
		args = append(args, mustContain)
	}
	q += ` ORDER BY j.id, c.chunk_idx, c.attempt`
	return queryAll(s.r, q,
		func(rows *sql.Rows) (RoleResponse, error) {
			var r RoleResponse
			err := rows.Scan(&r.Chapter, &r.ChunkIdx, &r.Attempt, &r.ResponseText, &r.FinishReason)
			return r, err
		}, args...)
}

// CheckpointUsage is one billed call's TOKENS and routing, addressed the way chunk_status is —
// (chapter, chunk, stage) — rather than by request_hash.
//
// It is the bridge a re-payment projection needs (row 181). chunk_status carries only a summed
// cost_usd in the currency of the day it was billed, and final_hash reaches a single checkpoint only on
// the ok path, so a flagged row has no route back to its own usage at all. The job join has one: jobs are
// (book, chapter, stage) and checkpoints carry chunk_idx, which is exactly the chunk_status key, for
// every attempt of every disposition.
type CheckpointUsage struct {
	Chapter        int
	ChunkIdx       int
	Stage          string
	ModelRequested string
	ModelActual    string
	UsageJSON      string
	CostUSD        float64 // what it was billed AT THE TIME — the historical figure, kept for the fallback
}

// CheckpointUsageForBook returns every checkpoint's usage for a book in INSERTION order. The order is
// load-bearing, not cosmetic: the table is append-only for the life of the book, so a position
// accumulates one call per re-purchase, and the newest can be told from a superseded one only by when it
// was written. `attempt` cannot do it — it restarts at 0 every run — so the ordering rides rowid.
//
// response_text is deliberately NOT selected: it is the megabytes of the table and no pricing question
// needs a byte of it.
func (s *Store) CheckpointUsageForBook(bookID string) ([]CheckpointUsage, error) {
	return queryAll(s.r, `
		SELECT j.chapter, c.chunk_idx, c.stage, c.model_requested, c.model_actual, c.usage_json, c.cost_usd
		FROM checkpoints c JOIN jobs j ON j.id = c.job_id
		WHERE j.book_id = ?
		ORDER BY c.rowid`,
		func(rows *sql.Rows) (CheckpointUsage, error) {
			var u CheckpointUsage
			err := rows.Scan(&u.Chapter, &u.ChunkIdx, &u.Stage, &u.ModelRequested, &u.ModelActual,
				&u.UsageJSON, &u.CostUSD)
			return u, err
		}, bookID)
}

// SpentUSD reports (committed, reserved) for a book across all days.
func (s *Store) SpentUSD(bookID string) (committed, reserved float64, err error) {
	ctx, cancel := opContext()
	defer cancel()
	err = s.r.QueryRowContext(ctx,
		`SELECT COALESCE(SUM(committed_usd),0), COALESCE(SUM(reserved_usd),0) FROM spend WHERE book_id = ?`,
		bookID).Scan(&committed, &reserved)
	return
}

// EscalationSpentUSD sums the settled cost of a book's escalation (fallback-draft)
// checkpoints — the money-path figure the runner gates against escalation.budget_usd
// (D12). It joins checkpoints→jobs by book_id and counts only escalation=1 rows, so
// it is durable across resume and never double-counts a checkpoint-hit (a resumed
// hop re-serves its checkpoint without a new settle).
// EscalationHops counts the escalation CALLS a book has paid for — the hop count the per-unit boolean
// (chunk_status.escalated) cannot express. Derived from the durable checkpoints, so it survives a resume
// and needs no schema change (the pack-16 precedent: a counter column on a rewritten read-model row
// would be zeroed by the next run, while checkpoints do not lie). Derived $0 checkpoints carry cost 0 but
// are never written with escalation=1, so they cannot inflate this.
func (s *Store) EscalationHops(bookID string) (int, error) {
	ctx, cancel := opContext()
	defer cancel()
	var n int
	err := s.r.QueryRowContext(ctx, `
		SELECT COUNT(*)
		FROM checkpoints c JOIN jobs j ON c.job_id = j.id
		WHERE j.book_id = ? AND c.escalation = 1`, bookID).Scan(&n)
	return n, err
}

// SpendByModel attributes a book's committed spend per REQUESTED model — the cheapest honest answer to
// "what did the label-routed endpoint cost", with no migration and no new call class: the model slug is
// already on every checkpoint, and which models a label routes to is knowable from the config. Keyed on
// model_requested (not model_actual) so a provider that canonicalises its slug in the response does not
// split one line of spend in two.
func (s *Store) SpendByModel(bookID string) (map[string]float64, error) {
	ctx, cancel := opContext()
	defer cancel()
	rows, err := s.r.QueryContext(ctx, `
		SELECT c.model_requested, COALESCE(SUM(c.cost_usd), 0)
		FROM checkpoints c JOIN jobs j ON c.job_id = j.id
		WHERE j.book_id = ?
		GROUP BY c.model_requested`, bookID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	out := map[string]float64{}
	for rows.Next() {
		var model string
		var sum float64
		if err := rows.Scan(&model, &sum); err != nil {
			return nil, err
		}
		if sum > 0 { // a $0 derived checkpoint is not a spend line
			out[model] = sum
		}
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	if len(out) == 0 {
		return nil, nil
	}
	return out, nil
}

func (s *Store) EscalationSpentUSD(bookID string) (float64, error) {
	ctx, cancel := opContext()
	defer cancel()
	var sum float64
	err := s.r.QueryRowContext(ctx, `
		SELECT COALESCE(SUM(c.cost_usd), 0)
		FROM checkpoints c JOIN jobs j ON c.job_id = j.id
		WHERE j.book_id = ? AND c.escalation = 1`, bookID).Scan(&sum)
	return sum, err
}