package store import ( "database/sql" "errors" "fmt" "time" ) // ledger.go: reserve-before-call / settle-after money discipline (a port of // the vojo Reserve/Settle/ReleaseReservation semantics onto SQLite). Ceilings are // per-book (in total) and per-day (Р7); checked against committed + reserved under // the write pool's immediate transaction, so that concurrent calls do not // overshoot the ceiling by more than one maximum reservation. // ReserveResult is the outcome of a pre-call admission check. type ReserveResult int const ( ReserveOK ReserveResult = iota ReserveDeniedBook // per-book USD ceiling hit ReserveDeniedDay // daily USD ceiling hit ) // Reservation is the handle Settle/Release need to undo the admission. type Reservation struct { BookID string Date string // UTC day the reservation was booked under Estimate float64 } // Ceilings are the admission limits. Zero means "no limit" for that axis — // fail-fast in config forbids an all-zero pair for real runs. type Ceilings struct { BookUSD float64 DayUSD float64 } func todayUTC() string { return time.Now().UTC().Format("2006-01-02") } // Reserve books estimate USD against the ceilings BEFORE the call. On success // the estimate is added to reserved_usd; Settle converts it to committed // spend, Release returns it. Denials are results, not errors. func (s *Store) Reserve(bookID string, estimate float64, c Ceilings) (Reservation, ReserveResult, error) { ctx, cancel := opContext() defer cancel() day := todayUTC() res := Reservation{BookID: bookID, Date: day, Estimate: estimate} tx, err := s.w.BeginTx(ctx, nil) // write pool: immediate tx, serialized if err != nil { return res, ReserveOK, err } defer tx.Rollback() var bookTotal, dayTotal float64 if err := tx.QueryRowContext(ctx, `SELECT COALESCE(SUM(committed_usd + reserved_usd), 0) FROM spend WHERE book_id = ?`, bookID, ).Scan(&bookTotal); err != nil { return res, ReserveOK, err } if err := tx.QueryRowContext(ctx, `SELECT COALESCE(SUM(committed_usd + reserved_usd), 0) FROM spend WHERE date = ?`, day, ).Scan(&dayTotal); err != nil { return res, ReserveOK, err } if c.BookUSD > 0 && bookTotal+estimate > c.BookUSD { return res, ReserveDeniedBook, nil } if c.DayUSD > 0 && dayTotal+estimate > c.DayUSD { return res, ReserveDeniedDay, nil } if _, err := tx.ExecContext(ctx, `INSERT INTO spend (book_id, date, reserved_usd) VALUES (?, ?, ?) ON CONFLICT (book_id, date) DO UPDATE SET reserved_usd = reserved_usd + excluded.reserved_usd`, bookID, day, estimate); err != nil { return res, ReserveOK, err } if err := tx.Commit(); err != nil { return res, ReserveOK, err } return res, ReserveOK, nil } // Release frees a reservation whose call produced no billable spend (transport // exhaustion, terminal 4xx before a 2xx). MAX(0, …) guards a double-release. func (s *Store) Release(res Reservation) error { ctx, cancel := opContext() defer cancel() _, err := s.w.ExecContext(ctx, `UPDATE spend SET reserved_usd = MAX(0, reserved_usd - ?) WHERE book_id = ? AND date = ?`, res.Estimate, res.BookID, res.Date) return err } // RepairSpentUSD sums the settled cost of a book's REPAIR calls (pack-16) — the money figure the runner // gates against gates.repair.budget_usd. It keys on the checkpoint ROLE rather than on a new column: a // repair call is addressed under its own synthetic role, so the existing schema already separates this // spend class (no migration). Derived checkpoints are written with cost 0, so they add nothing to the sum. func (s *Store) RepairSpentUSD(bookID string) (float64, error) { return s.RoleSpentUSD(bookID, "repair") } // RoleSpentUSD sums the settled cost of a book's calls made under one synthetic ROLE — the budget figure // every sub-step class gates against (repair, terminology). It keys on the checkpoint role rather than on // a per-class column, so a new call class needs no migration to become countable; derived checkpoints are // written with cost 0 and therefore add nothing. func (s *Store) RoleSpentUSD(bookID, role string) (float64, error) { ctx, cancel := opContext() defer cancel() var sum float64 err := s.r.QueryRowContext(ctx, ` SELECT COALESCE(SUM(c.cost_usd), 0) FROM checkpoints c JOIN jobs j ON c.job_id = j.id WHERE j.book_id = ? AND c.role = ?`, bookID, role).Scan(&sum) return sum, err } // RepairStats counts the book's repair OUTCOMES from durable artifacts alone — no counter column exists and // none is needed (pack-16 §15.2 B): a per-run counter on retrieval_state would be zeroed by the draft wave's // unconditional rewrite on every resumed run, whereas the checkpoints and the derived-export namespace // survive. Calls = repair-role checkpoints; Declined = those whose reply is the no-change sentinel; Applied = // units whose final_hash points at a repair export. Deterministic and self-healing on resume. func (s *Store) RepairStats(bookID, noChangeSentinel, derivedPrefix string) (calls, declined, applied int, err error) { ctx, cancel := opContext() defer cancel() err = s.r.QueryRowContext(ctx, ` SELECT COUNT(*), COALESCE(SUM(CASE WHEN TRIM(c.response_text) = ? THEN 1 ELSE 0 END), 0) FROM checkpoints c JOIN jobs j ON c.job_id = j.id WHERE j.book_id = ? AND c.role = ?`, noChangeSentinel, bookID, "repair").Scan(&calls, &declined) if err != nil { return 0, 0, 0, err } err = s.r.QueryRowContext(ctx, ` SELECT COUNT(*) FROM chunk_status WHERE book_id = ? AND final_hash LIKE ?`, bookID, derivedPrefix+"%").Scan(&applied) return calls, declined, applied, err } // Checkpoint is one persisted raw LLM response (see migrate.go v1). type Checkpoint struct { RequestHash string JobID int64 ChunkIdx int Attempt int Stage string Role string ModelRequested string ModelActual string ResponseText string UsageJSON string CostUSD float64 FinishReason string ProviderRequestID string // Escalation marks a single-hop fallback-draft checkpoint (D12): summed against // escalation.budget_usd, independent of the primary translation spend. Escalation bool } // SettleWithCheckpoint atomically (ONE transaction, one file) converts the // reservation into committed spend AND persists the raw response. This // closes the hole «settle went through — kill -9 — checkpoint not written = // double payment» (implementation-notes §3.3): after a restart there is either both the // debit and the checkpoint (the call won't repeat), or neither (recovery // releases the reservation, the call repeats and is paid for exactly once). The only // unrecoverable loss is the in-flight call, which is precisely the honest «≤1 call» acceptance. // // Idempotent per request_hash: a duplicate settle for the same hash books // nothing and keeps the original checkpoint. // // `spend` is the run-event seam (row 103) joining this transaction: it renders the cumulative-spend // journal line from the sequence number the outbox assigns here and the book's new committed total, read // after the debit and INSIDE this transaction. That is the ratified formula — the event is an outbox // projection of a committed row, not a second write — and this is the one place it is load-bearing: a // journal line claiming money the ledger never booked is a divergence about money, not about a progress // bar. nil disables it (every read-only path and every $0 test). func (s *Store) SettleWithCheckpoint(res Reservation, cost float64, cp Checkpoint, spend *SpendLine) error { ctx, cancel := opContext() defer cancel() tx, err := s.w.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() ins, err := tx.ExecContext(ctx, ` INSERT INTO checkpoints ( request_hash, job_id, chunk_idx, attempt, stage, role, model_requested, model_actual, response_text, usage_json, cost_usd, finish_reason, provider_request_id, escalation ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (request_hash) DO NOTHING`, cp.RequestHash, cp.JobID, cp.ChunkIdx, cp.Attempt, cp.Stage, cp.Role, cp.ModelRequested, cp.ModelActual, cp.ResponseText, cp.UsageJSON, cp.CostUSD, cp.FinishReason, cp.ProviderRequestID, cp.Escalation) if err != nil { return fmt.Errorf("store: checkpoint insert: %w", err) } inserted, _ := ins.RowsAffected() if inserted == 0 { // Duplicate settle (retried caller): release the reservation, book no // new spend — the original settle already did. if _, err := tx.ExecContext(ctx, `UPDATE spend SET reserved_usd = MAX(0, reserved_usd - ?) WHERE book_id = ? AND date = ?`, res.Estimate, res.BookID, res.Date); err != nil { return err } return tx.Commit() } if _, err := tx.ExecContext(ctx, ` INSERT INTO spend (book_id, date, committed_usd, reserved_usd) VALUES (?, ?, ?, 0) ON CONFLICT (book_id, date) DO UPDATE SET committed_usd = committed_usd + excluded.committed_usd, reserved_usd = MAX(0, reserved_usd - ?)`, res.BookID, res.Date, cost, res.Estimate); err != nil { return fmt.Errorf("store: settle spend: %w", err) } if spend != nil { var committed float64 if err := tx.QueryRowContext(ctx, `SELECT COALESCE(SUM(committed_usd), 0) FROM spend WHERE book_id = ?`, res.BookID).Scan(&committed); err != nil { return fmt.Errorf("store: read committed for the spend event: %w", err) } if _, err := enqueueEvent(ctx, tx, spend.RunID, func(seq int64) ([]byte, error) { return spend.Line(seq, committed) }); err != nil { return err } } return tx.Commit() } // SpendLine is how the driver puts its cumulative-spend event into the settle transaction: RunID names // the outbox stream (the process's engine run id) and Line renders the journal line from the sequence // number the outbox assigned and the book's new committed total. The store stays dumb storage — it // never learns what an event means, exactly as it never learns what a `disposition` string means. type SpendLine struct { RunID string Line func(seq int64, committedUSD float64) ([]byte, error) } // PutDerivedCheckpoint persists a $0 DERIVED checkpoint — a deterministic post-processing // artifact, NOT a billed provider response (D38 infra-pack): the output-sanitizer's cosmetic // strip commits the cleaned final text here so the standard final_hash→checkpoint.response_text // export contract (records.json / exp12_extract) yields the cleaned text for a flagged chunk that // would otherwise export empty (D35.4a). It touches NO spend/reservation (cost must be 0), keyed // by its own derived request_hash (namespaced, collision-free with real attempt hashes), and is // idempotent (ON CONFLICT DO NOTHING) so a re-run re-derives the identical row for free. Escalation // is deliberately false so it never counts toward escalation.budget_usd. It is deleted with its // stage's real checkpoints on `redrive` (ResetChunkStages joins by job/chunk/stage), never orphaned. func (s *Store) PutDerivedCheckpoint(cp Checkpoint) error { ctx, cancel := opContext() defer cancel() _, err := s.w.ExecContext(ctx, ` INSERT INTO checkpoints ( request_hash, job_id, chunk_idx, attempt, stage, role, model_requested, model_actual, response_text, usage_json, cost_usd, finish_reason, provider_request_id, escalation ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, '', 0) ON CONFLICT (request_hash) DO NOTHING`, cp.RequestHash, cp.JobID, cp.ChunkIdx, cp.Attempt, cp.Stage, cp.Role, cp.ModelRequested, cp.ModelActual, cp.ResponseText, cp.UsageJSON, cp.FinishReason) if err != nil { return fmt.Errorf("store: derived checkpoint insert: %w", err) } return nil } // GetCheckpoint returns the persisted response for a request hash, if any — // the resume path: a hit means the call is NOT repeated or re-billed. func (s *Store) GetCheckpoint(requestHash string) (*Checkpoint, error) { ctx, cancel := opContext() defer cancel() cp := Checkpoint{RequestHash: requestHash} err := s.r.QueryRowContext(ctx, ` SELECT job_id, chunk_idx, attempt, stage, role, model_requested, model_actual, response_text, usage_json, cost_usd, finish_reason, provider_request_id, escalation FROM checkpoints WHERE request_hash = ?`, requestHash).Scan( &cp.JobID, &cp.ChunkIdx, &cp.Attempt, &cp.Stage, &cp.Role, &cp.ModelRequested, &cp.ModelActual, &cp.ResponseText, &cp.UsageJSON, &cp.CostUSD, &cp.FinishReason, &cp.ProviderRequestID, &cp.Escalation) if errors.Is(err, sql.ErrNoRows) { return nil, nil } if err != nil { return nil, err } return &cp, nil } // RoleResponse is one persisted model answer with the chunk coordinates it was produced for — enough to // re-derive anything the raw text carried, and nothing more (no usage, no money, no hashes). type RoleResponse struct { Chapter int ChunkIdx int Attempt int ResponseText string FinishReason string } // RoleResponsesForBook returns EVERY persisted answer a role ever produced for a book, oldest job first — // including the attempts a later run superseded. // // It exists for the banknote union. The per-chunk telemetry row is keyed (book, chapter, chunk) and // UPSERTED, so when a chunk is genuinely re-drafted the new proposals overwrite the old ones — and that // channel was measured to propose a substantially different set on every sampling, so the overwrite loses // real coverage. Checkpoints keep every answer the book ever paid for, addressed by its own request hash, // so the union is DERIVED rather than stored: no new table, no migration, self-healing like the row it // supplements. // // It returns the $0 DERIVED rows too (the stripped export, the sanitized export), deliberately: they carry // a non-provider finish_reason, so the caller's ordinary "trust a complete generation only" gate already // excludes them, and filtering them here would put a second, silently drifting copy of that rule in SQL. // // mustContain is an optional VOLUME filter on the response text (empty = no filter). A book's whole // answer history is megabytes and grows with every re-purchase, so a caller that only cares about answers // carrying a marker passes it here instead of reading everything into memory. It is never a semantic // rule — the caller still parses whatever it gets — and the marker stays a Go constant, passed as a // value, so SQL holds no second copy of it. // // Ordering is (job, chunk, attempt) so a fold over the result is deterministic without sorting in Go. func (s *Store) RoleResponsesForBook(bookID, role, mustContain string) ([]RoleResponse, error) { q := `SELECT j.chapter, c.chunk_idx, c.attempt, c.response_text, c.finish_reason FROM checkpoints c JOIN jobs j ON j.id = c.job_id WHERE j.book_id = ? AND c.role = ?` args := []any{bookID, role} if mustContain != "" { q += ` AND instr(c.response_text, ?) > 0` args = append(args, mustContain) } q += ` ORDER BY j.id, c.chunk_idx, c.attempt` return queryAll(s.r, q, func(rows *sql.Rows) (RoleResponse, error) { var r RoleResponse err := rows.Scan(&r.Chapter, &r.ChunkIdx, &r.Attempt, &r.ResponseText, &r.FinishReason) return r, err }, args...) } // CheckpointUsage is one billed call's TOKENS and routing, addressed the way chunk_status is — // (chapter, chunk, stage) — rather than by request_hash. // // It is the bridge a re-payment projection needs (row 181). chunk_status carries only a summed // cost_usd in the currency of the day it was billed, and final_hash reaches a single checkpoint only on // the ok path, so a flagged row has no route back to its own usage at all. The job join has one: jobs are // (book, chapter, stage) and checkpoints carry chunk_idx, which is exactly the chunk_status key, for // every attempt of every disposition. type CheckpointUsage struct { Chapter int ChunkIdx int Stage string ModelRequested string ModelActual string UsageJSON string CostUSD float64 // what it was billed AT THE TIME — the historical figure, kept for the fallback // Escalation is true for the single-hop fallback call of a stage: its tokens went to the stage's // escalate_to model, not to the stage's own, so «the model this stage resolves to now» is the current // HOP for this row (pipeline/reprice.go). Without it a re-pricing would compare the hop's slug against // the primary's and call every escalated row a routing move. Escalation bool } // CheckpointUsageForBook returns every checkpoint's usage for a book in INSERTION order. The order is // load-bearing, not cosmetic: the table is append-only for the life of the book, so a position // accumulates one call per re-purchase, and the newest can be told from a superseded one only by when it // was written. `attempt` cannot do it — it restarts at 0 every run — so the ordering rides rowid. // // The book filter is the only thing between one book's consent number and another book's money: a // project database may be shared by two books (pipeline/events.go, unitOnceKey), their positions // coincide, and the re-pricing keys calls by position alone. Pinned by TestCheckpointUsageForBookIsScopedToTheBook // and catalogued (cmd/tmmutate, FC1-book-filter-store and FC1-book-filter-pipeline). // // response_text is deliberately NOT selected: it is the megabytes of the table and no pricing question // needs a byte of it. func (s *Store) CheckpointUsageForBook(bookID string) ([]CheckpointUsage, error) { return queryAll(s.r, ` SELECT j.chapter, c.chunk_idx, c.stage, c.model_requested, c.model_actual, c.usage_json, c.cost_usd, c.escalation FROM checkpoints c JOIN jobs j ON j.id = c.job_id WHERE j.book_id = ? ORDER BY c.rowid`, func(rows *sql.Rows) (CheckpointUsage, error) { var u CheckpointUsage err := rows.Scan(&u.Chapter, &u.ChunkIdx, &u.Stage, &u.ModelRequested, &u.ModelActual, &u.UsageJSON, &u.CostUSD, &u.Escalation) return u, err }, bookID) } // SpentUSD reports (committed, reserved) for a book across all days. func (s *Store) SpentUSD(bookID string) (committed, reserved float64, err error) { ctx, cancel := opContext() defer cancel() err = s.r.QueryRowContext(ctx, `SELECT COALESCE(SUM(committed_usd),0), COALESCE(SUM(reserved_usd),0) FROM spend WHERE book_id = ?`, bookID).Scan(&committed, &reserved) return } // EscalationSpentUSD sums the settled cost of a book's escalation (fallback-draft) // checkpoints — the money-path figure the runner gates against escalation.budget_usd // (D12). It joins checkpoints→jobs by book_id and counts only escalation=1 rows, so // it is durable across resume and never double-counts a checkpoint-hit (a resumed // hop re-serves its checkpoint without a new settle). // EscalationHops counts the escalation CALLS a book has paid for — the hop count the per-unit boolean // (chunk_status.escalated) cannot express. Derived from the durable checkpoints, so it survives a resume // and needs no schema change (the pack-16 precedent: a counter column on a rewritten read-model row // would be zeroed by the next run, while checkpoints do not lie). Derived $0 checkpoints carry cost 0 but // are never written with escalation=1, so they cannot inflate this. func (s *Store) EscalationHops(bookID string) (int, error) { ctx, cancel := opContext() defer cancel() var n int err := s.r.QueryRowContext(ctx, ` SELECT COUNT(*) FROM checkpoints c JOIN jobs j ON c.job_id = j.id WHERE j.book_id = ? AND c.escalation = 1`, bookID).Scan(&n) return n, err } // SpendByModel attributes a book's committed spend per REQUESTED model — the cheapest honest answer to // "what did the label-routed endpoint cost", with no migration and no new call class: the model slug is // already on every checkpoint, and which models a label routes to is knowable from the config. Keyed on // model_requested (not model_actual) so a provider that canonicalises its slug in the response does not // split one line of spend in two. func (s *Store) SpendByModel(bookID string) (map[string]float64, error) { ctx, cancel := opContext() defer cancel() rows, err := s.r.QueryContext(ctx, ` SELECT c.model_requested, COALESCE(SUM(c.cost_usd), 0) FROM checkpoints c JOIN jobs j ON c.job_id = j.id WHERE j.book_id = ? GROUP BY c.model_requested`, bookID) if err != nil { return nil, err } defer rows.Close() out := map[string]float64{} for rows.Next() { var model string var sum float64 if err := rows.Scan(&model, &sum); err != nil { return nil, err } if sum > 0 { // a $0 derived checkpoint is not a spend line out[model] = sum } } if err := rows.Err(); err != nil { return nil, err } if len(out) == 0 { return nil, nil } return out, nil } func (s *Store) EscalationSpentUSD(bookID string) (float64, error) { ctx, cancel := opContext() defer cancel() var sum float64 err := s.r.QueryRowContext(ctx, ` SELECT COALESCE(SUM(c.cost_usd), 0) FROM checkpoints c JOIN jobs j ON c.job_id = j.id WHERE j.book_id = ? AND c.escalation = 1`, bookID).Scan(&sum) return sum, err }