269 lines
14 KiB
Go
269 lines
14 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// rebill.go: CONSENT TO A RE-PAYMENT (D20.2-Q2, spec §7.3 of backend/docs/D15.2-*.md) — the Р6
|
||
// "consent to a CONCRETE spend" gate standing in front of every path that would bill again for work
|
||
// this book has already been billed for.
|
||
//
|
||
// GRANULARITY (ratified with this pack). The spec's §7 projection is PER-CHUNK and rides `guard_hash`,
|
||
// which does not exist in this engine at all: drift here is BOOLEAN (a stored row's snapshot either is
|
||
// or is not the current one) and a snapshot move misses EVERY checkpoint of its wave. So consent is
|
||
// built at the granularity the engine actually re-pays at — the SNAPSHOT — and the projection is the
|
||
// honest reading of that: "the units already paid for under a superseded snapshot will be paid for
|
||
// again; the sum of their stored chunk_status.cost_usd is $X".
|
||
//
|
||
// Two consequences of that granularity, both deliberate:
|
||
// - the spec's §7.1-бис editor-cascade fix ("a stage strictly below a re-billed unit is itself
|
||
// re-billed") is DEGENERATE here — everything of the moved wave is re-billed already, so the
|
||
// cascade rule adds nothing. Its ratified ordering ("the cascade lands BEFORE the flag semantics")
|
||
// is therefore not violated but inapplicable;
|
||
// - the spec's EstimateUSD fallback ("for cascade units with no past price") has no subject either:
|
||
// every unit counted here HAS a stored price, because it is counted precisely for having been paid.
|
||
// A paid row whose stored cost is $0 is a genuinely $0 model (local/priced-zero), so 0 is its honest
|
||
// contribution rather than a gap to estimate around.
|
||
//
|
||
// WHAT IS NOT PROJECTED (documented, not silent): the CONTENT axis. A source edit that leaves the chunk
|
||
// manifest intact moves a chunk's content_hash but not the snapshot, so it re-bills exactly the touched
|
||
// chunks and is invisible here — the same caveat status.bookChunks carries, and the half that belongs
|
||
// to content-addressed resume (v3), not to this gate.
|
||
|
||
// Ratified default threshold (D20.2-Q2): `min($0.50, 5% × ProjectedBookUSD)`, with the $0.50 acting as
|
||
// an absolute FLOOR when the book has no processed units yet (the 5% branch would then be $0 and would
|
||
// demand consent for a one-cent append). A book may override it with `rebill_consent_usd`.
|
||
const (
|
||
rebillConsentFloorUSD = 0.50
|
||
rebillConsentShare = 0.05
|
||
)
|
||
|
||
// RebillConsent is the operator's answer to `--accept-rebill[=usd]`: the Р6 consent to a CONCRETE
|
||
// spend. A bare flag accepts whatever the projection turns out to be; `--accept-rebill=1.50` accepts it
|
||
// only while it stays at or below $1.50, so the consent names an amount rather than a blanket.
|
||
type RebillConsent struct {
|
||
Given bool // the flag was passed at all
|
||
Capped bool // a ceiling was named (--accept-rebill=<usd>)
|
||
CapUSD float64 // that ceiling; meaningful only when Capped
|
||
}
|
||
|
||
// RebillProjection is what a run would re-pay: the chunk×stage units resolved under a superseded
|
||
// snapshot, and the sum of what they cost the first time.
|
||
type RebillProjection struct {
|
||
Rows int
|
||
USD float64
|
||
// Repinned counts the units the run will serve for $0 despite a moved snapshot (pack-20 point 5): the
|
||
// move was bank-only and their rendered bytes are unchanged. It is not part of the amount — it is the
|
||
// number that makes the amount believable, because before pack-20 every one of these was counted as a
|
||
// re-payment and the operator was asked to consent to a whole wave for one signed word.
|
||
Repinned int
|
||
}
|
||
|
||
// projectRebill sums the units that a run under the CURRENT config would pay for a second time.
|
||
//
|
||
// A unit counts when all three hold: it carries a snapshot (a row written by a real run), it was BILLED
|
||
// (ok/flagged — a `skipped` row never reached a provider and cost nothing), and its snapshot differs
|
||
// from what its own WAVE renders now. The per-wave comparison is load-bearing: a bank-mining enrichment
|
||
// moves only the edit-wave snapshot, and projecting the draft wave against it would report the whole
|
||
// book as re-billed when the drafts in fact resume at $0 (the "re-paid ONCE" invariant).
|
||
//
|
||
// Two kinds of stored row are ORPHANS — they exist but nothing will run them again, so counting them
|
||
// would ask the operator to consent to money that will not be spent:
|
||
// - a row whose STAGE the current pipeline no longer has (a renamed/retired stage);
|
||
// - a row whose CHUNK the current manifest no longer has (the source was shortened, so the position
|
||
// is gone). Both waves address their rows at manifest chunk positions — a draft row at its own
|
||
// chunk, an edit row at its unit's leader chunk — so one membership test covers both.
|
||
//
|
||
// Over-estimating is the safe direction for a consent gate, but a number the operator is asked to
|
||
// approve and then not charged is exactly what makes such a number stop being read.
|
||
//
|
||
// CONTENT AWARENESS (pack-20 point 5, closing the phase-1 P3 finding). Snapshot divergence alone
|
||
// over-counts: since the bank is folded per wave, signing ONE term marked the whole edit wave as re-billed
|
||
// even though almost every unit's rendered bytes are unchanged. The projection now models the resume
|
||
// predicate the run actually applies (repin.go): a bank-only snapshot move whose unit renders the SAME
|
||
// content hash resumes at $0 and is counted as re-pinned, not re-paid. What remains in the amount is the
|
||
// units where the changed term genuinely occurs — the "N units, ~$X" the owner asked to see before
|
||
// consenting. The reproduction is $0 (string rendering + store reads, no provider), and a position whose
|
||
// inputs cannot be reproduced falls back to the conservative answer: counted as a re-payment.
|
||
//
|
||
// The wave snapshots are rendered LAZILY, so a draft-only pipeline never renders an edit-wave snapshot
|
||
// it has no stages for.
|
||
func (r *Runner) projectRebill(statuses []store.ChunkStatus, chunks []chunk.Chunk) (RebillProjection, error) {
|
||
var p RebillProjection
|
||
decider := newRepinDecider(r)
|
||
var contentHashes map[chunkKey]map[string]string
|
||
live := make(map[chunkKey]bool, len(chunks))
|
||
for _, ch := range chunks {
|
||
live[chunkKey{ch.Chapter, ch.ChunkIdx}] = true
|
||
}
|
||
draftNames := stageNameSet(r.waveStagesIndexed(waveDraft))
|
||
editNames := stageNameSet(r.waveStagesIndexed(waveEdit))
|
||
rendered := map[wave]string{}
|
||
current := func(w wave) (string, error) {
|
||
if s, ok := rendered[w]; ok {
|
||
return s, nil
|
||
}
|
||
s, _, err := r.snapshotIDForWave(w)
|
||
if err != nil {
|
||
return "", fmt.Errorf("pipeline: render the current snapshot for the re-bill projection: %w", err)
|
||
}
|
||
rendered[w] = s
|
||
return s, nil
|
||
}
|
||
for _, cs := range statuses {
|
||
if cs.SnapshotID == "" || cs.Disposition == string(DispSkipped) {
|
||
continue
|
||
}
|
||
if !live[chunkKey{cs.Chapter, cs.ChunkIdx}] {
|
||
continue // the position is gone from the manifest — nothing will re-run it
|
||
}
|
||
var w wave
|
||
switch {
|
||
case draftNames[cs.Stage]:
|
||
w = waveDraft
|
||
case editNames[cs.Stage]:
|
||
w = waveEdit
|
||
default:
|
||
continue // a stage the current pipeline does not run is never re-billed
|
||
}
|
||
cur, err := current(w)
|
||
if err != nil {
|
||
return p, err
|
||
}
|
||
if cs.SnapshotID == cur {
|
||
continue // resumes at $0
|
||
}
|
||
// The snapshot moved — but that is not yet a re-payment. Ask the two questions the resume path asks.
|
||
bankOnly, berr := decider.bankOnlyMove(cs.SnapshotID, w)
|
||
if berr != nil {
|
||
return p, berr
|
||
}
|
||
if bankOnly {
|
||
if contentHashes == nil { // rendered once, lazily: a book with no bank move never pays for it
|
||
contentHashes = r.renderedContentHashes(chunks, precomputeSticky(chunks, r.baseMemory, r.Pipeline.Context.GlossaryTokenBudget))
|
||
}
|
||
if h, ok := contentHashes[chunkKey{cs.Chapter, cs.ChunkIdx}][cs.Stage]; ok && h == cs.ContentHash {
|
||
p.Repinned++
|
||
continue // the bank moved somewhere else in the book; this unit is re-pinned for $0
|
||
}
|
||
}
|
||
p.Rows++
|
||
p.USD += cs.CostUSD
|
||
}
|
||
return p, nil
|
||
}
|
||
|
||
// projectBookUSD extrapolates the book's total cost from the units already FULLY attempted (done or
|
||
// flagged), which is the base of the 5% consent threshold. It is the SINGLE definition of the number
|
||
// `tmctl status` reports as projected_book_usd — status used to compute it inline, and a threshold
|
||
// computed from a second, drifting definition of the same quantity is exactly the class of bug the
|
||
// memberDrops helper was extracted to remove.
|
||
//
|
||
// It deliberately does NOT use book-committed spend: committed also carries partial spend on
|
||
// IN-PROGRESS units, which are outside the denominator and would over-estimate the book.
|
||
func projectBookUSD(units []editUnit, byChunk map[chunkKey][]store.ChunkStatus, nDraftStages, nEditStages int) float64 {
|
||
var processedCost float64
|
||
processed := 0
|
||
for _, u := range units {
|
||
expected := len(u.Members)*nDraftStages + nEditStages
|
||
res := resolveChunkState(unitRows(u, byChunk), expected)
|
||
if res.State != ChunkDone && res.State != ChunkFlagged {
|
||
continue
|
||
}
|
||
processedCost += res.CostUSD
|
||
processed++
|
||
}
|
||
if processed == 0 {
|
||
return 0
|
||
}
|
||
return processedCost / float64(processed) * float64(len(units))
|
||
}
|
||
|
||
// rebillConsentThreshold resolves the consent threshold for this book: the book's own
|
||
// `rebill_consent_usd` when it declares one, else the ratified `min($0.50, 5% × ProjectedBookUSD)` with
|
||
// the $0.50 floor for a book that has processed nothing yet.
|
||
func (r *Runner) rebillConsentThreshold(projectedBookUSD float64) (usd float64, source string) {
|
||
if r.Book.RebillConsentUSD > 0 {
|
||
return r.Book.RebillConsentUSD, "book.rebill_consent_usd"
|
||
}
|
||
if projectedBookUSD <= 0 {
|
||
return rebillConsentFloorUSD, "the $0.50 floor — the book has no processed units to take 5% of"
|
||
}
|
||
if share := rebillConsentShare * projectedBookUSD; share < rebillConsentFloorUSD {
|
||
return share, fmt.Sprintf("5%% of the projected book cost $%.6f", projectedBookUSD)
|
||
}
|
||
return rebillConsentFloorUSD, fmt.Sprintf("the $0.50 cap, under 5%% of the projected book cost $%.6f", projectedBookUSD)
|
||
}
|
||
|
||
// checkRebillConsent is the gate: it refuses BEFORE any reservation when the run would re-pay more than
|
||
// the book's consent threshold and the operator has not consented to that amount.
|
||
//
|
||
// It is called from both write paths — TranslateBook (before the waves, hence before the first
|
||
// Reserve) and Redrive (before the DESTRUCTIVE reset, so a refusal cannot leave the flag telemetry
|
||
// deleted, the external-review 1c torn-state discipline). The $0 read-only surfaces (status / report /
|
||
// export, D20.4) never reach it, so a book that needs consent stays fully inspectable.
|
||
//
|
||
// It is deliberately NOT conditioned on r.Resnapshot. --resnapshot is the permission to RE-PIN, and a
|
||
// permission that names no amount cannot carry a Р6 consent to a concrete spend — that is precisely
|
||
// the debt this closes. It also covers the case --resnapshot does not: a run interrupted midway through
|
||
// a re-pin leaves jobs on the new snapshot while their chunk_status rows still carry the old one, and
|
||
// the next plain `translate` then re-bills them with no gate at all.
|
||
func (r *Runner) checkRebillConsent(ctx context.Context, chunks []chunk.Chunk) error {
|
||
statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
|
||
if err != nil {
|
||
return fmt.Errorf("pipeline: read chunk_status for the re-bill projection: %w", err)
|
||
}
|
||
proj, err := r.projectRebill(statuses, chunks)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if proj.Rows == 0 {
|
||
return nil // nothing already-paid is superseded — this run bills only new work
|
||
}
|
||
|
||
byChunk := map[chunkKey][]store.ChunkStatus{}
|
||
for _, cs := range statuses {
|
||
byChunk[chunkKey{cs.Chapter, cs.ChunkIdx}] = append(byChunk[chunkKey{cs.Chapter, cs.ChunkIdx}], cs)
|
||
}
|
||
bookUSD := projectBookUSD(r.outputUnits(chunks), byChunk,
|
||
len(r.waveStagesIndexed(waveDraft)), len(r.waveStagesIndexed(waveEdit)))
|
||
threshold, source := r.rebillConsentThreshold(bookUSD)
|
||
|
||
// A NAMED ceiling is an instruction, not merely a consent form: it is honoured even below the
|
||
// threshold, so an operator who wrote "no more than $X" is never billed $X+ε on the grounds that the
|
||
// amount was small enough not to need asking.
|
||
if r.AcceptRebill.Capped && proj.USD > r.AcceptRebill.CapUSD {
|
||
return fmt.Errorf("pipeline: the projected re-payment is ~$%.6f (%d chunk×stage unit(s) already billed under a superseded snapshot) but --accept-rebill=%g caps consent at $%.6f — refusing. NOTHING was reserved and no row was touched. Raise the ceiling, or pass a bare --accept-rebill to accept the full projected amount",
|
||
proj.USD, proj.Rows, r.AcceptRebill.CapUSD, r.AcceptRebill.CapUSD)
|
||
}
|
||
if r.AcceptRebill.Given {
|
||
r.Log.WarnContext(ctx, "accepting a projected re-payment of already-billed work (--accept-rebill)",
|
||
"rebill_units", proj.Rows, "rebill_usd", fmt.Sprintf("%.6f", proj.USD),
|
||
"repinned_free", proj.Repinned, "threshold_usd", fmt.Sprintf("%.6f", threshold))
|
||
return nil
|
||
}
|
||
if proj.USD <= threshold {
|
||
// Under the threshold the run continues without friction (the ratified behaviour: a term append
|
||
// touching three chunks costs cents). Continuing is not the same as being silent — the amount is
|
||
// money and it goes to the log.
|
||
r.Log.InfoContext(ctx, "re-paying already-billed work under the consent threshold; continuing without asking",
|
||
"rebill_units", proj.Rows, "rebill_usd", fmt.Sprintf("%.6f", proj.USD),
|
||
"repinned_free", proj.Repinned, "threshold_usd", fmt.Sprintf("%.6f", threshold))
|
||
return nil
|
||
}
|
||
|
||
hint := ""
|
||
if !r.Resnapshot {
|
||
hint = " The run also needs --resnapshot: without it the superseded jobs stop it anyway."
|
||
}
|
||
repin := ""
|
||
if proj.Repinned > 0 {
|
||
repin = fmt.Sprintf(" (%d further unit(s) are re-pinned for $0 — the bank moved but their injected bytes did not)", proj.Repinned)
|
||
}
|
||
return fmt.Errorf("pipeline: this run would RE-PAY for work already billed: %d chunk×stage unit(s) are resolved under a superseded snapshot and would be paid for again, ~$%.6f (the sum of their stored cost_usd)%s. That is over this book's consent threshold $%.6f (%s), and Р6 requires consent to a CONCRETE spend, not a blanket one (D20.2-Q2). NOTHING was reserved and no row was touched. Re-run with --accept-rebill to accept the whole projected amount, or --accept-rebill=<usd> to accept it only up to a ceiling (a ceiling below the projection refuses).%s",
|
||
proj.Rows, proj.USD, repin, threshold, source, hint)
|
||
}
|