textmachine/backend/internal/pipeline/rebill.go

package pipeline

import (
	"context"
	"fmt"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/store"
)

// rebill.go: CONSENT TO A RE-PAYMENT (D20.2-Q2, spec §7.3 of backend/docs/D15.2-*.md) — the Р6
// "consent to a CONCRETE spend" gate standing in front of every path that would bill again for work
// this book has already been billed for.
//
// GRANULARITY (ratified with this pack). The spec's §7 projection is PER-CHUNK and rides `guard_hash`,
// which does not exist in this engine at all: drift here is BOOLEAN (a stored row's snapshot either is
// or is not the current one) and a snapshot move misses EVERY checkpoint of its wave. So consent is
// built at the granularity the engine actually re-pays at — the SNAPSHOT — and the projection is the
// honest reading of that: "the units already paid for under a superseded snapshot will be paid for
// again; their recorded tokens at TODAY'S price table come to $X" (reprice.go, row 181 — until then it
// summed the historical cost_usd, which is a different table the moment a vendor moves its prices).
//
// Two consequences of that granularity, both deliberate:
//   - the spec's §7.1-бис editor-cascade fix ("a stage strictly below a re-billed unit is itself
//     re-billed") is DEGENERATE here — everything of the moved wave is re-billed already, so the
//     cascade rule adds nothing. Its ratified ordering ("the cascade lands BEFORE the flag semantics")
//     is therefore not violated but inapplicable;
//   - the spec's EstimateUSD fallback ("for cascade units with no past price") has no subject either:
//     every unit counted here HAS stored usage, because it is counted precisely for having been paid.
//     A paid row that re-prices to $0 is a genuinely $0 model (local/priced-zero), so 0 is its honest
//     contribution rather than a gap to estimate around.
//
// WHAT IS NOT PROJECTED (documented, not silent): the CONTENT axis. A source edit that leaves the chunk
// manifest intact moves a chunk's content_hash but not the snapshot, so it re-bills exactly the touched
// chunks and is invisible here — the same caveat status.bookChunks carries, and the half that belongs
// to content-addressed resume (v3), not to this gate.

// Ratified default threshold (D20.2-Q2): `min($0.50, 5% × ProjectedBookUSD)`, with the $0.50 acting as
// an absolute FLOOR when the book has no processed units yet (the 5% branch would then be $0 and would
// demand consent for a one-cent append). A book may override it with `rebill_consent_usd`.
const (
	rebillConsentFloorUSD = 0.50
	rebillConsentShare    = 0.05
)

// RebillConsent is the operator's answer to `--accept-rebill[=usd]`: the Р6 consent to a CONCRETE
// spend. A bare flag accepts whatever the projection turns out to be; `--accept-rebill=1.50` accepts it
// only while it stays at or below $1.50, so the consent names an amount rather than a blanket.
type RebillConsent struct {
	Given  bool    // the flag was passed at all
	Capped bool    // a ceiling was named (--accept-rebill=<usd>)
	CapUSD float64 // that ceiling; meaningful only when Capped
}

// RebillProjection is what a run would re-pay: the chunk×stage units resolved under a superseded
// snapshot, and what buying them again would cost AT THE CURRENT PRICE TABLE (reprice.go).
type RebillProjection struct {
	Rows int
	USD  float64
	// HistoricalRows counts the units inside USD that could not be fully re-priced and carry the amount
	// they were billed at instead: a billed-decode row with no usage on file, a row whose calls no longer
	// account for its money, or one whose newest calls are provably not its own (reprice.go). It exists so
	// the operator-facing text can name what the number is made of rather than claim a re-pricing it did
	// not achieve.
	HistoricalRows int
	// OutputUnits is the same re-payment counted in OUTPUT UNITS — the granularity the manifest publishes
	// as units_total, that `--max-units` bounds, and that the platform sells chapters in.
	//
	// ⚠ IT EXISTS BECAUSE `Rows` IS A DIFFERENT UNIT WEARING THE SAME WORD. Rows counts chunk×stage, the
	// BILLING unit; every other "units" number the engine puts on the wire (total_units, progress totals,
	// chapters[].units_total) is the OUTPUT unit. Measured on a three-chapter fixture, one document carried
	// `rebill_units: 15` two lines under `total_units: 6` with nothing marking the difference — and the
	// ratio is not a constant a consumer could divide out: it is len(Members)·nDraftStages + nEditStages,
	// which varies unit by unit WITHIN one book and whose factors never cross the seam at all. A platform
	// sizing a re-pass from the estimate and passing that number to --max-units would buy several times the
	// book it meant to, which is this pack's own defect reproduced one layer down.
	OutputUnits int
	// Repinned counts the units the run will serve for $0 despite a moved snapshot (pack-20 point 5): the
	// move was bank-only and their rendered bytes are unchanged. It is not part of the amount — it is the
	// number that makes the amount believable, because before pack-20 every one of these was counted as a
	// re-payment and the operator was asked to consent to a whole wave for one signed word.
	Repinned int
}

// projectRebill sums the units that a run under the CURRENT config would pay for a second time.
//
// A unit counts when all three hold: it carries a snapshot (a row written by a real run), it was BILLED
// (ok/flagged — a `skipped` row never reached a provider and cost nothing), and its snapshot differs
// from what its own WAVE renders now. The per-wave comparison is load-bearing: a bank-mining enrichment
// moves only the edit-wave snapshot, and projecting the draft wave against it would report the whole
// book as re-billed when the drafts in fact resume at $0 (the "re-paid ONCE" invariant).
//
// Two kinds of stored row are ORPHANS — they exist but nothing will run them again, so counting them
// would ask the operator to consent to money that will not be spent:
//   - a row whose STAGE the current pipeline no longer has (a renamed/retired stage);
//   - a row whose CHUNK the current manifest no longer has (the source was shortened, so the position
//     is gone). Both waves address their rows at manifest chunk positions — a draft row at its own
//     chunk, an edit row at its unit's leader chunk — so one membership test covers both.
//
// Over-estimating is the safe direction for a consent gate, but a number the operator is asked to
// approve and then not charged is exactly what makes such a number stop being read. Which is why the
// amount is not a safety margin either way: it is the stored tokens re-priced through the same seam the
// reservation will use, so the two answer the same question with the same table.
//
// CONTENT AWARENESS (pack-20 point 5, closing the phase-1 P3 finding). Snapshot divergence alone
// over-counts: since the bank is folded per wave, signing ONE term marked the whole edit wave as re-billed
// even though almost every unit's rendered bytes are unchanged. The projection now models the resume
// predicate the run actually applies (repin.go): a bank-only snapshot move whose unit renders the SAME
// content hash resumes at $0 and is counted as re-pinned, not re-paid. What remains in the amount is the
// units where the changed term genuinely occurs — the "N units, ~$X" the owner asked to see before
// consenting. The reproduction is $0 (string rendering + store reads, no provider), and a position whose
// inputs cannot be reproduced falls back to the conservative answer: counted as a re-payment.
//
// The wave snapshots are rendered LAZILY, so a draft-only pipeline never renders an edit-wave snapshot
// it has no stages for.
//
// `manifest` supplies only the POSITIONS still in the book (the membership test above), which is why it
// may be the text-free projection of the persisted manifest (manifest.go). `withText` supplies the same
// chunks WITH their source, and is called only on the content-aware branch below — the one that has to
// re-render injected bytes and therefore genuinely needs the text. Splitting the two is what lets a
// status read skip a 1.4 s re-chunk it almost never needs, without ever computing a content hash over an
// empty string.
func (r *Runner) projectRebill(statuses []store.ChunkStatus, manifest []chunk.Chunk, withText func() ([]chunk.Chunk, error), rp *repricer) (RebillProjection, error) {
	var p RebillProjection
	decider := newRepinDecider(r)
	var contentHashes map[chunkKey]map[string]string
	live := make(map[chunkKey]bool, len(manifest))
	for _, ch := range manifest {
		live[chunkKey{ch.Chapter, ch.ChunkIdx}] = true
	}
	// Which OUTPUT unit each position belongs to, so the same re-payment can be reported in the unit the
	// seam sells in as well as the one it bills in. An edit row lives at its unit's leader chunk and a
	// leader is one of the unit's members, so this one map covers both waves.
	leaderOf := make(map[chunkKey]chunkKey, len(manifest))
	for _, u := range r.outputUnits(manifest) {
		leader := chunkKey{u.Chapter, u.FirstChunkIdx}
		for _, m := range u.Members {
			leaderOf[chunkKey{m.Chapter, m.ChunkIdx}] = leader
		}
	}
	touched := map[chunkKey]bool{}
	draftNames := stageNameSet(r.waveStagesIndexed(waveDraft))
	editNames := stageNameSet(r.waveStagesIndexed(waveEdit))
	rendered := map[wave]string{}
	current := func(w wave) (string, error) {
		if s, ok := rendered[w]; ok {
			return s, nil
		}
		s, _, err := r.snapshotIDForWave(w)
		if err != nil {
			return "", fmt.Errorf("pipeline: render the current snapshot for the re-bill projection: %w", err)
		}
		rendered[w] = s
		return s, nil
	}
	for _, cs := range statuses {
		if cs.SnapshotID == "" || cs.Disposition == string(DispSkipped) {
			continue
		}
		if !live[chunkKey{cs.Chapter, cs.ChunkIdx}] {
			continue // the position is gone from the manifest — nothing will re-run it
		}
		var w wave
		switch {
		case draftNames[cs.Stage]:
			w = waveDraft
		case editNames[cs.Stage]:
			w = waveEdit
		default:
			continue // a stage the current pipeline does not run is never re-billed
		}
		cur, err := current(w)
		if err != nil {
			return p, err
		}
		if cs.SnapshotID == cur {
			continue // resumes at $0
		}
		// The snapshot moved — but that is not yet a re-payment. Ask the two questions the resume path asks.
		bankOnly, berr := decider.bankOnlyMove(cs.SnapshotID, w)
		if berr != nil {
			return p, berr
		}
		if bankOnly {
			if contentHashes == nil { // rendered once, lazily: a book with no bank move never pays for it
				full, ferr := withText()
				if ferr != nil {
					return p, fmt.Errorf("pipeline: re-chunk the source for the re-bill content check: %w", ferr)
				}
				contentHashes = r.cachedRenderedContentHashes(full, precomputeSticky(full, r.baseMemory, r.Pipeline.Context.GlossaryTokenBudget))
			}
			if h, ok := contentHashes[chunkKey{cs.Chapter, cs.ChunkIdx}][cs.Stage]; ok && h == cs.ContentHash {
				p.Repinned++
				continue // the bank moved somewhere else in the book; this unit is re-pinned for $0
			}
		}
		p.Rows++
		touched[leaderOf[chunkKey{cs.Chapter, cs.ChunkIdx}]] = true
		usd, fromHistory := rp.usd(cs)
		p.USD += usd
		if fromHistory {
			p.HistoricalRows++
		}
	}
	p.OutputUnits = len(touched)
	return p, nil
}

// projectBookUSD extrapolates what the WHOLE book costs, from the units already FULLY attempted (done or
// flagged), at the CURRENT price table. It is the SINGLE definition of the number `tmctl status` reports
// as projected_book_usd — status used to compute it inline, and a threshold computed from a second,
// drifting definition of the same quantity is exactly the class of bug the memberDrops helper was
// extracted to remove.
//
// It re-prices for the same reason the re-payment amount does (row 181), and specifically so the consent
// gate cannot end up half-historical: the threshold is 5% of THIS, so leaving the base in old money
// while the amount moves to the new table would compare two different currencies and quietly change how
// often the operator is asked at all.
//
// It deliberately does NOT use book-committed spend: committed also carries partial spend on
// IN-PROGRESS units, which are outside the denominator and would over-estimate the book. Nor does it
// re-use resolveChunkState's CostUSD, which is the HISTORICAL sum the per-chapter passports report —
// what a chapter has cost is a fact, not a projection.
func projectBookUSD(units []editUnit, byChunk map[chunkKey][]store.ChunkStatus, nDraftStages, nEditStages int, rp *repricer) float64 {
	var processedCost float64
	processed := 0
	for _, u := range units {
		expected := len(u.Members)*nDraftStages + nEditStages
		rows := unitRows(u, byChunk)
		res := resolveChunkState(rows, expected)
		if res.State != ChunkDone && res.State != ChunkFlagged {
			continue
		}
		for _, cs := range rows {
			usd, _ := rp.usd(cs)
			processedCost += usd
		}
		processed++
	}
	if processed == 0 {
		return 0
	}
	return processedCost / float64(processed) * float64(len(units))
}

// rebillConsentThreshold resolves the consent threshold for this book: the book's own
// `rebill_consent_usd` when it declares one, else the ratified `min($0.50, 5% × ProjectedBookUSD)` with
// the $0.50 floor for a book that has processed nothing yet.
func (r *Runner) rebillConsentThreshold(projectedBookUSD float64) (usd float64, source string) {
	if r.Book.RebillConsentUSD > 0 {
		return r.Book.RebillConsentUSD, "book.rebill_consent_usd"
	}
	if projectedBookUSD <= 0 {
		return rebillConsentFloorUSD, "the $0.50 floor — the book has no processed units to take 5% of"
	}
	if share := rebillConsentShare * projectedBookUSD; share < rebillConsentFloorUSD {
		return share, fmt.Sprintf("5%% of the projected book cost $%.6f", projectedBookUSD)
	}
	return rebillConsentFloorUSD, fmt.Sprintf("the $0.50 cap, under 5%% of the projected book cost $%.6f", projectedBookUSD)
}

// projectionBasis says, inside the consent text, what the amount is made of. The parenthetical used to
// read "the sum of their stored cost_usd" and became a lie the day the projection stopped summing it
// (row 181) — in the single most visible place, the sentence the operator's consent is given to. It
// names the un-re-priced remainder rather than rounding it away: an amount partly quoted in last
// season's money is still a fact about the number, and hiding it is how the previous text went stale.
func projectionBasis(p RebillProjection) string {
	if p.HistoricalRows > 0 {
		return fmt.Sprintf("their recorded tokens at the CURRENT price table; %d of the %d unit(s) carry, wholly or in part, the amount they were originally billed at — that much of their usage could not be re-priced", p.HistoricalRows, p.Rows)
	}
	return "their recorded tokens at the CURRENT price table"
}

// checkRebillConsent is the gate: it refuses BEFORE any reservation when the run would re-pay more than
// the book's consent threshold and the operator has not consented to that amount.
//
// It is called from both write paths — TranslateBook (before the waves, hence before the first
// Reserve) and Redrive (before the DESTRUCTIVE reset, so a refusal cannot leave the flag telemetry
// deleted, the external-review 1c torn-state discipline). The $0 read-only surfaces (status / report /
// export, D20.4) never reach it, so a book that needs consent stays fully inspectable.
//
// It is deliberately NOT conditioned on r.Resnapshot. --resnapshot is the permission to RE-PIN, and a
// permission that names no amount cannot carry a Р6 consent to a concrete spend — that is precisely
// the debt this closes. It also covers the case --resnapshot does not: a run interrupted midway through
// a re-pin leaves jobs on the new snapshot while their chunk_status rows still carry the old one, and
// the next plain `translate` then re-bills them with no gate at all.
// `scope` is the run's VOLUME ceiling (volume.go), nil when none is in force. It produces TWO figures,
// and which of them each half of the gate is judged against is the whole correctness of this function.
//
// ⚠ THE THRESHOLD IS JUDGED AGAINST THE BOOK, NEVER AGAINST THE RUN — and the first version of this
// pack got that wrong. Scoping the amount to the admitted units and leaving the threshold book-wide
// sounds symmetrical and is not: with a volume ceiling the caller CHOOSES how small each run is, so
// `--resnapshot --max-units N` in a loop re-pays the entire book without a single run ever crossing the
// threshold and without consent being asked once. That is precisely the silent re-purchase Р6 exists to
// prevent, and "the change is inert without the flag" is true and beside the point: with the flag it
// opens the door the gate is the door for. A threshold a caller can defeat by splitting is not a
// threshold. So the question "must we ask at all" is answered by the book's whole drift, which does not
// shrink when a purchase does.
//
// ⚠ THE NAMED CAP IS JUDGED AGAINST THE RUN, because it is a different kind of statement. A threshold is
// the book's policy on when a human must be consulted; `--accept-rebill=X` is the caller's instruction
// "spend no more than X". Measuring an instruction about SPEND against work this run will not do would
// refuse a caller whose cap fully covers what they are about to be charged — the platform funds that cap
// from the run's own hold, so it is sized to the purchase, not to the book.
//
// With no ceiling in force the two projections are the same object and exactly one is computed, so every
// existing run is judged by byte-identical figures.
func (r *Runner) checkRebillConsent(ctx context.Context, chunks []chunk.Chunk, scope *volumeScope) error {
	statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
	if err != nil {
		return fmt.Errorf("pipeline: read chunk_status for the re-bill projection: %w", err)
	}
	rp, err := r.newRepricer()
	if err != nil {
		return err
	}
	// A write path already holds the real split, so the lazy provider just hands it back — no second
	// ingest, and no branch where the consent gate could see text-free chunks.
	withText := func() ([]chunk.Chunk, error) { return chunks, nil }
	book, err := r.projectRebill(statuses, chunks, withText, rp)
	if err != nil {
		return err
	}
	if book.Rows == 0 {
		return nil // nothing already-paid is superseded — this run bills only new work
	}
	// What THIS run will actually re-pay. Only computed when a ceiling narrows it; otherwise it IS the
	// book's figure, which keeps the un-bounded path at one projection exactly as before.
	proj := book
	if scope != nil {
		if proj, err = r.projectRebill(scope.admittedStatuses(statuses), chunks, withText, rp); err != nil {
			return err
		}
	}
	if proj.Rows == 0 {
		// The book carries drift, but none of it is inside what this run was granted, so this run re-pays
		// nothing and there is no concrete spend to consent to. Not a hole in the threshold above: the
		// moment a run is granted a superseded unit, proj stops being empty and the book-wide threshold
		// below decides — so splitting cannot walk past the gate, it can only postpone meeting it.
		return nil
	}

	// The denominator is the WHOLE book's stored rows, unfiltered by the volume scope: the threshold is
	// "5% of what this book costs", a property of the book.
	byChunk := map[chunkKey][]store.ChunkStatus{}
	for _, cs := range statuses {
		byChunk[chunkKey{cs.Chapter, cs.ChunkIdx}] = append(byChunk[chunkKey{cs.Chapter, cs.ChunkIdx}], cs)
	}
	bookUSD := projectBookUSD(r.outputUnits(chunks), byChunk,
		len(r.waveStagesIndexed(waveDraft)), len(r.waveStagesIndexed(waveEdit)), rp)
	threshold, source := r.rebillConsentThreshold(bookUSD)

	// A NAMED ceiling is an instruction, not merely a consent form: it is honoured even below the
	// threshold, so an operator who wrote "no more than $X" is never billed $X+ε on the grounds that the
	// amount was small enough not to need asking.
	if r.AcceptRebill.Capped && proj.USD > r.AcceptRebill.CapUSD {
		return fmt.Errorf("pipeline: the projected re-payment is ~$%.6f (%d chunk×stage unit(s) already billed under a superseded snapshot) but --accept-rebill=%g caps consent at $%.6f — refusing. NOTHING was reserved and no row was touched. Raise the ceiling, or pass a bare --accept-rebill to accept the full projected amount",
			proj.USD, proj.Rows, r.AcceptRebill.CapUSD, r.AcceptRebill.CapUSD)
	}
	if r.AcceptRebill.Given {
		r.Log.WarnContext(ctx, "accepting a projected re-payment of already-billed work (--accept-rebill)",
			"rebill_units", proj.Rows, "rebill_usd", fmt.Sprintf("%.6f", proj.USD),
			"book_rebill_units", book.Rows, "book_rebill_usd", fmt.Sprintf("%.6f", book.USD),
			"units_not_repriced", proj.HistoricalRows,
			"repinned_free", proj.Repinned, "threshold_usd", fmt.Sprintf("%.6f", threshold))
		return nil
	}
	// ⚠ THE BOOK'S figure is what the threshold judges, not this run's. See the doc comment: a caller
	// that chooses how small each run is could otherwise re-pay the whole book a slice at a time and
	// never once be asked. `book` equals `proj` whenever no volume ceiling is in force, so this is the
	// same comparison it has always been for every un-bounded run.
	if book.USD <= threshold {
		// Under the threshold the run continues without friction (the ratified behaviour: a term append
		// touching three chunks costs cents). Continuing is not the same as being silent — the amount is
		// money and it goes to the log.
		r.Log.InfoContext(ctx, "re-paying already-billed work under the consent threshold; continuing without asking",
			"rebill_units", proj.Rows, "rebill_usd", fmt.Sprintf("%.6f", proj.USD),
			"book_rebill_units", book.Rows, "book_rebill_usd", fmt.Sprintf("%.6f", book.USD),
			"units_not_repriced", proj.HistoricalRows,
			"repinned_free", proj.Repinned, "threshold_usd", fmt.Sprintf("%.6f", threshold))
		return nil
	}

	hint := ""
	if !r.Resnapshot {
		hint = " The run also needs --resnapshot: without it the superseded jobs stop it anyway."
	}
	// ⚠ EVERY FIGURE IN A SENTENCE MUST COME FROM THE SAME PROJECTION AS THE SENTENCE. This parenthetical
	// hangs off the BOOK-wide clause, so it takes the BOOK's re-pin count. The first version spliced the
	// volume-scoped one in, which under an active ceiling printed a smaller number inside a sentence whose
	// every other figure was book-wide — an operator adding them up would have got a book that did not
	// exist. Two independent review lenses caught the same splice, which is what mixed-provenance numbers
	// in one sentence reliably produce.
	repin := ""
	if book.Repinned > 0 {
		repin = fmt.Sprintf(" (%d further unit(s) of the book are re-pinned for $0 — the bank moved but their injected bytes did not)", book.Repinned)
	}
	// The two projections are printed SEPARATELY when a volume ceiling makes them differ, because
	// collapsing them is how this gate goes wrong in either direction: quoting only the book's total at a
	// small purchase asks for consent to money nobody will be charged, and quoting only the run's slice
	// hides that the book is being re-bought a slice at a time. The scoped clause carries the scoped
	// re-pin count for the same provenance reason.
	scoped := ""
	if book.Rows != proj.Rows || book.USD != proj.USD {
		scopedRepin := ""
		if proj.Repinned != book.Repinned {
			scopedRepin = fmt.Sprintf(" (and re-pin %d of them for $0)", proj.Repinned)
		}
		scoped = fmt.Sprintf(" THIS run, bounded by --max-units, would re-pay %d of them, ~$%.6f%s — but the threshold is judged against the book, because a ceiling the caller sizes could otherwise re-pay the whole book a slice at a time without ever being asked.",
			proj.Rows, proj.USD, scopedRepin)
	}
	return fmt.Errorf("pipeline: this run would RE-PAY for work already billed: %d chunk×stage unit(s) of this book are resolved under a superseded snapshot and would be paid for again, ~$%.6f (%s)%s.%s That is over this book's consent threshold $%.6f (%s), and Р6 requires consent to a CONCRETE spend, not a blanket one (D20.2-Q2). NOTHING was reserved and no row was touched. Re-run with --accept-rebill to accept the whole projected amount, or --accept-rebill=<usd> to accept it only up to a ceiling (the ceiling is measured against what THIS run re-pays, and one below that refuses).%s",
		book.Rows, book.USD, projectionBasis(book), repin, scoped, threshold, source, hint)
}