textmachine/backend/internal/pipeline/paidtail.go

package pipeline

import (
	"fmt"

	"textmachine/backend/internal/store"
)

// paidtail.go: WHAT THE MONEY BOUGHT, decomposed — the largest single line of the first paid run and the
// one axis no $0 surface showed.
//
// Measured on the cold run of 31.08: the money that did not become shipped text was the biggest item in
// the run, larger than escalation, and both `status` and `report` published only a TOTAL. An operator was
// told what a book cost and never what part of it bought nothing (disclosure law §2.1).
//
// ⛔ WHY THIS IS NOT THE OBVIOUS QUERY, and the reason is measured rather than argued. The obvious slice is
// `request_log WHERE ok = 0 AND cost_usd > 0`. On that run it selects 11 rows and $0.12316089 — and THREE
// different things:
//   - the classifier's three calls ($0.00918827), which SUCCEEDED. Their replies are healthy term tables
//     (`三转蛊师<TAB>term …`, finish=stop), flagged because the CJK-echo exemption covered ONLY the
//     terminologist AT THE TIME — it covers both bank roles now (terminologist.go, `isBankRole`) — so a
//     bilingual table read as an echo of its own source;
//   - one editor call ($0.01865424) whose text SHIPPED after a cosmetic sanitizer strip;
//   - and the rest, which genuinely bought nothing.
// A surface built on that slice would have told the operator he lost 29.2% more than he did — a money
// figure with a false cause, which is the disease this pack is about, committed by its own cure.
//
// SO THE DECOMPOSITION IS BUILT FROM THE CHECKPOINTS, not from the verdict column. Checkpoints are the
// money (`committed == SUM(checkpoints)` is an invariant of the ledger), they are append-only, and their
// INSERTION ORDER is the only thing that distinguishes a superseded call from the one that stands — which
// is why CheckpointUsageForBook is documented as ordering by rowid and `attempt` explicitly cannot do it.
//
// FOUR classes, exhaustive over paid checkpoints, and each states something literally true:
//   - SUPERSEDED — a later paid call for the SAME position replaced it. Retries and escalation hops both
//     land here, from the same rule, with no need to read either column.
//   - BANK — it bought the book's TERMINOLOGY rather than a chunk of its text.
//   - WITHHELD — the position's last paid call produced nothing shippable (its row carries no final hash).
//   - SHIPPED — it bought the bytes a reader gets.
// Nothing here re-derives money: every figure is a sum of costs the ledger already settled.
//
// ⛔ THE BANK CLASS EXISTS BECAUSE ITS ABSENCE MADE THIS FILE COMMIT THE DEFECT IT WAS BUILT AGAINST, and
// the acceptance caught it. The bank roles are checkpointed under a SYNTHETIC stage at chapter 0 and write
// NO chunk_status row at all — deliberately, they are not a wave — so `shipped` can never be true for
// them and the whole terminology contour fell into WITHHELD. On the cold run that is $0.04980482 of a
// $0.133 «loss», i.e. the surface overstated by 59.8%: worse than the 29.2% overstatement this file's own
// header rejects the naive `ok = 0` slice for. A live probe on a HEALTHY book printed «$0.010920 (85.7%)
// did NOT become shipped text» about a glossary pass that worked perfectly.
//
// The class is identified by the STAGE, which is the addressing label those calls carry
// (terminologyStageName) — equivalent to reading `checkpoints.role` and needing no store change. The
// REPAIR role is deliberately NOT in it: repair checkpoints carry their stage's real name, so they map to
// a chunk_status position like any other call and are classified by whether that position shipped.
//
// ⚠ A superseded BANK call is still superseded, and that is not a special case but the point: the whole
// contour is re-bought whenever the drafted set grows (backlog row 233), and that money genuinely bought
// nothing the second time. Only the call that STANDS is the bank's product.

// PaidTail is the decomposition of a book's committed spend by WHAT IT BOUGHT.
type PaidTail struct {
	// ShippedUSD bought the text that ships. SupersededUSD was replaced by a later paid call for the same
	// position. WithheldUSD bought a position that produced nothing shippable.
	ShippedUSD    float64 `json:"shipped_usd"`
	SupersededUSD float64 `json:"superseded_usd"`
	WithheldUSD   float64 `json:"withheld_usd"`
	// BankUSD bought the book's TERMINOLOGY — the consolidated glossary the editor injects — rather than a
	// chunk of its text. It is a product, not a loss, and it is separated because the bank roles write no
	// chunk_status row, so the "did it ship" question is not merely false for them: it is undefined.
	BankUSD float64 `json:"bank_usd"`
	// The call counts behind the money — a single expensive call and twenty cheap ones are different
	// problems with the same dollar figure.
	ShippedCalls    int `json:"shipped_calls"`
	SupersededCalls int `json:"superseded_calls"`
	WithheldCalls   int `json:"withheld_calls"`
	BankCalls       int `json:"bank_calls"`
	// TotalUSD is the sum of the three, i.e. every paid checkpoint of the book. It is published so a
	// reader can check the decomposition against the ledger's own committed figure instead of trusting it.
	TotalUSD float64 `json:"total_usd"`
	// WorstPosition names the single position that spent the most on money that did not ship, so the
	// largest loss is a place an operator can go to rather than a number. Empty when nothing was lost.
	WorstPosition string  `json:"worst_position,omitempty"`
	WorstUSD      float64 `json:"worst_usd,omitempty"`
}

// LostUSD is the money that bought NOTHING — neither shipped text nor bank. It excludes BankUSD by
// construction: a glossary pass that worked is not a loss, and printing it as one is the exact shape of
// false-cause this whole pack exists to remove.
func (t PaidTail) LostUSD() float64 { return t.SupersededUSD + t.WithheldUSD }

// paidTail decomposes the book's paid checkpoints. Pure over its inputs and free of I/O so the
// classification can be tested without a database behind it.
//
// `usage` MUST be in insertion order (CheckpointUsageForBook guarantees rowid order) — the whole
// superseded/standing distinction is that order, and a caller that sorts it differently gets a different
// and wrong answer.
func paidTail(usage []store.CheckpointUsage, statuses []store.ChunkStatus) PaidTail {
	type pos struct {
		chapter, chunkIdx int
		stage             string
	}
	shipped := make(map[pos]bool, len(statuses))
	for _, cs := range statuses {
		// A position ships when its row carries the hash the read models resolve text through. A flagged
		// row carries none, and neither does a skipped one.
		shipped[pos{cs.Chapter, cs.ChunkIdx, cs.Stage}] = cs.FinalHash != ""
	}
	// The LAST paid call at each position is the one that stands; every earlier one was replaced.
	lastPaid := map[pos]int{}
	for i, u := range usage {
		if u.CostUSD <= 0 {
			continue // derived $0 projections (banknote/sanitized exports) are not calls and cost nothing
		}
		lastPaid[pos{u.Chapter, u.ChunkIdx, u.Stage}] = i
	}
	var t PaidTail
	lost := map[pos]float64{}
	for i, u := range usage {
		if u.CostUSD <= 0 {
			continue
		}
		p := pos{u.Chapter, u.ChunkIdx, u.Stage}
		t.TotalUSD += u.CostUSD
		switch {
		case lastPaid[p] != i:
			t.SupersededUSD += u.CostUSD
			t.SupersededCalls++
			lost[p] += u.CostUSD
		case u.Stage == terminologyStageName:
			// The call that STANDS for a bank position bought the bank. It has no chunk_status row to
			// have shipped, and asking «did it ship» of it is a category error, not a failure.
			t.BankUSD += u.CostUSD
			t.BankCalls++
		case shipped[p]:
			t.ShippedUSD += u.CostUSD
			t.ShippedCalls++
		default:
			t.WithheldUSD += u.CostUSD
			t.WithheldCalls++
			lost[p] += u.CostUSD
		}
	}
	// Deterministic worst-position: ties break on the position, never on map order.
	for p, v := range lost {
		name := positionName(p.chapter, p.chunkIdx, p.stage)
		if v > t.WorstUSD || (v == t.WorstUSD && name < t.WorstPosition) {
			t.WorstUSD, t.WorstPosition = v, name
		}
	}
	return t
}

// positionName is the operator-facing address of a chunk×stage. Chapter 0 is the BOOK level (the bank
// roles address their batches there), so it is spelled as such rather than as a chapter nobody has.
func positionName(chapter, chunkIdx int, stage string) string {
	if chapter == 0 {
		return fmt.Sprintf("book/batch%d/%s", chunkIdx, stage)
	}
	return fmt.Sprintf("ch%d/chunk%d/%s", chapter, chunkIdx, stage)
}