textmachine/backend/internal/pipeline/escalation.go

package pipeline

import (
	"context"
	"errors"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/config"
	"textmachine/backend/internal/llm"
	"textmachine/backend/internal/store"
)

// escalation.go: single-hop escalation of a deterministic content failure (D12/D3) —
// one hop to a different model under budget_usd, the result is RE-gated, the retry
// budget is not reset. The hop model is the RESOLVED one: for a book with no content labels that is the
// stage's escalate_to, and for a labelled book it is the head of the label's chain, which REPLACES
// escalate_to so one call has one source of truth (D39.26 point 3 — a chain with more members is a loud
// load error, never a silently truncated walk).

// applyModelFloor raises a derived max_tokens budget to the CALL model's per-model
// floor (D24.3, capabilities.min_max_tokens): a thinking model returns finish=length
// or empty content below a per-model minimum because reasoning consumes the budget
// before content is emitted. The floor is taken by the model of THIS call — a primary
// attempt floors against the resolved stage model, an escalation hop against the resolved hop — so a hop
// that inherits a smaller primary budget re-derives its own (exactly what broke the
// draft→deepseek-v4-pro hops on acceptance stage A: they inherited 2048/3291 and
// returned length). 0 floor (reasoning-off models) leaves the budget untouched.
// Applied BEFORE the request_hash so it is wire-visible and snapshot-folded.
//
// ⚠ LATENT INTERACTION (D28.3б, note not a defect — no floored local model exists today): a
// `provider_local` kind overrides max_tokens on the wire AFTER the request-hash (prov.MaxTokens,
// snapshot.go ProviderMaxTok fold), so for a FLOORED local model the provider override would win
// over this floor on the actual wire. Both are in the snapshot (floor via Capability.MinMaxTokens,
// override via ProviderMaxTok), so it is a loud --resnapshot either way — but if a local model ever
// gets a min_max_tokens floor, decide explicitly whether the provider max_tokens override or the
// floor takes precedence (today the local provider max_tokens=8192 already covers thinking budgets).
func (r *Runner) applyModelFloor(base int, model string) int {
	if floor := r.Models.MinMaxTokens(model); floor > base {
		return floor
	}
	return base
}

// errReserveCeiling is wrapped into the error runAttempt returns when a book/day USD
// ceiling denies a reservation. The PRIMARY path propagates it (the book durably
// pauses and resumes once the ceiling is raised, D4); the OPTIONAL escalation hop
// catches it and degrades to keeping the primary flag instead of aborting the whole
// book on every run (self-review finding).
var errReserveCeiling = errors.New("reserve ceiling reached")

// CeilingHalt is the ceiling stop as a TYPE, so the two callers that must tell it from a crash can do so
// without reading prose: the CLI maps it to its own exit code (never 1 — the platform's contract forbids
// calling a resumable stop `failed`, PD-113) and the driver emits the stream's `ceiling` event from it.
// Scope names WHICH ceiling stopped the run, which is the diagnosis half of PD-157: a book's daily
// ceiling is one the platform neither sets nor sees, and today it cannot tell that stop from the one it
// chose itself.
//
// It WRAPS errReserveCeiling rather than replacing it, so every `errors.Is` that already degrades on a
// ceiling — the optional escalation hop, the repair sub-step, the terminology pass — keeps working
// unchanged.
type CeilingHalt struct {
	Scope string // runevents.ScopeBook | runevents.ScopeDay
	// ShortfallMicroUSD is HOW MUCH WAS MISSING — the amount by which the store's admission inequality
	// failed, in integer micro-USD rounded up (shortfallMicroUSD). It rides on the type because the
	// stream's `ceiling` event is built from this value and from nothing else the driver still has: the
	// figures it is derived from are read inside runAttempt and are gone by the time the error surfaces.
	// Zero means «not stated» — a day-scope stop, or a ledger read that failed — and the wire omits it.
	ShortfallMicroUSD int64
	err               error
}

func (e *CeilingHalt) Error() string { return e.err.Error() }
func (e *CeilingHalt) Unwrap() error { return e.err }

// escalationBudgetRemains reports whether the book may still spend on a single-hop
// fallback draft: escalation is OPT-IN via escalation.budget_usd (0 = disabled, the
// boevoy default — a stage's escalate_to is inert until a premium budget is set), and
// capped at that budget summed over the book's escalation checkpoints (money-path
// durable, resume-safe). A PRE-HOP soft cap: the gate admits a hop while spent <
// budget and does NOT pre-estimate the hop's own cost, so a single hop may overshoot
// the budget by up to its full cost; the NEXT chunk's escalation is then denied. Size
// the budget with that worst case in mind — it bounds TOTAL escalation, not per-hop.
//
// ⚠ Row 135 (the per-call ceiling gate) deliberately stopped here. The row's target is GRANULARITY —
// "a gate on every paid call, not on a unit of work" — and this gate is already per call: a hop IS one
// call. What it does not do is PRICE the call it admits, which is a different tightening, and the
// overshoot it allows is documented above, pinned by three tests and therefore a ratified bound rather
// than an oversight. Changing it is a question for the orchestrator, not a side effect of this pack;
// the repair sub-step, which took ONE decision per unit and then bought several calls under it, is the
// gate the row named and the one this pack tightened.
func (r *Runner) escalationBudgetRemains() (bool, error) {
	budget := r.Pipeline.Escal.BudgetUSD
	if budget <= 0 {
		return false, nil
	}
	spent, err := r.Store.EscalationSpentUSD(r.Book.BookID)
	if err != nil {
		return false, err
	}
	return spent < budget, nil
}

// escalationOutcome is maybeEscalate's report to runStage. attempted is true ONLY
// when a hop actually executed (fresh or replayed from its checkpoint) — a chunk
// whose hop was skipped (not escalatable / no escalate_to / budget exhausted) or
// denied by a USD ceiling reports attempted=false, so runStage keeps the primary
// flag and records Escalated=false (the exact pre-extraction semantics).
type escalationOutcome struct {
	attempted bool
	fb        stageAttempt // the fallback attempt; meaningful only when attempted
}

// maybeEscalate runs the single-hop escalation (D12 deterministic-content-failure
// class): a flag that a DIFFERENT model might fix (echo / excision / refusal) is
// routed ONCE to the stage's named fallback, under the book's escalation.budget_usd,
// and RE-GATED (classifyOutput runs on the fallback output too — §3.8). The editor
// declares no escalate_to → pinned per book (its style must not drift to a foreign
// model, D12/2605.13368). The fallback uses its OWN model (the request_hash axis), so
// its checkpoint never collides with the failed primary's, and it is exactly ONE call
// (not a fresh retry budget): total calls/chunk = primary attempts + 1 hop, never
// reset on the fallback (the LiteLLM #19985 retry×fallback blow-up guard). primary is
// the attempt loop's terminal (flagged) attempt.
func (r *Runner) maybeEscalate(ctx context.Context, st config.Stage, snapID string, ch chunk.Chunk, job *store.Job, baseMaxTokens int, msgs []llm.Message, primary stageAttempt, isFinal bool) (escalationOutcome, error) {
	var out escalationOutcome
	if !primary.cls.Reason.escalatable() || st.ResolvedHop == "" {
		return out, nil
	}
	// The hop budget is the primary's base re-floored against the HOP's own model
	// (D24.3): the fallback may be a thinking model with a larger minimum than the
	// primary (draft deepseek-v4-flash → hop deepseek-v4-pro both floor 8000; a
	// floor-less grok primary → a floored hop lifts to the hop's minimum). This is
	// exactly the stage-A fix — the hop no longer inherits a sub-floor 2048/3291.
	hopMaxTokens := r.applyModelFloor(baseMaxTokens, st.ResolvedHop)
	// Idempotency (self-review): if the fallback hop already happened its
	// checkpoint exists and was already paid — REPLAY it for free regardless of the
	// budget, so a crash AFTER the hop settled but BEFORE chunk_status was written
	// re-serves it on resume rather than discarding a paid, successful translation
	// and flipping the verdict OK→flagged. Only a FRESH hop is budget-gated. The identity is BUILT by the
	// same helper the hop's own runAttempt uses (attemptRequest), so "mirrors runAttempt" is structural
	// rather than a promise two field lists keep by vigilance.
	fbHash := RequestHash(r.attemptRequest(st, st.ResolvedHop, snapID, ch, 0, hopMaxTokens, msgs))
	fbExists, err := r.Store.GetCheckpoint(fbHash)
	if err != nil {
		return out, err
	}
	mayHop := fbExists != nil
	if !mayHop {
		// Fresh hop: serialize the budget admission THROUGH the paid settle across the parallel the draft wave draft
		// workers (R1, escMu). escalationBudgetRemains is a non-atomic read-then-act over EscalationSpentUSD,
		// so without this N concurrent draft chunks could each read spent<budget and all be admitted →
		// overshoot by up to N-1 hops. Holding escMu until this function returns (past runAttempt's settle)
		// makes a later worker read the UPDATED spend, restoring the sequential soft-cap (overshoot ≤ 1 hop).
		// The $0 replay path above (fbExists != nil) stays lock-free; escalations are the rare content-failure
		// exception, so the contention is negligible.
		r.escMu.Lock()
		defer r.escMu.Unlock()
		if mayHop, err = r.escalationBudgetRemains(); err != nil {
			return out, err
		}
	}
	if !mayHop {
		return out, nil
	}
	// NOT mandatory: this hop degrades on a ceiling two lines below, and it holds escMu for its whole
	// life — a hop that queued for headroom would block every other worker's escalation while spending
	// the money the primary wave needs to finish what it has already started.
	fb, err := r.runAttempt(ctx, st, st.ResolvedHop, snapID, ch, job, 0, hopMaxTokens, msgs, true, isFinal, false)
	if err != nil {
		// An OPTIONAL hop that trips a USD ceiling must NOT abort the whole book
		// (and re-abort on every resume): the chunk is already flagged, so keep
		// the primary flag and continue. Any other infra error still aborts.
		if !errors.Is(err, errReserveCeiling) {
			return out, err
		}
		r.Log.WarnContext(ctx, "escalation hop denied by a USD ceiling; keeping the primary flag",
			"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "reason", string(primary.cls.Reason))
		return out, nil
	}
	out.attempted, out.fb = true, fb
	r.Log.WarnContext(ctx, "stage escalated to a fallback model",
		"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx,
		"primary_reason", string(primary.cls.Reason), "fallback", st.ResolvedHop,
		"fallback_disposition", string(fb.cls.Reason.disposition()))
	return out, nil
}