107 lines
5.3 KiB
Go
107 lines
5.3 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/llm"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// escalation.go: single-hop эскалация детерминированного контент-провала (D12/D3) —
|
||
// один хоп на другую модель под budget_usd, результат РЕ-гейтится, retry-бюджет не
|
||
// сбрасывается. Канал B (18+) Ф2 расширяет ИМЕННО этот файл (permissive-цепочки).
|
||
|
||
// errReserveCeiling is wrapped into the error runAttempt returns when a book/day USD
|
||
// ceiling denies a reservation. The PRIMARY path propagates it (the book durably
|
||
// pauses and resumes once the ceiling is raised, D4); the OPTIONAL escalation hop
|
||
// catches it and degrades to keeping the primary flag instead of aborting the whole
|
||
// book on every run (self-review finding).
|
||
var errReserveCeiling = errors.New("reserve ceiling reached")
|
||
|
||
// escalationBudgetRemains reports whether the book may still spend on a single-hop
|
||
// fallback draft: escalation is OPT-IN via escalation.budget_usd (0 = disabled, the
|
||
// boevoy default — a stage's escalate_to is inert until a premium budget is set), and
|
||
// capped at that budget summed over the book's escalation checkpoints (money-path
|
||
// durable, resume-safe). A PRE-HOP soft cap: the gate admits a hop while spent <
|
||
// budget and does NOT pre-estimate the hop's own cost, so a single hop may overshoot
|
||
// the budget by up to its full cost; the NEXT chunk's escalation is then denied. Size
|
||
// the budget with that worst case in mind — it bounds TOTAL escalation, not per-hop.
|
||
func (r *Runner) escalationBudgetRemains() (bool, error) {
|
||
budget := r.Pipeline.Escal.BudgetUSD
|
||
if budget <= 0 {
|
||
return false, nil
|
||
}
|
||
spent, err := r.Store.EscalationSpentUSD(r.Book.BookID)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
return spent < budget, nil
|
||
}
|
||
|
||
// escalationOutcome is maybeEscalate's report to runStage. attempted is true ONLY
|
||
// when a hop actually executed (fresh or replayed from its checkpoint) — a chunk
|
||
// whose hop was skipped (not escalatable / no escalate_to / budget exhausted) or
|
||
// denied by a USD ceiling reports attempted=false, so runStage keeps the primary
|
||
// flag and records Escalated=false (the exact pre-extraction semantics).
|
||
type escalationOutcome struct {
|
||
attempted bool
|
||
fb stageAttempt // the fallback attempt; meaningful only when attempted
|
||
}
|
||
|
||
// maybeEscalate runs the single-hop escalation (D12 deterministic-content-failure
|
||
// class): a flag that a DIFFERENT model might fix (echo / excision / refusal) is
|
||
// routed ONCE to the stage's named fallback, under the book's escalation.budget_usd,
|
||
// and RE-GATED (classifyOutput runs on the fallback output too — §3.8). The editor
|
||
// declares no escalate_to → pinned per book (its style must not drift to a foreign
|
||
// model, D12/2605.13368). The fallback uses its OWN model (the request_hash axis), so
|
||
// its checkpoint never collides with the failed primary's, and it is exactly ONE call
|
||
// (not a fresh retry budget): total calls/chunk = primary attempts + 1 hop, never
|
||
// reset on the fallback (the LiteLLM #19985 retry×fallback blow-up guard). primary is
|
||
// the attempt loop's terminal (flagged) attempt.
|
||
func (r *Runner) maybeEscalate(ctx context.Context, st config.Stage, snapID string, ch Chunk, job *store.Job, baseMaxTokens int, msgs []llm.Message, primary stageAttempt) (escalationOutcome, error) {
|
||
var out escalationOutcome
|
||
if !primary.cls.Reason.escalatable() || st.EscalateTo == "" {
|
||
return out, nil
|
||
}
|
||
// Idempotency (self-review): if the fallback hop already happened its
|
||
// checkpoint exists and was already paid — REPLAY it for free regardless of the
|
||
// budget, so a crash AFTER the hop settled but BEFORE chunk_status was written
|
||
// re-serves it on resume rather than discarding a paid, successful translation
|
||
// and flipping the verdict OK→flagged. Only a FRESH hop is budget-gated. The
|
||
// hash mirrors runAttempt's for (model=EscalateTo, attempt=0, maxTokens=base).
|
||
fbHash := RequestHash(r.Book.BookID, ch.Chapter, ch.ChunkIdx, 0, st.Name, st.Role, st.EscalateTo,
|
||
st.Temperature, st.Reasoning, false, baseMaxTokens, snapID, msgs)
|
||
fbExists, err := r.Store.GetCheckpoint(fbHash)
|
||
if err != nil {
|
||
return out, err
|
||
}
|
||
mayHop := fbExists != nil
|
||
if !mayHop {
|
||
if mayHop, err = r.escalationBudgetRemains(); err != nil {
|
||
return out, err
|
||
}
|
||
}
|
||
if !mayHop {
|
||
return out, nil
|
||
}
|
||
fb, err := r.runAttempt(ctx, st, st.EscalateTo, snapID, ch, job, 0, baseMaxTokens, msgs, true)
|
||
if err != nil {
|
||
// An OPTIONAL hop that trips a USD ceiling must NOT abort the whole book
|
||
// (and re-abort on every resume): the chunk is already flagged, so keep
|
||
// the primary flag and continue. Any other infra error still aborts.
|
||
if !errors.Is(err, errReserveCeiling) {
|
||
return out, err
|
||
}
|
||
r.Log.WarnContext(ctx, "escalation hop denied by a USD ceiling; keeping the primary flag",
|
||
"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "reason", string(primary.cls.Reason))
|
||
return out, nil
|
||
}
|
||
out.attempted, out.fb = true, fb
|
||
r.Log.WarnContext(ctx, "stage escalated to a fallback model",
|
||
"stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx,
|
||
"primary_reason", string(primary.cls.Reason), "fallback", st.EscalateTo,
|
||
"fallback_disposition", string(fb.cls.Reason.disposition()))
|
||
return out, nil
|
||
}
|