255 lines
15 KiB
Go
255 lines
15 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/llm"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// cutcall.go: the money and the disposition of a call THE ENGINE ITSELF cut short (backlog row 360;
|
||
// the owner's word on the amount is D39.230 п.1). The transport half — how a cut is detected and told
|
||
// apart from an undelivered request — lives in internal/llm/attemptcut.go; this file decides what it
|
||
// COSTS and what happens to the chunk.
|
||
//
|
||
// A delivered request is generated whether or not we are still on the line. Three things end our side
|
||
// of it, they share the money, and they differ in everything else:
|
||
//
|
||
// attempt_timeout our deadline fired mid-generation. 1 call, flagged, NOT retried. Resume: 0 calls.
|
||
// cancelled a person stopped the run. 1 call, marked `cancelled`. Resume: 1 call, SAME budget.
|
||
// connection_lost the socket broke after delivery. 2 calls, then an infra pause. Resume: 1 call, SAME budget.
|
||
//
|
||
// Until this file existed all three arrived at a branch commented «No 2xx ever arrived: nothing was
|
||
// billed», which released the reservation and booked ZERO. On the provider that ships today that
|
||
// comment is false by construction: DeepSeek answers 200 with empty lines while a request waits to be
|
||
// scheduled, so a status line says nothing about whether a generation was bought.
|
||
|
||
// cutCall is the identity of the call being settled — the same tuple runAttempt already holds, passed
|
||
// as one value because thirteen positional arguments is how a chapter index and a chunk index end up
|
||
// swapped without the compiler noticing.
|
||
type cutCall struct {
|
||
stage config.Stage
|
||
chunk chunk.Chunk
|
||
job *store.Job
|
||
model string
|
||
reqHash string
|
||
resv store.Reservation
|
||
estimate float64
|
||
attempt int
|
||
escalation bool
|
||
}
|
||
|
||
// cancelledPosition is the chunk×stage a stopped run was working on. It is a struct for the reason
|
||
// cutCall is: `snapID` and `contentHash` are adjacent strings, and a swap between them compiles,
|
||
// writes a row addressed to nothing, and is found by nobody.
|
||
type cancelledPosition struct {
|
||
stage config.Stage
|
||
chunk chunk.Chunk
|
||
snapshotID string
|
||
contentHash string
|
||
cumCostUSD float64
|
||
attempts int
|
||
}
|
||
|
||
// settleUSDForCutCall is the ONE place that answers «what does a call we cut short cost».
|
||
//
|
||
// The answer is the reservation's own estimate (D39.230 п.1). Nobody knows what the provider actually
|
||
// billed, and whether it bills a cut call at all is deliberately OPEN — the paid balance probe that
|
||
// would answer it was declined — so the estimate is an upper bound under the assumption that the
|
||
// vendor charges for what it generated. Booking zero understates the book against its own ceiling and
|
||
// hands the platform a margin it cannot measure.
|
||
//
|
||
// It is a function and not a YAML field because a money policy behind a config knob can be changed by
|
||
// an operator who is not deciding a money policy, silently, between two runs of one book — leaving the
|
||
// ledger with two answers to one question and nothing on the row saying which.
|
||
func settleUSDForCutCall(estimate float64) float64 { return estimate }
|
||
|
||
// settleCutCall books the money for a delivered call the engine cut short, records the row, and
|
||
// returns the disposition that cause deserves.
|
||
//
|
||
// The money is written FIRST, before anything decides what happens to the chunk: the provider's side
|
||
// of the transaction already happened, and a process that dies between here and the verdict must leave
|
||
// the spend recorded. The checkpoint is what makes booking it possible at all — the store cannot write
|
||
// spend without one — and for two of the three causes it is also what stops the next run from buying
|
||
// the same call again.
|
||
func (r *Runner) settleCutCall(ctx context.Context, c cutCall, cut *llm.AttemptCutError, err error, att stageAttempt) (stageAttempt, error) {
|
||
// ⛔ THE MONEY IS DRAWN ON THE PROVIDER'S OWN ACKNOWLEDGEMENT, not on our write. A request whose
|
||
// bytes entered the peer's TCP window may never reach the application behind it — a load balancer
|
||
// accepts, the backend never sees it — and settling for that charges a reader for a call nobody
|
||
// ran, which is the one direction the canon forbids (D39.196 п.2а). A 2xx object in our hands is
|
||
// the acknowledgement; without it the same checkpoint is written for ZERO.
|
||
//
|
||
// The row is identical either way: same key, same class, same flag, same resume. Only the number
|
||
// and the `estimated` mark move, so an operator sees the class and the reader pays only for what a
|
||
// provider confirmed taking.
|
||
cost := 0.0
|
||
if cut.Billable {
|
||
cost = settleUSDForCutCall(c.estimate)
|
||
}
|
||
finish := string(cut.Cause)
|
||
if serr := r.Store.SettleWithCheckpoint(c.resv, cost, store.Checkpoint{
|
||
RequestHash: c.reqHash, JobID: c.job.ID, ChunkIdx: c.chunk.ChunkIdx, Attempt: c.attempt,
|
||
Stage: c.stage.Name, Role: c.stage.Role, ModelRequested: c.model, ModelActual: c.model,
|
||
ResponseText: "", UsageJSON: "{}", CostUSD: cost, FinishReason: finish,
|
||
Escalation: c.escalation,
|
||
}, r.events.spendLine()); serr != nil {
|
||
// The rule of every settle on this path: money state falling behind reality is a loud infra
|
||
// fault, never something to continue on top of.
|
||
return att, fmt.Errorf("pipeline: settle after a %s cut a delivered call: %w", finish, serr)
|
||
}
|
||
r.events.flush()
|
||
att.finish = finish
|
||
// `att` arrives carrying the money of any burned keys runAttempt walked over, and that money is
|
||
// this chunk's just as much as this call's — dropping it would under-state a position that was cut
|
||
// twice. `runCost` takes only what THIS run bought.
|
||
att.cumCost, att.runCost = att.cumCost+cost, cost
|
||
|
||
rl := r.baseRequestLog(c.stage, c.chunk, c.model, c.reqHash)
|
||
rl.CostUSD, rl.LatencyMS, rl.FinishReason = cost, att.latency, finish
|
||
rl.Degraded, rl.Err, rl.OK = finish, err.Error(), false
|
||
if cut.Deliveries > 1 {
|
||
// ⚠ THE LEDGER IS SHORT BY DESIGN HERE, AND THE ROW SAYS SO. The provider was asked to generate
|
||
// more than once inside one retry chain, and the store books spend only through a checkpoint —
|
||
// they share one key, so one settle covers all of them. Under-counting is the ratified
|
||
// direction (D39.196 п.2а: the deploy absorbs it, the user's balance is never overstated), so
|
||
// this pack does NOT quietly multiply the charge; it makes the gap readable instead, because a
|
||
// silent under-count is what row 360 was opened about.
|
||
// ⛔ THE NOTE GOES FIRST, and that is not cosmetics. The column an operator reads is bounded —
|
||
// `errTail` keeps 120 bytes — and the transport error in front of it is routinely longer than
|
||
// that, so a note appended to the end reached nobody, ever. It also says what was ACTUALLY
|
||
// booked: on a cut the provider never acknowledged, the answer is nothing, and a line claiming
|
||
// «one estimate is booked» beside a $0 row is a sentence that has to be disbelieved to be used.
|
||
rl.Err = cutErrLine(err, cut.Deliveries, cost)
|
||
}
|
||
// «Never write a silent $0 usage; estimate and FLAG it» is the one discipline every surveyed
|
||
// harness converges on (research/21 §5.3: crush `EstimatedUsage`, goose `CostSource`). The flag is
|
||
// what the operator's legend, the machine-readable pair beside committed_usd and the platform's
|
||
// «≥ X, up to Y» all read.
|
||
rl.Estimated, rl.EstTokens = cost > 0, r.estOutTokens(c.chunk.Text)
|
||
r.Store.LogRequest(ctx, r.Log, rl)
|
||
|
||
r.Log.WarnContext(ctx, "we cut a delivered call; the reservation estimate is charged as an ESTIMATE only when the provider had acknowledged it with a reply",
|
||
"stage", c.stage.Name, "chapter", c.chunk.Chapter, "chunk", c.chunk.ChunkIdx, "attempt", c.attempt,
|
||
"cause", finish, "after_headers", cut.AfterHeaders, "bytes_read", cut.BytesRead,
|
||
"whitespace_only", cut.WhitespaceOnly, "elapsed", cut.Elapsed.Round(time.Millisecond).String(),
|
||
"deliveries", cut.Deliveries, "estimate_usd", fmt.Sprintf("%.6f", cost))
|
||
|
||
if cut.Cause == llm.CutBySelfDeadline {
|
||
// Our deadline, the run is healthy: a disposition, not an infra failure. Nothing retries it —
|
||
// a retry asks the provider to generate, and bill, the very thing it is generating right now.
|
||
att.cls = classification{FlagAttemptTimeout, cut.Error()}
|
||
r.setJobStatus(ctx, c.job.ID, "done")
|
||
return att, nil
|
||
}
|
||
// A stopped run and a dead socket both end this stage. The money above is recorded either way; the
|
||
// error travels so the run pauses instead of walking the rest of the book against a dead provider.
|
||
r.setJobStatus(ctx, c.job.ID, "failed")
|
||
return att, fmt.Errorf("pipeline: stage %s call (ch%d/chunk%d, model %s): %w",
|
||
c.stage.Name, c.chunk.Chapter, c.chunk.ChunkIdx, c.model, err)
|
||
}
|
||
|
||
// cutErrLine is the sentence an operator reads when one retry chain asked the provider to generate more
|
||
// than once. It is a named function rather than a Sprintf at the call site because both of its
|
||
// properties are load-bearing and neither is visible from there.
|
||
//
|
||
// ⛔ THE NOTE COMES FIRST. The column that prints this is bounded (cmd/tmctl's errTail keeps 120 bytes)
|
||
// and the transport error is routinely longer, so a note appended at the end reached nobody — while
|
||
// being the only place the gap between what the provider generated and what the ledger booked is
|
||
// visible at all.
|
||
//
|
||
// ⛔ AND IT SAYS WHAT WAS ACTUALLY BOOKED. On a cut the provider never acknowledged, the answer is
|
||
// nothing; «one estimate is booked» printed beside a $0 row is a sentence a reader has to disbelieve
|
||
// before they can use it.
|
||
func cutErrLine(err error, deliveries int, cost float64) string {
|
||
booked := "ONE estimate is booked for all of them"
|
||
if cost <= 0 {
|
||
booked = "NOTHING is booked: the provider acknowledged none of them"
|
||
}
|
||
return fmt.Sprintf("[the provider was asked %d times; %s] %s", deliveries, booked, err.Error())
|
||
}
|
||
|
||
// recordCancelledStage marks a position whose call a HUMAN stopped, so the stop leaves no unexplained
|
||
// gap. It never changes the outcome it is called on: the run is ending, and this says what it was
|
||
// doing when it did.
|
||
//
|
||
// The mark carries its own reason. The call was healthy — the stop button ended it — so
|
||
// `flagged(cancelled)` reads as «stopped; the resume re-does it», while a shared flag would send an
|
||
// operator hunting for a defect that is not there and no mark at all would let the chapter export a
|
||
// hole nobody knows about. A flag lying about its cause is forbidden in its own right (D39.93 п.2).
|
||
//
|
||
// A write failure is logged, not returned: the caller is already returning the error that stopped the
|
||
// run, and replacing it with a bookkeeping failure would hide why the run stopped.
|
||
func (r *Runner) recordCancelledStage(ctx context.Context, p cancelledPosition, err error) {
|
||
// ⛔ THE STOP IS ASKED OF THE ERROR AS A WHOLE, and the delivery of ANY cut in it — not of whichever
|
||
// cut `errors.As` happens to reach first. A retry chain hands up the cut with the strongest money
|
||
// claim (llm's chainError), which is deliberately the EARLIER one when a later free cut would mask
|
||
// it — so a run stopped over an attempt behind a paid `connection_lost` arrives here carrying that
|
||
// cause, and a guard reading only the first cut's cause decided «this is not a stop» and returned in
|
||
// silence. The money was settled, the position got no row, and the export showed an unexplained gap:
|
||
// the hole §4.2 forbids «at any moment».
|
||
// ⚠ «A cut» IS «a delivered cut», by construction and in one place: cutError returns NIL when the
|
||
// request never went out, and the single construction of the type sets Delivered true. So one
|
||
// errors.As answers both halves, and a walk over the error tree looking for a delivered one would be
|
||
// asking a question that cannot come back different.
|
||
var cut *llm.AttemptCutError
|
||
if !errors.Is(err, context.Canceled) || !errors.As(err, &cut) {
|
||
return
|
||
}
|
||
if uerr := r.Store.UpsertChunkStatus(store.ChunkStatus{
|
||
BookID: r.Book.BookID, Chapter: p.chunk.Chapter, ChunkIdx: p.chunk.ChunkIdx, Stage: p.stage.Name,
|
||
SnapshotID: p.snapshotID, ContentHash: p.contentHash,
|
||
Disposition: string(DispFlagged), FlagReason: string(FlagCancelled),
|
||
Attempts: p.attempts, FinalHash: "", CostUSD: p.cumCostUSD,
|
||
Detail: "the run was stopped while this call was in flight; it was paid for at the reservation estimate and the resume re-does it on the same budget",
|
||
}); uerr != nil {
|
||
r.Log.ErrorContext(ctx, "could not mark the stopped position; its money is recorded but the chunk will read as never started",
|
||
"stage", p.stage.Name, "chapter", p.chunk.Chapter, "chunk", p.chunk.ChunkIdx, "err", uerr)
|
||
return
|
||
}
|
||
r.Log.WarnContext(ctx, "the run was stopped over a call that had already gone out; the position is marked cancelled and the resume re-does it on the same budget",
|
||
"stage", p.stage.Name, "chapter", p.chunk.Chapter, "chunk", p.chunk.ChunkIdx, "cost_usd", fmt.Sprintf("%.6f", p.cumCostUSD))
|
||
}
|
||
|
||
// paidAfterBurns answers «was this call already paid for AND answered», asked THE WAY THE FUNNEL WILL
|
||
// ASK IT. It is the single definition of that question for every pre-gate that decides money before a
|
||
// call, and it exists because asking it any other way has now been wrong in both directions.
|
||
//
|
||
// ⛔ A FIXED ATTEMPT INDEX CANNOT ANSWER IT. runAttempt does not stop at a burned key: it walks to the
|
||
// next index at the SAME budget and buys there, so after a stopped run a position reads «attempt 0
|
||
// burned, attempt 1 paid and answered». A probe that looks only at the starting index sees the burn and
|
||
// says «not paid» — and the caller, finding no budget left, discards a translation that was already
|
||
// bought. A probe that ignores burns says «paid» — and the caller skips its budget check while the
|
||
// funnel goes and buys the work again. Both were measured, one after the other, on this very code.
|
||
//
|
||
// So the probe walks exactly as the funnel walks (stagerun.go, the burn loop) and answers about the key
|
||
// the funnel will actually use. Two questions asked one way cannot disagree; that is the whole point of
|
||
// this function existing rather than three call sites each getting the walk right.
|
||
func (r *Runner) paidAfterBurns(st config.Stage, model, snapID string, ch chunk.Chunk, attempt, maxTokens int, msgs []llm.Message) (bool, error) {
|
||
for {
|
||
cp, err := r.Store.GetCheckpoint(RequestHash(r.attemptRequest(st, model, snapID, ch, attempt, maxTokens, msgs)))
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if cp == nil {
|
||
return false, nil // the walk ran out of keys: nothing here is paid for
|
||
}
|
||
if !burnedByCut(cp) {
|
||
return true, nil // an answer — this is what the funnel would serve for $0
|
||
}
|
||
attempt++
|
||
}
|
||
}
|
||
|
||
// resolvedForResume says a stored disposition is an ANSWER the resume may serve without calling
|
||
// anybody. Everything terminal is; `cancelled` is the one row that is not, because it records a stop
|
||
// rather than a verdict and the work behind it was never done. Reading it as terminal degenerates the
|
||
// whole construction into its opposite — never re-doing anything that was interrupted — which is the
|
||
// failure mode this design is most at risk of, since it would look perfectly green.
|
||
func resolvedForResume(cs *store.ChunkStatus) bool {
|
||
return FlagReason(cs.FlagReason) != FlagCancelled
|
||
}
|