From 3f05fab941021f727f42a2dbd9290c36694dedfc Mon Sep 17 00:00:00 2001 From: heaven Date: Thu, 10 Sep 2026 18:50:01 +0300 Subject: [PATCH] Take in the engine zone's work on calls the engine cuts itself, as the zone built it: a delivered break is asked about once, paid once, and never bought twice --- backend/cmd/tmctl/exitcontract_test.go | 44 + backend/cmd/tmctl/render.go | 12 + backend/cmd/tmctl/render_test.go | 106 ++ backend/cmd/tmmutate/mutations.json | 1192 +++++++++++++- backend/configs/models.yaml | 47 +- backend/internal/config/models.go | 69 +- .../internal/config/models_catalog_test.go | 163 ++ backend/internal/llm/attemptcut.go | 343 ++++ backend/internal/llm/attemptcut_test.go | 1330 +++++++++++++++ backend/internal/llm/httpllm.go | 270 ++- backend/internal/llm/provider_anthropic.go | 13 +- backend/internal/llm/provider_local.go | 10 +- .../internal/pipeline/burnedpregates_test.go | 1193 ++++++++++++++ backend/internal/pipeline/cutcall.go | 255 +++ backend/internal/pipeline/cutcall_test.go | 1460 +++++++++++++++++ backend/internal/pipeline/disposition.go | 112 +- backend/internal/pipeline/escalation.go | 17 +- .../internal/pipeline/flagseverity_test.go | 145 +- backend/internal/pipeline/paidtail.go | 58 + backend/internal/pipeline/priceprojection.go | 9 +- backend/internal/pipeline/repair.go | 18 +- backend/internal/pipeline/stagerun.go | 224 ++- backend/internal/pipeline/status.go | 69 +- backend/internal/pipeline/terminologist.go | 59 +- .../pipeline/testdata/operator-messages.txt | 12 +- backend/internal/pipeline/volume.go | 12 + docs/PROGRESS.md | 349 +++- .../CUT_CALLS_DOFIX_WORK_ORDER_2026-09-08.md | 172 ++ 28 files changed, 7621 insertions(+), 142 deletions(-) create mode 100644 backend/internal/llm/attemptcut.go create mode 100644 backend/internal/llm/attemptcut_test.go create mode 100644 backend/internal/pipeline/burnedpregates_test.go create mode 100644 backend/internal/pipeline/cutcall.go create mode 100644 backend/internal/pipeline/cutcall_test.go diff --git a/backend/cmd/tmctl/exitcontract_test.go b/backend/cmd/tmctl/exitcontract_test.go index 3ee98826..1d339bea 100644 --- a/backend/cmd/tmctl/exitcontract_test.go +++ b/backend/cmd/tmctl/exitcontract_test.go @@ -1,6 +1,8 @@ package main import ( + "textmachine/backend/internal/llm" + "context" "encoding/json" "errors" @@ -486,3 +488,45 @@ func TestACrashJoinedWithACeilingStillExitsAsACrash(t *testing.T) { t.Fatalf("a crash carrying a ceiling exited %d, want 1 — exit 4 would tell a supervisor to resume a process that died", code) } } + +// TestTheCutErrorTypeKeepsItsExitCode pins the linkage between the pack's new error type and the number +// a caller reads. Nothing held it: `AttemptCutError` appeared 0 times in this package's tests against 26 +// calls of the exit mapping, so every claim about «a stop exits 5, a cut exits 1» rested on reading the +// switch rather than on running it. +// +// The distinction is the whole point of the type. All three causes are one Go type, and two of them are +// ordinary failures while the third is a person pressing stop — if the mapping stopped telling them +// apart, an operator's stop would report as an engine failure, or an engine failure as a stop. +func TestTheCutErrorTypeKeepsItsExitCode(t *testing.T) { + cut := func(cause llm.CutCause, parent error) error { + return &llm.AttemptCutError{Provider: "p", Cause: cause, Delivered: true, + Err: errors.New("transport"), Parent: parent} + } + cases := []struct { + name string + err error + want int + }{ + {"a person stopped the run", cut(llm.CutByParent, context.Canceled), 5}, + {"our own deadline cut the call", cut(llm.CutBySelfDeadline, nil), 1}, + {"the socket died after delivery", cut(llm.CutByConnection, nil), 1}, + // Wrapped the way the engine actually hands it up: through fmt.Errorf and a join. + {"a stop wrapped by the pipeline", fmt.Errorf("pipeline: stage edit call: %w", + cut(llm.CutByParent, context.Canceled)), 5}, + {"a stop joined with what it interrupted", errors.Join(context.Canceled, + cut(llm.CutByConnection, nil)), 5}, + } + for _, c := range cases { + if got := exitCode(c.err); got != c.want { + t.Errorf("%s: exit %d, want %d — the three causes share one Go type and two of them are "+ + "ordinary failures while the third is a person pressing stop; a mapping that stops "+ + "telling them apart reports a stop as an engine failure or the reverse", c.name, got, c.want) + } + } + // The control: a ceiling still outranks a cut that rode along with it. A money stop is a different + // event from a transport one, and the caller's runbook branches on it. + ceiling := errors.Join(&pipeline.CeilingHalt{}, cut(llm.CutByConnection, nil)) + if got := exitCode(ceiling); got != 4 { + t.Errorf("a ceiling halt carrying a cut must still exit 4, got %d", got) + } +} diff --git a/backend/cmd/tmctl/render.go b/backend/cmd/tmctl/render.go index 7fce7818..f27606c2 100644 --- a/backend/cmd/tmctl/render.go +++ b/backend/cmd/tmctl/render.go @@ -517,6 +517,11 @@ func errTail(degraded, errText string) string { } s += errText } + // ⛔ ONE ROW IS ONE LINE. `errors.Join` separates its members with a newline, and an engine error + // that carries both a cut and what ended the chain is exactly such a join — so a table row printed + // its tail across two lines, breaking the column alignment of everything under it and pushing the + // second half where no reader looks for it. + s = strings.Join(strings.Fields(s), " ") if len(s) > 120 { cut := 120 for cut > 0 && !utf8.RuneStart(s[cut]) { @@ -875,6 +880,13 @@ func renderStatusHuman(w io.Writer, rep *pipeline.StatusReport, cfgPath string) } fmt.Fprintf(w, "Money: committed=$%.6f reserved=$%.6f%s · book forecast ~$%.6f\n", rep.CommittedUSD, rep.ReservedUSD, ceil, rep.ProjectedBookUSD) + // The estimated share of that committed figure, on the surface an operator reads before deciding + // whether to keep paying. The same pair rides in --json for the platform; printing it here too is + // what makes «committed» readable as a range rather than as a measurement. + if rep.EstimatedRows > 0 { + fmt.Fprintf(w, " of which ESTIMATED: $%.6f over %d call(s) — booked at the reservation estimate because the provider never reported usage for them (a body that would not decode, a call our own deadline or a stop cut short, a 2xx with no usage)\n", + rep.EstimatedUSD, rep.EstimatedRows) + } if rep.ETASeconds > 0 { fmt.Fprintf(w, "ETA: ~%.0fs (mean throughput of fresh calls, NOT EWMA — D12-deviation; secondary)\n", rep.ETASeconds) } diff --git a/backend/cmd/tmctl/render_test.go b/backend/cmd/tmctl/render_test.go index aaa6ed42..a19e6099 100644 --- a/backend/cmd/tmctl/render_test.go +++ b/backend/cmd/tmctl/render_test.go @@ -588,3 +588,109 @@ func TestRenderSignatureStopShowsBothDisagreements(t *testing.T) { t.Errorf("control: an unmarked row must print neither marker:\n%s", quiet.String()) } } + +// TestStatusJSONPublishesTheEstimatedShareBesideCommitted is the SEAM contract, not a formatting test. +// The platform bills a user `committed_usd` from this JSON, and until the pair below existed it could +// say «the run cost at least this much» and never «at least X, up to Y» (PD-441). The owner's word on +// charging a reader for a call we cut short came with a condition — that the estimate be visible as an +// estimate (D39.230 п.1) — and for the platform «visible» means a NUMBER, not a line of screen text. +// +// ⛔ ZERO MUST PUBLISH AS ZERO. Rendered with `omitempty` the fields would vanish on a clean book, and a +// consumer cannot tell «none of it was estimated» from «this engine is too old to know»: the safe read +// of an absent field is the pessimistic one, so a perfectly measured book would be billed as uncertain. +func TestStatusJSONPublishesTheEstimatedShareBesideCommitted(t *testing.T) { + decode := func(t *testing.T, rep *pipeline.StatusReport) map[string]any { + t.Helper() + var b strings.Builder + if err := renderStatusJSON(&b, rep); err != nil { + var flagged *pipeline.CompletedWithFlags + if !errors.As(err, &flagged) { + t.Fatalf("render: %v", err) + } + } + var decoded map[string]any + if jerr := json.Unmarshal([]byte(b.String()), &decoded); jerr != nil { + t.Fatalf("output must be valid JSON: %v", jerr) + } + return decoded + } + cut := decode(t, &pipeline.StatusReport{TotalUnits: 1, Done: 1, CommittedUSD: 0.5, EstimatedRows: 2, EstimatedUSD: 0.25}) + for field, want := range map[string]any{"committed_usd": 0.5, "estimated_usd": 0.25, "estimated_rows": float64(2)} { + if cut[field] != want { + t.Fatalf("the platform reads %q from this JSON: got %v, want %v (full: %v)", field, cut[field], want, cut) + } + } + clean := decode(t, &pipeline.StatusReport{TotalUnits: 1, Done: 1, CommittedUSD: 0.5}) + for _, field := range []string{"estimated_usd", "estimated_rows"} { + v, present := clean[field] + if !present { + t.Fatalf("%q vanished on a book with nothing estimated — a consumer cannot tell that from an "+ + "engine that does not publish it, and the safe reading of the absence is the wrong one", field) + } + if v != float64(0) { + t.Fatalf("%q on a fully measured book must be 0, got %v", field, v) + } + } +} + +// TestStatusHumanShowsTheEstimatedShare is the human twin of the JSON seam, and it had no pin at all: +// removing the whole block left both packages green. It is the surface an operator reads before +// deciding whether to keep paying, and the number beside `committed` is what makes that figure a RANGE +// rather than a measurement — the condition the owner's word came with (D39.230 п.1). +// +// Both sides, because a line that always prints is as useless as one that never does. +func TestStatusHumanShowsTheEstimatedShare(t *testing.T) { + render := func(t *testing.T, rep *pipeline.StatusReport) string { + t.Helper() + var b strings.Builder + if err := renderStatusHuman(&b, rep, "book.yaml"); err != nil { + t.Fatalf("render: %v", err) + } + return b.String() + } + withEst := render(t, &pipeline.StatusReport{TotalUnits: 1, Done: 1, CommittedUSD: 0.5, EstimatedRows: 2, EstimatedUSD: 0.25}) + if !strings.Contains(withEst, "ESTIMATED") { + t.Fatalf("an operator deciding whether to keep paying must see which part of `committed` is an "+ + "estimate; the screen said nothing:\n%s", withEst) + } + if !strings.Contains(withEst, "0.250000") || !strings.Contains(withEst, "2 call") { + t.Fatalf("the line must carry BOTH the money and the call count — one expensive call and twenty "+ + "cheap ones are different problems with the same figure:\n%s", withEst) + } + // The control: on a book with nothing estimated the line must be absent, or it becomes noise an + // operator learns to scroll past — and then it is not there on the day it matters. + clean := render(t, &pipeline.StatusReport{TotalUnits: 1, Done: 1, CommittedUSD: 0.5}) + if strings.Contains(clean, "ESTIMATED") { + t.Fatalf("a fully measured book must print no estimate line:\n%s", clean) + } +} + +// TestTheOperatorTailIsOneLineAndKeepsTheNoteThatMatters pins the column an operator actually reads. +// Two independent ways it failed: an engine error built by errors.Join carries a NEWLINE, so one table +// row printed as two and broke the alignment of everything under it; and the note the cut path appends — +// how many times the provider was asked, and what was booked — sat at the END of a transport error +// routinely longer than the 120-byte bound, so it reached nobody. +func TestTheOperatorTailIsOneLineAndKeepsTheNoteThatMatters(t *testing.T) { + long := strings.Repeat("провайдер разорвал соединение на середине тела ответа. ", 6) + note := "[the provider was asked 3 times; NOTHING is booked: the provider acknowledged none of them]" + // The newline sits right behind the note, INSIDE the bound: put it past 120 bytes and the truncation + // removes it for free, and the fixture would pass on a renderer that collapses nothing. + tail := errTail("connection_lost", note+"\n"+long) + + if strings.ContainsAny(tail, "\n\r") { + t.Fatalf("one row is one line: a joined error put a newline into the column and the table split "+ + "under it. got %q", tail) + } + if !strings.Contains(tail, "asked 3 times") { + t.Fatalf("the note is the only place the gap between what was generated and what was booked is "+ + "visible; the bound must not be what removes it. got %q", tail) + } + if !strings.Contains(tail, "NOTHING is booked") { + t.Fatalf("the note must survive whole enough to be read, not just start: %q", tail) + } + // The control: the bound is still a bound, or this test would pass on a renderer that stopped + // truncating and let a whole provider body into the table. + if len([]rune(tail)) > 121 { + t.Fatalf("the tail must still be bounded: %d runes", len([]rune(tail))) + } +} diff --git a/backend/cmd/tmmutate/mutations.json b/backend/cmd/tmmutate/mutations.json index 012b38b3..97079a8e 100644 --- a/backend/cmd/tmmutate/mutations.json +++ b/backend/cmd/tmmutate/mutations.json @@ -1771,8 +1771,8 @@ "edits": [ { "file": "internal/pipeline/status.go", - "find": "const severityUnknown = 8\n", - "replace": "const severityUnknown = 8\n\n// planted\nconst FlagPlantedElsewhere FlagReason = \"planted_elsewhere\"\n" + "find": "const severityUnknown = 9\n", + "replace": "const severityUnknown = 9\n\n// planted\nconst FlagPlantedElsewhere FlagReason = \"planted_elsewhere\"\n" } ] }, @@ -1987,8 +1987,8 @@ "edits": [ { "file": "internal/pipeline/status.go", - "find": "const severityUnknown = 8\n", - "replace": "const severityUnknown = 8\n\n// planted\nconst FlagPlantedConv = FlagReason(\"planted_conv\")\n" + "find": "const severityUnknown = 9\n", + "replace": "const severityUnknown = 9\n\n// planted\nconst FlagPlantedConv = FlagReason(\"planted_conv\")\n" } ] }, @@ -2173,8 +2173,8 @@ "edits": [ { "file": "internal/pipeline/priceprojection.go", - "find": "\t\tfor attempt := 0; attempt <= p.maxRegen; attempt++ {", - "replace": "\t\tfor attempt := 0; attempt <= 0; attempt++ {" + "find": "\t\tfor escalations := 0; escalations <= p.maxRegen; escalations++ {", + "replace": "\t\tfor escalations := 0; escalations <= 0; escalations++ {" } ] }, @@ -2233,8 +2233,8 @@ "edits": [ { "file": "internal/pipeline/stagerun.go", - "find": "\t\tr.releaseReservation(ctx, resv)\n\t\tr.setJobStatus(ctx, job.ID, \"failed\")\n\t\treturn att, err\n\t}\n\tatt.cumCost, att.runCost = cost, cost", - "replace": "\t\treturn att, err\n\t}\n\tatt.cumCost, att.runCost = cost, cost" + "find": "\t\tr.releaseReservation(ctx, resv)\n\t\tr.setJobStatus(ctx, job.ID, \"failed\")\n\t\treturn att, err\n\t}\n\t// Same as the billed-decode branch above: the burn walk's money is added, never replaced.\n\tatt.cumCost, att.runCost = burnedCost+cost, cost", + "replace": "\t\tr.setJobStatus(ctx, job.ID, \"failed\")\n\t\treturn att, err\n\t}\n\t// Same as the billed-decode branch above: the burn walk's money is added, never replaced.\n\tatt.cumCost, att.runCost = burnedCost+cost, cost" } ], "expect": "survives" @@ -3528,5 +3528,1181 @@ "replace": "\tif res.BatchesDropped > 0 || res.ClassifyBatchesDropped > 0 {\n\t\tr.Log.WarnContext(ctx, \"terminology: this bank is PARTIALLY consolidated" } ] + }, + { + "id": "CUTCALL-everything-counts-as-delivered", + "why": "the money boundary is DELIVERY. Read as always-true it settles an estimate for a request that never left the process — a TLS handshake that hung, a dead host — which is the one case where «nothing was bought» is true by construction. That is the boundary inverted: paying for calls nobody received.", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\tif !tr.delivered(answered) {\n\t\treturn nil\n\t}", + "replace": "\tif false {\n\t\treturn nil\n\t}" + } + ] + }, + { + "id": "CUTCALL-nothing-counts-as-delivered", + "why": "the same boundary read as always-false: every delivered-and-cut call goes back to booking ZERO, which is the defect backlog row 360 measured at 23–34% of an editor run. The pin has to fail on BOTH readings of one bit, or it is pinning the bit in one direction only.", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\tif !tr.delivered(answered) {\n\t\treturn nil\n\t}", + "replace": "\tif true {\n\t\treturn nil\n\t}" + } + ] + }, + { + "id": "CUTCALL-our-own-deadline-is-retried", + "why": "a call OUR deadline cut is still being generated by the provider; retrying it buys that generation a second time. The old code did exactly this because it judged truncation by SIZE, and a body our deadline cut is small.", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "func (e *AttemptCutError) retryable() bool { return e.Cause == CutByConnection }", + "replace": "func (e *AttemptCutError) retryable() bool { return e.Cause != CutByParent }" + } + ] + }, + { + "id": "CUTCALL-a-self-cut-leaves-no-resolution", + "why": "THE THIRD LEAK, and the one that is invisible without this pin: without a resolved position the resume walks back to the provider and buys the call AGAIN through the resume door, so removing the retry alone would close the class halfway. Its test carries no money assertion on purpose — a red about the ledger here would be a right verdict for the wrong reason.", + "package": "./internal/pipeline/", + "run": "TestAfterASelfCutTheResumeCallsNobody", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\tif cut.Cause == llm.CutBySelfDeadline {", + "replace": "\tif false && cut.Cause == llm.CutBySelfDeadline {" + } + ] + }, + { + "id": "CUTCALL-the-cut-costs-nothing", + "why": "the amount is the owner’s decision (D39.230 п.1): a call we cut is settled at the reservation estimate, not at zero. Booking zero understates the book against its own ceiling and hands the platform a margin it cannot measure.", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "func settleUSDForCutCall(estimate float64) float64 { return estimate }", + "replace": "func settleUSDForCutCall(estimate float64) float64 { return 0 }" + } + ] + }, + { + "id": "CUTCALL-a-stopped-run-leaves-no-mark", + "why": "the mark for a stopped position is what keeps a paid, unfinished chunk from reading as never started; inverting the stop test makes the guard mark everything EXCEPT a stop, which is the same silence in the mirror", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\tif !errors.Is(err, context.Canceled) || !errors.As(err, &cut) {", + "replace": "\tif errors.Is(err, context.Canceled) || !errors.As(err, &cut) {" + } + ] + }, + { + "id": "CUTCALL-a-cancelled-mark-is-read-as-a-verdict", + "why": "`cancelled` records a STOP, not an answer. Served as a resolved verdict the resume re-does nothing, and the construction degenerates into its opposite — never re-doing anything that was interrupted — while every money assertion stays green.", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\treturn FlagReason(cs.FlagReason) != FlagCancelled", + "replace": "\treturn true" + } + ] + }, + { + "id": "CUTCALL-a-paid-cut-checkpoint-is-replayed-as-a-result", + "why": "a `cancelled`/`connection_lost` checkpoint records money and NO result. Replayed as a completion it hands the stage an empty answer it never received, and the interrupted work is silently never re-done.", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/disposition.go", + "find": "\treturn cp.FinishReason == cancelledFinish || cp.FinishReason == connectionLostFinish\n", + "replace": "\treturn false\n" + } + ] + }, + { + "id": "CUTCALL-the-redo-doubles-the-budget", + "why": "the re-done call must carry the budget the cut one was granted, and a REGENERATION after it must be ONE doubling and not two. Keyed on the attempt index instead of the doubling count, a burn followed by a truncated answer asks for FOUR times the base and reserves the money for it. ⚠ The simple case stopped proving this the day the burn walk moved inside runAttempt — the caller's index never advances for a burn, so both readings agree there and this mutation SURVIVED a green package. Only the compound sequence (burn → length → regenerate) separates them.", + "package": "./internal/pipeline/", + "run": "TestABurnFollowedByARegenerationDoesNotOverBuy", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\t\tmaxTokens := maxTokensForAttempt(baseMaxTokens, escalations)", + "replace": "\t\tmaxTokens := maxTokensForAttempt(baseMaxTokens, attempt)" + } + ] + }, + { + "id": "CUTCALL-a-stop-forgets-what-it-interrupted", + "why": "the FIRST cancellation exit. The retry loop used to return a bare ctx.Err() there, throwing away the only evidence that the stopped run had a request ON THE WIRE — so the runner could not settle a call it was never told about, and the position was left with no mark at all.", + "package": "./internal/llm/", + "run": "TestACancelledRunKeepsBothTruthsInOneError", + "battery": true, + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\t\treturn zero, chainError(cancelledDuring(ctx.Err(), err), owedCut)", + "replace": "\t\t\treturn zero, ctx.Err()" + } + ] + }, + { + "id": "CUTCALL-a-non-2xx-becomes-a-purchase", + "why": "the status line is read BEFORE the body’s read error. Reversed, every 4xx/5xx whose small body happens to land on the deadline settles an estimate for a generation that never happened — the money boundary leaking through the one door that is supposed to be free.", + "package": "./internal/pipeline/", + "run": "TestANonTwoHundredIsNotAPurchase", + "battery": true, + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\tif resp.StatusCode < 200 || resp.StatusCode >= 300 {\n\t\tretryable := retryableStatus(resp.StatusCode, data)", + "replace": "\tif false {\n\t\tretryable := retryableStatus(resp.StatusCode, data)" + } + ] + }, + { + "id": "CUTDEADLINE-the-deadline-is-a-constant-again", + "why": "attempt_s read as THE deadline rather than as its floor is the original defect: 240 s is what the vendor rate gives for the DRAFT, and it was carried to an editor budgeted at 16 000 with a doubling to 32 000 — a call that could not finish inside it at any real speed.", + "package": "./internal/llm/", + "run": "TestTheDeadlineClampsAtBothEnds", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\td := p.deriveDeadline(maxTokens)\n\tif d < p.AttemptTimeout {", + "replace": "\td := p.AttemptTimeout\n\tif d < p.AttemptTimeout {" + } + ] + }, + { + "id": "CUTDEADLINE-attempt-s-stops-being-a-floor", + "why": "attempt_s being a FLOOR is what let this land on every shipping config at once — no provider loses a second it has today. Without the clamp a fast declared floor SHORTENS an existing deadline, and calls that fit today start being cut and billed as estimates.", + "package": "./internal/llm/", + "run": "TestTheDeadlineClampsAtBothEnds", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\tif d < p.AttemptTimeout {\n\t\td = p.AttemptTimeout\n\t}", + "replace": "\tif false {\n\t\td = p.AttemptTimeout\n\t}" + } + ] + }, + { + "id": "CUTDEADLINE-an-unset-floor-divides-by-zero", + "why": "a config that omits tok_s_floor must wait the vendor default — too long, never not at all. A zero floor read literally makes the derived deadline meaningless and would cut every call the instant it went out, billing each one as an estimate.", + "package": "./internal/llm/", + "run": "TestAnUnsetOrBrokenFloorFallsBackToTheVendorDefault", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\tif floor <= 0 {\n\t\tfloor = defaultTokensPerSecFloor()\n\t}", + "replace": "\tif floor < 0 {\n\t\tfloor = defaultTokensPerSecFloor()\n\t}" + } + ] + }, + { + "id": "CUTMONEY-the-estimated-share-publishes-as-zero", + "why": "the machine-readable estimated pair beside committed_usd is the CONDITION the owner’s word came with, and the engine half of PD-441: without it the platform bills committed_usd and can say «at least this much» but never «at least X, up to Y».", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/paidtail.go", + "find": "\t\tif tok.PromptTokens == 0 && tok.CompletionTokens == 0 && tok.ReasoningTokens == 0 {", + "replace": "\t\tif false {" + } + ] + }, + { + "id": "CUTMONEY-every-paid-row-is-called-an-estimate", + "why": "the other side of the same field. Marking every paid row estimated satisfies every positive assertion in the table and publishes a figure that says nothing — the shape a disclosure fails in quietly.", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/paidtail.go", + "find": "\t\tif tok.PromptTokens == 0 && tok.CompletionTokens == 0 && tok.ReasoningTokens == 0 {", + "replace": "\t\tif true {" + } + ] + }, + { + "id": "CUTWARN-the-default-floor-notice-is-never-said", + "why": "a provider with no measured speed runs on the vendor default and may wait a quarter of an hour. That is legitimate and must never be a surprise: the notice is what makes a new provider usable as DATA rather than as a mystery.", + "package": "./internal/llm/", + "run": "TestTheFloorWarningIsSaidOnceAndOnlyWhenItApplies", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\tif c.profile.TokensPerSecFloor <= 0 && d > c.profile.AttemptTimeout {", + "replace": "\tif false {" + } + ] + }, + { + "id": "CUTWARN-the-notice-is-said-for-a-provider-that-does-not-need-it", + "why": "the silent half. A notice printed for a provider whose calls all fit inside its own attempt_s is a warning about nothing, and an operator learns to scroll past the line that was supposed to explain a fifteen-minute wait.", + "package": "./internal/llm/", + "run": "TestTheFloorWarningIsSaidOnceAndOnlyWhenItApplies", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\tif c.profile.TokensPerSecFloor <= 0 && d > c.profile.AttemptTimeout {", + "replace": "\tif c.profile.TokensPerSecFloor <= 0 {" + } + ] + }, + { + "id": "CUTWAIT-the-waiting-line-is-never-said", + "why": "with a deadline derived from the budget a call may legitimately run for a quarter of an hour, and one «calling model» line then leaves an operator watching a silence indistinguishable from a wedged process.", + "package": "./internal/pipeline/", + "run": "TestTheWaitSaysSoWhileItLastsAndIsSilentOtherwise", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\tstopHeartbeat := r.logWaitingForProvider(ctx, st, ch, model, attempt, maxTokens)", + "replace": "\tstopHeartbeat := func() {}" + } + ] + }, + { + "id": "CUTWAIT-the-interval-ignores-the-wait-it-reports", + "why": "the heartbeat is paced off the wait the engine actually granted. Fixed instead of derived, the line exists only for calls longer than the constant — which is every call nobody can observe and no test can afford to watch.", + "package": "./internal/pipeline/", + "run": "TestTheHeartbeatIsPacedByTheWaitItReports", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\tq := deadline / 4\n", + "replace": "\tq := deadline\n" + } + ] + }, + { + "id": "CUTCATALOG-a-long-waiting-provider-declares-nothing", + "why": "the catalogue gate is what makes «a provider the repository has never seen» work as DATA: one whose declared floors make its calls outgrow its own attempt_s must name either the speed it holds or how long we will wait. Dropped, the config says 240 s while the engine waits twenty minutes.", + "package": "./internal/config/", + "run": "TestAProviderThatWillWaitLongSaysSoInTheCatalog", + "battery": true, + "edits": [ + { + "file": "configs/models.yaml", + "find": ", tok_s_floor: 50, queue_slack_s: 600, attempt_max_s: 1240 }", + "replace": " }" + } + ] + }, + { + "id": "CUTSEAM-the-estimated-pair-vanishes-on-a-clean-book", + "why": "the platform bills committed_usd from status --json and needs the estimated pair BESIDE it to say «≥ X, up to Y» instead of only «≥» (PD-441; the condition on D39.230 п.1). With omitempty the fields disappear exactly when the answer is «none of it was estimated», and a consumer cannot tell that from an engine too old to publish them — so a fully measured book reads as uncertain.", + "package": "./cmd/tmctl/", + "run": "TestStatusJSONPublishesTheEstimatedShareBesideCommitted", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/status.go", + "find": "\tEstimatedRows int `json:\"estimated_rows\"`\n\tEstimatedUSD float64 `json:\"estimated_usd\"`", + "replace": "\tEstimatedRows int `json:\"estimated_rows,omitempty\"`\n\tEstimatedUSD float64 `json:\"estimated_usd,omitempty\"`" + } + ] + }, + { + "id": "CUTFLAG-a-cut-chunk-is-the-mildest-thing-that-can-happen", + "why": "a flag with no entry in flagSeverity falls to severityUnknown, which is LAST — so a chapter whose worst problem is «we paid for a call, our own deadline cut it and the text is gone» would report any other flag as its worst. Measured on this very pack: both new flags landed there while the exhaustiveness gate stayed GREEN, because it could not read a FlagReason declared as a conversion of another package's constant (18 ranks against 16 constants it could see).", + "package": "./internal/pipeline/", + "run": "TestEveryFlagReasonIsRanked", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/status.go", + "find": "\tFlagDecodeError: 4,\n\tFlagAttemptTimeout: 4,\n", + "replace": "\tFlagDecodeError: 4,\n" + } + ] + }, + { + "id": "CUTFLAG-the-rank-gate-goes-blind-to-a-converted-constant", + "why": "the gate's own guard. A FlagReason whose value is a conversion of another package's constant has no literal to unquote, and the walk used to `continue` past it — a silent skip inside an exhaustiveness test, which is the very defect the test exists to catch, one level up. Removing the resolver must make it FAIL LOUD about the unreadable value, never go quiet again.", + "package": "./internal/pipeline/", + "run": "TestEveryFlagReasonIsRanked", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/flagseverity_test.go", + "find": "\t\"llm.CutBySelfDeadline\": FlagReason(llm.CutBySelfDeadline),\n", + "replace": "" + } + ] + }, + { + "id": "CUTCALL-a-word-is-enough-to-burn-a-paid-answer", + "why": "the finish_reason string shares a namespace with whatever a vendor prints — the adapter already normalises invented values like «sensitive» — so keying the burn on the word alone throws away a real translation that happened to arrive under one of our own names, and buys it again. A genuine burn is written by this engine and is always textless.", + "package": "./internal/pipeline/", + "run": "TestABurnIsMoneyWithoutAnAnswerAndNotJustAWord", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/disposition.go", + "find": "\tif cp.ResponseText != \"\" {\n\t\treturn false // a reply with content is an answer, whatever it calls its finish reason\n\t}\n", + "replace": "" + } + ] + }, + { + "id": "CUTVOLUME-a-stopped-position-is-counted-as-a-free-resume", + "why": "the volume planner asks «does this row resume without a provider call» with its own copy of the predicate. Not knowing about `cancelled`, it counts a unit whose last row is one as costing nothing — so a grant of ONE unit pays for two, and the volume report, which speaks only when something was carried, says nothing at all. Two copies of one question is how the disclosure went silent.", + "package": "./internal/pipeline/", + "run": "TestACancelledRowIsNotAFreeResume", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/volume.go", + "find": "\t\tif !resolvedForResume(&cs) {\n\t\t\treturn false\n\t\t}\n", + "replace": "" + } + ] + }, + { + "id": "CUTCALL-a-reply-that-outruns-our-own-write-books-zero", + "why": "net/http hands back a response before WroteRequest fires (Request.write defers it onto the write loop; roundTrip returns on the response channel), so a provider answering EARLY — which DeepSeek documents doing while a request waits to be scheduled — can have its 200 overtake our own callback while the request body still drains. Read as «not delivered» that call books $0 and is re-asked under the full attempt budget: the very leak this pack closes, entering through its own door.", + "package": "./internal/llm/", + "run": "TestAReplyThatOutrunsOurOwnWriteIsStillDelivered", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\treturn t.wrote.Load() || (answered && t.firstByte.Load())", + "replace": "\treturn t.wrote.Load()" + } + ] + }, + { + "id": "CUTCALL-a-failed-write-counts-as-a-delivery", + "why": "`WroteRequest` fires with an ERROR when the write itself failed: the provider received nothing, generated nothing and owes nothing, so settling an estimate charges a reader for a call nobody got. The guard had no pin at all until an adversarial pass removed it and watched both packages stay green.", + "package": "./internal/llm/", + "run": "TestAFailedWriteIsNotADelivery", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\t\t\tif info.Err == nil {\n\t\t\t\tt.wrote.Store(true)\n\t\t\t}", + "replace": "\t\t\tt.wrote.Store(true)" + } + ] + }, + { + "id": "CUTCEILING-a-cancelled-run-reports-a-money-stop", + "why": "a run that is ENDING did not stop on money, whatever the ledger says at that instant. Paying for a cut call (D39.230 п.1) made this reachable: a stop commits every flying call's estimate, which can carry a book past its own ceiling, and the next worker then publishes a `ceiling` event with a shortfall. The platform lets that event survive any exit code, so it records `paused` and tells a person to add money for a run that person stopped themselves. Caught by the existing wait-cancellation pin only intermittently — the guard first sat at the top of the loop, and waitForSettle answers «nothing in flight» BEFORE it looks at the context, so a worker fell through it to the halt.", + "package": "./internal/pipeline/", + "run": "TestACancelledRunNeverReportsAMoneyStop", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\t\tif ctx.Err() != nil {\n\t\t\tr.setJobStatus(ctx, job.ID, \"failed\")\n\t\t\treturn att, fmt.Errorf(\"pipeline: reserve $%.6f for %s/ch%d/chunk%d/%s: the run ended before the reservation was granted: %w\",\n\t\t\t\testimate, r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, ctx.Err())\n\t\t}\n", + "replace": "" + } + ] + }, + { + "id": "CUTCALL-a-stop-during-a-backoff-forgets-what-it-interrupted", + "why": "the SECOND cancellation exit. A run stopped during a retry backoff has a delivered, possibly billed attempt behind it — a cut connection, an undecodable 2xx — and a bare ctx.Err() there leaves the runner nothing to settle and the chunk no mark at all. Measured: a delivered connection_lost plus a stop inside the backoff booked $0 and wrote no chunk_status row. The window is the whole sleep, up to a minute on the shipping config, and it opens exactly where a flapping provider makes an operator reach for the stop.", + "package": "./internal/llm/", + "run": "TestAStopDuringABackoffKeepsTheEvidence", + "battery": true, + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\t\treturn zero, chainError(cancelledDuring(ctx.Err(), lastErr), owedCut)", + "replace": "\t\t\treturn zero, ctx.Err()" + } + ] + }, + { + "id": "CUTMONEY-money-a-redrive-unaccounted-for-reads-as-measured", + "why": "a redrive DELETES the checkpoints of the stages it re-attacks and leaves their spend committed («after a redrive committed(spend) >= SUM(checkpoints), the safe direction», store/chunkstatus.go). Derived from the surviving rows alone the published estimate falls to zero, and the platform is told that money nobody can account for was MEASURED — the one direction this figure exists to prevent. Measured on a live redrive: committed unchanged at $0.001056, estimated dropped to $0.", + "package": "./internal/pipeline/", + "run": "TestMoneyWithNoCheckpointLeftIsStillAnEstimate", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/paidtail.go", + "find": "\tif gap := committedUSD - accounted; gap > 1e-9 {\n\t\trows, usd = rows+1, usd+gap\n\t}\n", + "replace": "" + } + ] + }, + { + "id": "CUTCALL-a-refusal-pays-when-the-reply-outruns-the-write", + "why": "the regression the early-200 fix introduced, and the more expensive direction of the same bit. A provider that REFUSES (401/403/413/quota-429) and resets while our body is still writing gives GotFirstResponseByte=true and a WRITE error, so http.Client.Do returns the write failure and the status line is never in our hands. Taking the response byte alone as proof of delivery paid an estimate for every one: measured 22 refusals in 25, one of them $0.80 for a request the provider declined — and the delivered-cut retry sent the whole body again.", + "package": "./internal/llm/", + "run": "TestARefusalIsNotAPurchaseEvenWhenTheReplyOutrunsTheWrite", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\treturn t.wrote.Load() || (answered && t.firstByte.Load())", + "replace": "\treturn t.wrote.Load() || t.firstByte.Load()" + } + ] + }, + { + "id": "CUTVOLUME-a-retired-stage-costs-a-slot", + "why": "the per-row walk drops the stages this pipeline no longer runs — nothing will call them, so they cannot cost anything. Asking «does this row resume for free» BEFORE that drop makes a unit paid for a `cancelled` row of a retired stage, so an operator who edits the pipeline over a stopped run spends volume slots on calls that will never be made. Same error as the one this predicate fixed, in the opposite direction.", + "package": "./internal/pipeline/", + "run": "TestADroppedStageIsFreeEvenWhenItsRowSaysCancelled", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/volume.go", + "find": "\t\tvar cur string\n\t\tvar w wave", + "replace": "\t\tif !resolvedForResume(&cs) {\n\t\t\treturn false\n\t\t}\n\t\tvar cur string\n\t\tvar w wave" + } + ] + }, + { + "id": "CUTCALL-a-redirect-carries-delivery-to-the-next-hop", + "why": "`Do` spans the WHOLE redirect chain and the delivery trace does not reset between its legs, so the first leg reaching a redirector sets WroteRequest for good — and a second leg whose connect is REFUSED still looked delivered. Measured through the ledger: $0.001056 booked for a `connect: connection refused` that never put a byte on any wire, AfterHeaders true, zero bytes from the target. A stale `http://` in base_url is enough to trigger it; following the redirect at all also carries an Authorization header to a host nobody chose.", + "package": "./internal/llm/", + "run": "TestARedirectIsNotADelivery", + "battery": true, + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\treturn &http.Client{Transport: base, CheckRedirect: doNotFollowRedirects}", + "replace": "\treturn &http.Client{Transport: base}" + } + ] + }, + { + "id": "CUTWAIT-the-waiting-line-claims-a-delivery-it-never-checked", + "why": "the runner cannot see the transport's trace, so «a call that has been delivered» was an assertion nobody had asked. On a request that never went out it printed eighteen times, telling the operator the opposite of what happened — the same defect as a flag naming the wrong cause (D39.93 п.2).", + "package": "./internal/pipeline/", + "run": "TestTheWaitSaysSoWhileItLastsAndIsSilentOtherwise", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\"still waiting for the provider on a call in flight\"", + "replace": "\"still waiting for the provider on a call that has been delivered\"" + } + ] + }, + { + "id": "CUTCALL-the-local-client-still-follows-redirects", + "why": "the redirect guard first went only on the cloud client, whose own comment says the local provider «passes its OWN no-proxy client, so it is untouched» — true of keepalive, and read as permission for redirects too. Both go through the same attempt() and the same delivery trace, so a leg whose connect is refused still looks delivered. Measured on the local client after the cloud one was fixed: an AttemptCutError for a `connect: connection refused` with two hops. The money is $0 there only because the local model is priced at zero; the behaviour is wrong either way.", + "package": "./internal/llm/", + "run": "TestARedirectIsNotADelivery", + "battery": true, + "edits": [ + { + "file": "internal/llm/provider_local.go", + "find": "\treturn &http.Client{Transport: &http.Transport{Proxy: nil}, CheckRedirect: doNotFollowRedirects}", + "replace": "\treturn &http.Client{Transport: &http.Transport{Proxy: nil}}" + } + ] + }, + { + "id": "CUTWAIT-a-per-attempt-deadline-is-printed-as-the-whole-wait", + "why": "the heartbeat wraps client.Complete — the WHOLE retry chain — while the only deadline it can name is one ATTEMPT's. Under the key `of` that printed «waited=4s of=1s», a contradiction that reads as exactly the hung process the line exists to rule out; measured at 18 such lines in one run. The key `attempt` was a second half of the same: the transport logs an `attempt` of its own, and two different numbers under one name in one stream is a question an operator cannot answer.", + "package": "./internal/pipeline/", + "run": "TestTheWaitSaysSoWhileItLastsAndIsSilentOtherwise", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\"stage_attempt\", attempt, \"waited\", time.Since(started).Round(time.Millisecond).String(),\n\t\t\t\t\t\"attempt_deadline\", r.Models.AttemptDeadline(model, maxTokens).String())", + "replace": "\"attempt\", attempt, \"waited\", time.Since(started).Round(time.Millisecond).String(),\n\t\t\t\t\t\"of\", r.Models.AttemptDeadline(model, maxTokens).String())" + } + ] + }, + { + "id": "CUTCALL-the-anthropic-client-carries-its-key-through-a-redirect", + "why": "the third client this package builds, and the one where the redirect is a CREDENTIAL leak rather than a money one: it sends `x-api-key`, and net/http strips only Authorization/Cookie/WWW-Authenticate across a host change. Measured: the redirect target received the key verbatim and the call returned a nil error. The adapter is deprecated with no live provider, but clients.go builds it for any config declaring `kind: anthropic`.", + "package": "./internal/llm/", + "run": "TestARedirectIsNotADelivery", + "battery": true, + "edits": [ + { + "file": "internal/llm/provider_anthropic.go", + "find": "\t\thttp: &http.Client{CheckRedirect: doNotFollowRedirects},", + "replace": "\t\thttp: &http.Client{}," + } + ] + }, + { + "id": "CUTMONEY-the-operator-screen-drops-the-estimated-share", + "why": "the human twin of the status --json seam, and it had no pin at all — removing the block left both packages green while renderStatusHuman itself is 76% covered. It is the surface an operator reads before deciding whether to keep paying, and the number beside `committed` is what makes that figure a RANGE rather than a measurement (the condition attached to D39.230 п.1).", + "package": "./cmd/tmctl/", + "run": "TestStatusHumanShowsTheEstimatedShare", + "battery": true, + "edits": [ + { + "file": "cmd/tmctl/render.go", + "find": "\tif rep.EstimatedRows > 0 {", + "replace": "\tif false {" + } + ] + }, + { + "id": "CUTCALL-a-stop-over-the-hop-leaves-no-mark", + "why": "the cancelled-position mark lived at the attempt loop's error return, and the escalation hop, the repair sub-step and the bank batches all leave runStage through OTHER returns. Measured by an acceptance verifier: a run stopped over the HOP settled the money and left chunk_status_rows=0 — the unmarked hole §4.2 forbids — while the resume made ZERO fresh calls because the hop addresses a fixed attempt 0 and the burned key there is hit forever.", + "package": "./internal/pipeline/", + "run": "TestAStopOverAnEscalationHopLeavesAMarkAndIsRedone", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\tdefer func() {\n\t\tr.recordCancelledStage(ctx, cancelledPosition{\n\t\t\tstage: st, chunk: ch, snapshotID: snapID, contentHash: contentHash,\n\t\t\tcumCostUSD: cumCost, attempts: attemptsMade,\n\t\t}, err)\n\t}()\n", + "replace": "" + } + ] + }, + { + "id": "CUTCALL-a-burned-key-is-spent-only-inside-the-loop", + "why": "walking over a burned key first lived in runStage's loop, leaving the three callers OUTSIDE it with a key burnt forever — the hop most visibly, since it addresses a fixed attempt 0. Moving the walk into runAttempt gives every caller the rule; putting it back into the loop alone re-opens «money spent, work never re-done» for the hop, the repair sub-step and the bank batches at once.", + "package": "./internal/pipeline/", + "run": "TestAStopOverAnEscalationHopLeavesAMarkAndIsRedone", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\t\tif cp == nil || !burnedByCut(cp) {\n\t\t\tbreak\n\t\t}", + "replace": "\t\tif cp == nil || true {\n\t\t\tbreak\n\t\t}" + } + ] + }, + { + "id": "CUTCALL-a-cut-checkpoint-is-written-under-the-wrong-key", + "why": "§4.3's real claim: the resume finds the CHECKPOINT of a self-cut. The ordinary resume resolves from chunk_status before any render, so a pin that stops there proves the first door shut and says nothing about the second — an acceptance verifier planted this and it SURVIVED a green package. The torn-store fixture enters through the door the checkpoint exists for.", + "package": "./internal/pipeline/", + "run": "TestASelfCutResumesFromITSCHECKPOINTAndNotOnlyFromChunkStatus", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\t\tRequestHash: c.reqHash, JobID: c.job.ID", + "replace": "\t\tRequestHash: c.reqHash + \"z\", JobID: c.job.ID" + } + ] + }, + { + "id": "CUTCALL-classify-forgets-what-a-cut-checkpoint-means", + "why": "a replayed cut checkpoint with no verdict falls through to «empty completion» — and `empty` is RETRYABLE, so the chunk is re-bought on a DOUBLED budget. The chunk_status door hides it: this mutation SURVIVED on a green package until a fixture reached the checkpoint path directly.", + "package": "./internal/pipeline/", + "run": "TestASelfCutResumesFromITSCHECKPOINTAndNotOnlyFromChunkStatus", + "battery": true, + "edits": [ + { + "file": "internal/pipeline/disposition.go", + "find": "\tcase attemptTimeoutFinish:", + "replace": "\tcase attemptTimeoutFinish + \"-dead\":" + } + ] + }, + { + "id": "CUTCALL-a-delivered-cut-is-erased-by-a-later-terminal-status", + "why": "only the LAST error left the retry loop, so a chain that cut a delivered request and then met a terminal 4xx returned the status alone: the runner read «the request never went out», released the reservation and booked $0 for a generation the provider had made, leaving the position unmarked. Reachable in the ordinary way — a 400/401/403/413 on the retry after a broken socket.", + "package": "./internal/llm/", + "run": "TestADeliveredCutSurvivesATerminalStatusLaterInTheChain", + "battery": true, + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\t\treturn zero, chainError(err, owedCut)", + "replace": "\t\t\treturn zero, err" + } + ] + }, + { + "id": "CUTCALL-the-delivery-count-is-not-carried", + "why": "a broken connection is worth one retry, so the provider is asked to generate TWICE while the store books ONE estimate — they share a key and spend is written only through a checkpoint. Under-counting is the ratified direction (D39.196 п.2а), but a SILENT under-count is what row 360 was opened about: the count is what makes the gap readable in the ledger row and the log.", + "package": "./internal/llm/", + "run": "TestABrokenConnectionAfterDeliveryIsRetriedOnceAndOnlyOnce", + "battery": true, + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\t\t\tchainCuts = append(chainCuts, cut)\n", + "replace": "" + } + ] + }, + { + "id": "CUTMONEY-a-write-into-the-peers-window-is-taken-as-a-purchase", + "why": "`WroteRequest` says our bytes entered the peer's TCP window; it says NOTHING about the application behind it — a load balancer accepts while the backend never sees the request. Measured on a stopped run: 3 of 25 cancelled in-flight calls settled an estimate for a request no handler ever entered. On a non-streaming wire the only signal a provider's application acknowledged the request is a 2xx object reaching us, so money is drawn on that; delivery still drives RETRY. Charging a reader for a call nobody ran is the one direction the canon forbids (D39.196 п.2а).", + "package": "./internal/pipeline/", + "run": "TestAnAcceptedSocketThatNobodyReadCostsNothing", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\t\tBillable: answered && tr.afterHeaders(),", + "replace": "\t\tBillable: true," + } + ] + }, + { + "id": "CUTMONEY-an-acknowledged-cut-stops-being-paid", + "why": "the other side of the same boundary: a call the provider DID acknowledge with a 2xx and then had cut short is exactly what row 360 was opened about, and booking zero for it takes the engine back to «No 2xx ever arrived: nothing was billed» — the comment that was false by construction on the shipping provider.", + "package": "./internal/pipeline/", + "run": "TestTheNineOutcomesOfACall", + "battery": true, + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\t\tBillable: answered && tr.afterHeaders(),", + "replace": "\t\tBillable: false," + } + ] + }, + { + "id": "CUTKEEPALIVE-the-read-idle-bound-is-not-installed", + "why": "the h2 keepalive pair is the reason a long thinking-heavy call survives an idle proxy: with no DATA frame flowing, a PING is what keeps the socket open, and a dead connection then surfaces as a clean retryable transport error instead of a mid-generation RST that re-bills the call. Wiring the bound nowhere leaves the comment true and the transport bare.", + "package": "./internal/llm/", + "run": "TestTheKeepalivePairSitsOnTheTransportTheCloudClientBuilds", + "battery": true, + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\th2.ReadIdleTimeout = h2ReadIdleTimeout\n", + "replace": "" + } + ] + }, + { + "id": "CUTBURN-the-paid-contract-ignores-the-burn", + "why": "with the burn ignored, ANY row at the key reads as paid — and the pre-gates then skip the budget check while the funnel, which cannot replay a burned row, goes and buys the work again outside every bound", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget|TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget|TestTheBankPaidProbeSeesThroughABurnedCheckpoint", + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\t\tif !burnedByCut(cp) {\n", + "replace": "\t\tif cp != nil {\n" + } + ] + }, + { + "id": "CUTBURN-the-paid-contract-stops-at-the-first-key", + "why": "the funnel does not stop at a burned key — it walks to the next index at the same budget and buys there, so the answer to «already paid» often sits one index up. A probe that stops at the starting key hides a PAID, ANSWERED call, and the caller with no budget left discards work it already bought", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAPaidHopBehindABurnedKeyStillReplaysFree|TestAPaidRepairBehindABurnedKeyStillReplaysFree|TestTheBankProbeFindsThePaidBatchBehindABurnedKey", + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\t\tattempt++\n\t}\n}\n", + "replace": "\t\treturn false, nil\n\t}\n}\n" + } + ] + }, + { + "id": "CUTBURN-the-repair-pregate-asks-a-fixed-index", + "why": "the one definition of «already paid» exists so three pre-gates cannot drift apart; a site that goes back to asking a fixed attempt index answers a different question from the funnel and discards a paid repair", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAPaidRepairBehindABurnedKeyStillReplaysFree", + "edits": [ + { + "file": "internal/pipeline/repair.go", + "find": "\treturn r.paidAfterBurns(rst, model, snapID, ch, ordinal, maxTokens, msgs)\n", + "replace": "\tcp, err := r.Store.GetCheckpoint(RequestHash(r.attemptRequest(rst, model, snapID, ch, ordinal, maxTokens, msgs)))\n\treturn cp != nil && !burnedByCut(cp), err\n" + } + ] + }, + { + "id": "CUTBURN-the-escalation-pregate-asks-a-fixed-index", + "why": "same drift on the hop: a fixed index hides a paid, successful hop behind a burned key, and the exhausted escalation budget then throws that translation away on every resume", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAPaidHopBehindABurnedKeyStillReplaysFree", + "edits": [ + { + "file": "internal/pipeline/escalation.go", + "find": "\tmayHop, err := r.paidAfterBurns(st, st.ResolvedHop, snapID, ch, 0, hopMaxTokens, msgs)\n", + "replace": "\tfbCP, err := r.Store.GetCheckpoint(RequestHash(r.attemptRequest(st, st.ResolvedHop, snapID, ch, 0, hopMaxTokens, msgs)))\n\tmayHop := fbCP != nil && !burnedByCut(fbCP)\n" + } + ] + }, + { + "id": "CUTBURN-the-bank-pregate-asks-a-fixed-index", + "why": "same drift on the terminology batch, the site the other two were written from: a fixed index hides the batch the funnel already bought one index up, and the role sub-budget refuses to serve work it has paid for", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestTheBankProbeFindsThePaidBatchBehindABurnedKey", + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\treturn r.paidAfterBurns(st, st.Model, snapID, ch, 0, maxTokens, msgs)\n", + "replace": "\tcp, err := r.Store.GetCheckpoint(RequestHash(r.attemptRequest(st, st.Model, snapID, ch, 0, maxTokens, msgs)))\n\treturn cp != nil && !burnedByCut(cp), err\n" + } + ] + }, + { + "id": "CUTKEEPALIVE-the-cloud-client-is-not-tuned-at-all", + "why": "the keepalive bounds are only worth their comment if the CLIENT THE ENGINE BUILDS carries them; a check written against a transport the test tuned itself passes even when the construction stopped tuning anything, which is exactly how the write bound's pin survived its own mutation", + "package": "./internal/llm/", + "run": "TestTheKeepalivePairSitsOnTheTransportTheCloudClientBuilds", + "battery": true, + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\th2 := tuneHTTP2(base)\n", + "replace": "\tvar h2 *http2.Transport\n" + } + ] + }, + { + "id": "CUTBANK-admission-is-a-prefix-again", + "why": "an already-paid batch costs nothing and is admitted whatever the budget says, so a refused batch can sit in FRONT of batches that are free to serve; a prefix bound drops every paid batch behind the first refusal, buying nothing and losing a consolidated bank — while the loop's own comment promises they are admitted regardless", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAnUnaffordableBatchDoesNotTakeThePaidBatchesBehindIt", + "edits": [ + { + "file": "internal/pipeline/terminologist.go", + "find": "\t\t\tcontinue // this one cannot be afforded; the ones after it may still be free\n", + "replace": "\t\t\tbreak\n" + } + ] + }, + { + "id": "CUTCHAIN-the-exit-ranks-cuts-by-type-not-money", + "why": "the exit hands the caller ONE error; deciding which cut it carries by TYPE returns the ending cut whatever it cost, so a free cut nobody acknowledged erases the paid one behind it and the engine books $0", + "package": "./internal/llm/", + "battery": true, + "run": "TestAFreeCutLaterDoesNotMaskThePaidCutEarlier", + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\tif errors.As(owed, &oc) && oc.Billable && !fc.Billable {\n", + "replace": "\t\tif errors.As(owed, &oc) && false {\n" + } + ] + }, + { + "id": "CUTCHAIN-the-owed-cut-is-joined-last", + "why": "errors.As hands back the FIRST match it meets walking the tree, so joining the free ending cut ahead of the paid owed one leaves the caller settling from the free one — the masking the branch exists to undo, restored by an argument order", + "package": "./internal/llm/", + "battery": true, + "run": "TestAFreeCutLaterDoesNotMaskThePaidCutEarlier|TestTheCancellationExitCarriesWhatTheChainOwes", + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\t\treturn errors.Join(owed, final)\n", + "replace": "\t\t\treturn errors.Join(final, owed)\n" + } + ] + }, + { + "id": "CUTCHAIN-the-stop-exit-drops-what-is-owed", + "why": "a run stopped right after an attempt can have a paid cut behind it and an ordinary retryable in front; returning only «cancelled + the last error» leaves the runner nothing to settle and the position unmarked", + "package": "./internal/llm/", + "battery": true, + "run": "TestTheCancellationExitCarriesWhatTheChainOwes", + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\t\treturn zero, chainError(cancelledDuring(ctx.Err(), err), owedCut)\n", + "replace": "\t\t\treturn zero, cancelledDuring(ctx.Err(), err)\n" + } + ] + }, + { + "id": "CUTCHAIN-a-plain-retryable-erases-the-paid-cut", + "why": "the chain can be billed for an attempt that is not the one whose error ends it; handing the caller only the terminal 503 books $0 for a generation the provider made", + "package": "./internal/llm/", + "battery": true, + "run": "TestAPaidCutSurvivesAPlainRetryableLaterInTheChain", + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\treturn zero, chainError(fmt.Errorf(\"%s: exhausted %d attempts: %w\", name, profile.MaxAttempts, lastErr), owedCut)\n", + "replace": "\treturn zero, fmt.Errorf(\"%s: exhausted %d attempts: %w\", name, profile.MaxAttempts, lastErr)\n" + } + ] + }, + { + "id": "CUTROW-the-burn-money-is-overwritten", + "why": "the funnel walks past a burned key and buys the work again one index up; the row must carry BOTH the money that bought nothing and the money that bought the text. Overwriting the walk's total with the fresh call's cost puts chunk_status.cost_usd below SUM(checkpoints) for the position — money the book spent and the projection does not show", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestABurnedKeysMoneyReachesTheRow", + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\tatt.cumCost, att.runCost = burnedCost+cost, cost\n", + "replace": "\tatt.cumCost, att.runCost = cost, cost\n" + } + ] + }, + { + "id": "CUTROW-the-hop-money-waits-for-the-verdict", + "why": "a hop that was CUT reports its cost and an error at the same time; adding the cost only on the success path leaves the deferred cancelled mark carrying the primary's money alone — measured at ledger 0.001176 against a row of 0.000120", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAStoppedRunLeavesEveryRowMatchingItsOwnLedger", + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\tcumCost += esc.fb.cumCost\n\trunCost += esc.fb.runCost\n\tanyFresh = anyFresh || esc.fb.freshCall\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif esc.attempted {\n\t\tescalated = true\n", + "replace": "\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif esc.attempted {\n\t\tcumCost += esc.fb.cumCost\n\t\trunCost += esc.fb.runCost\n\t\tanyFresh = anyFresh || esc.fb.freshCall\n\t\tescalated = true\n" + } + ] + }, + { + "id": "CUTROW-the-repair-money-waits-for-the-verdict", + "why": "same shape on the repair sub-step: a repair call the engine cut reports its cost together with the error, and the row is the only place that cost can land", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAStoppedRunCarriesTheRepairsMoneyToTheRow", + "edits": [ + { + "file": "internal/pipeline/repair.go", + "find": "\t\tres.CostUSD += att.runCost\n\t\tres.CumUSD += att.cumCost\n\t\tres.Fresh = res.Fresh || att.freshCall\n\t\tif err != nil {\n", + "replace": "\t\tif err != nil {\n" + } + ] + }, + { + "id": "CUTSTATE-a-stopped-position-decides-the-unit", + "why": "a cancelled row records a position the engine was stopped over: the work was never done and the resume re-does it and pays again. Reading it as a decided unit puts it in the extrapolation's denominator, so one stop makes the book look further along and cheaper than it is — and the re-bill consent threshold, min($0.50, 5% x projected), shrinks with it", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAStoppedPositionIsNotADecidedUnit", + "edits": [ + { + "file": "internal/pipeline/status.go", + "find": "\t\t\tif !resolvedForResume(&cs) {\n\t\t\t\tcontinue\n\t\t\t}\n", + "replace": "" + } + ] + }, + { + "id": "CUTROW-the-attempt-count-waits-for-the-verdict", + "why": "the deferred cancelled mark reports the attempt count beside money that was really paid; writing the count only after the error check leaves `attempts=0` on a position whose ledger says 0.001056 — a row that reports work nobody did, and the number an operator reads to tell «one call was cut» from «the position never started»", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAStopOnAStagesFirstCallStillReportsAnAttempt", + "edits": [ + { + "file": "internal/pipeline/stagerun.go", + "find": "\t\tattempt = att.attempt\n\t\tattemptsMade = attempt + 1\n\t\tif err != nil {\n\t\t\treturn nil, err // infra failure; the cancelled-position mark is the defer above\n\t\t}\n", + "replace": "\t\tif err != nil {\n\t\t\treturn nil, err // infra failure; the cancelled-position mark is the defer above\n\t\t}\n\t\tattempt = att.attempt\n\t\tattemptsMade = attempt + 1\n" + } + ] + }, + { + "id": "CUTMONEY-response-bytes-alone-mean-payment", + "why": "a cut is billable only when the provider ACKNOWLEDGED the request with a reply; dropping the `answered` half makes response BYTES enough, so a broken proxy writing garbage — or any peer whose bytes arrive without a status line — books an estimate for a generation nobody made. This is the direction D39.196 п.2а forbids", + "package": "./internal/llm/", + "battery": true, + "run": "TestResponseBytesWithoutAReplyAreNotAPurchase", + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\t\tBillable: answered && tr.afterHeaders(),\n", + "replace": "\t\tBillable: tr.afterHeaders(),\n" + } + ] + }, + { + "id": "CUTSEV-cancelled-becomes-the-worst-problem", + "why": "the passport reports a chapter's WORST flag; `cancelled` is a state the next run erases, so ranking it above a durable finding hides the finding behind it. The exhaustiveness test proves only that every reason HAS a rank — the order itself was pinned by nothing", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestTheSeverityTableMeansWhatItsCommentsSay", + "edits": [ + { + "file": "internal/pipeline/status.go", + "find": "\tFlagCancelled: 8,\n", + "replace": "\tFlagCancelled: 0,\n" + } + ] + }, + { + "id": "CUTSEV-the-paid-and-lost-pair-stops-ranking-together", + "why": "decode_error and attempt_timeout are the same thing to a reader — the chunk is lost and the money is spent — and the table says so in its own comment; splitting them makes the passport call one of the two worse than the others for no reason a reader could act on", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestTheSeverityTableMeansWhatItsCommentsSay", + "edits": [ + { + "file": "internal/pipeline/status.go", + "find": "\tFlagAttemptTimeout: 4,\n", + "replace": "\tFlagAttemptTimeout: 5,\n" + } + ] + }, + { + "id": "CUTSEV-an-unknown-reason-outranks-a-diagnosis", + "why": "a reason this build has never heard of — an older schema's row, or junk — must not out-rank a diagnosis the engine actually made; putting it first makes an unparsed string the headline of a chapter that has real findings", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestTheSeverityTableMeansWhatItsCommentsSay", + "edits": [ + { + "file": "internal/pipeline/status.go", + "find": "const severityUnknown = 9\n", + "replace": "const severityUnknown = 0\n" + } + ] + }, + { + "id": "CUTCFG-a-cap-below-the-floor-loads-quietly", + "why": "the derivation clamps UP to attempt_s and then DOWN to attempt_max_s, so a cap below the floor wins: every call to that provider gets less time than the file declares, attempt_s stops being the FLOOR its doccomment calls it, and nothing logs the difference — the calls simply come back cut", + "package": "./internal/config/", + "battery": true, + "run": "TestACapBelowTheFloorIsRefusedAtLoad", + "edits": [ + { + "file": "internal/config/models.go", + "find": "\tif t.AttemptMaxS > 0 && t.AttemptMaxS < t.AttemptS {\n", + "replace": "\tif false {\n" + } + ] + }, + { + "id": "CUTCFG-the-timeout-check-is-not-wired-in", + "why": "the relation check exists and nothing calls it — the shape this pack has removed twice already, a guard living in the reader's head and not in the code", + "package": "./internal/config/", + "battery": true, + "run": "TestACapBelowTheFloorIsRefusedAtLoad", + "edits": [ + { + "file": "internal/config/models.go", + "find": "\t\tvalidateTimeouts(bad, name, p.Timeouts)\n", + "replace": "" + } + ] + }, + { + "id": "CUTCFG-a-measured-speed-floor-is-deleted", + "why": "a speed floor is the p10 of that provider's own request_log over hundreds of rows; deleting the line returns the provider to the vendor default, which is within two percent of the measured value — nothing breaks, the measurement is simply gone, and the catalogue gate cannot see it because a deleted line and a never-declared field are the same zero", + "package": "./internal/config/", + "battery": true, + "run": "TestEveryDeclaredDeadlineKnobStaysDeclared", + "edits": [ + { + "file": "configs/models.yaml", + "find": " timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60, tok_s_floor: 35 }\n", + "replace": " timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60 }\n" + } + ] + }, + { + "id": "CUTCOUNT-deliveries-counts-cuts-again", + "why": "the ledger line beside a cut says «the provider was asked N times; ONE estimate is booked for all of them» — the only place the gap between what was generated and what was billed is visible. Counting the CUTS instead of the DELIVERIES makes it understate a mixed chain: a 503 is an answer, so the peer read the request", + "package": "./internal/llm/", + "battery": true, + "run": "TestTheDeliveryCountCountsDeliveriesNotCuts", + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\t\t\tcc.Deliveries = askedToGenerate\n", + "replace": "\t\t\t\tcc.Deliveries = deliveredCutSeen\n" + } + ] + }, + { + "id": "CUTLINE-the-tail-splits-across-rows", + "why": "errors.Join separates its members with a newline, and an engine error carrying both a cut and what ended the chain is such a join; without collapsing it one table row prints as two and breaks the alignment of everything under it", + "package": "./cmd/tmctl/", + "battery": true, + "run": "TestTheOperatorTailIsOneLineAndKeepsTheNoteThatMatters", + "edits": [ + { + "file": "cmd/tmctl/render.go", + "find": "\ts = strings.Join(strings.Fields(s), \" \")\n", + "replace": "" + } + ] + }, + { + "id": "CUTLINE-the-note-goes-back-to-the-end", + "why": "the operator's column is bounded at 120 bytes and the transport error in front of the note is routinely longer, so a note appended at the end reaches nobody — and it is the only place the gap between what the provider generated and what the ledger booked is visible", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestTheAskedNTimesNoteReachesTheReader", + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\treturn fmt.Sprintf(\"[the provider was asked %d times; %s] %s\", deliveries, booked, err.Error())\n", + "replace": "\treturn fmt.Sprintf(\"%s [the provider was asked %d times; %s]\", err.Error(), deliveries, booked)\n" + } + ] + }, + { + "id": "CUTLINE-the-note-claims-money-that-was-not-booked", + "why": "on a cut the provider never acknowledged, nothing is booked; a line saying «ONE estimate is booked for all of them» beside a $0 row is a sentence a reader has to disbelieve before they can use it, and this is the one line where the generated-versus-billed gap is visible", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestTheAskedNTimesNoteReachesTheReader", + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\tif cost <= 0 {\n\t\tbooked = \"NOTHING is booked: the provider acknowledged none of them\"\n\t}\n", + "replace": "" + } + ] + }, + { + "id": "CUTCHAIN-the-stop-exit-drops-the-cancellation", + "why": "a stopped run must read as cancelled even when the attempt in flight failed of its own accord — the exit code (5, not 1) and the `stopped` the stream publishes both hang off it. The attempt's own error knows nothing about the context, so only this exit can carry it out; the wire fixture beside this one cannot see the loss, because there the attempt's error is itself a parent-cancelled cut and carries the cancellation on its own", + "package": "./internal/llm/", + "battery": true, + "run": "TestTheStopExitStillReadsAsCancelledWhenTheAttemptDidNot", + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\t\treturn zero, chainError(cancelledDuring(ctx.Err(), err), owedCut)\n", + "replace": "\t\t\treturn zero, chainError(err, owedCut)\n" + } + ] + }, + { + "id": "CUTEXIT-a-stop-stops-mapping-to-five", + "why": "the three cut causes share one Go type and two of them are ordinary failures while the third is a person pressing stop; a mapping that stops telling them apart reports an operator's stop as an engine failure — and the number is what a runbook and the platform's stream both branch on", + "package": "./cmd/tmctl/", + "battery": true, + "run": "TestTheCutErrorTypeKeepsItsExitCode", + "edits": [ + { + "file": "cmd/tmctl/main.go", + "find": "\tcase errors.Is(err, context.Canceled):\n\t\treturn 5\n", + "replace": "" + } + ] + }, + { + "id": "CUTDEADLINE-the-vendor-budget-moves", + "why": "every provider without its own measured floor derives its attempt deadline from this pair; a budget four times too generous makes the deadline four times too SHORT, so calls the provider is still generating get cut by us — and since this pack a self-cut the provider acknowledged is PAID FOR. Five of the eight catalogued providers run on this default, and the formula test cannot see the change: both of its sides read these same constants, so a corruption moves the expectation with the result", + "package": "./internal/llm/", + "battery": true, + "run": "TestTheVendorsPublishedPairIsWhatTheVendorPublishes", + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\tvendorHourlyTokenBudget = 128000\n", + "replace": "\tvendorHourlyTokenBudget = 512000\n" + } + ] + }, + { + "id": "CUTDEADLINE-the-vendor-window-moves", + "why": "the other half of the same pair: a window of a minute instead of an hour makes every derived deadline sixty times too long, and a run that looks hung is the only place it shows", + "package": "./internal/llm/", + "battery": true, + "run": "TestTheVendorsPublishedPairIsWhatTheVendorPublishes", + "edits": [ + { + "file": "internal/llm/attemptcut.go", + "find": "\tvendorBudgetWindow = time.Hour\n", + "replace": "\tvendorBudgetWindow = time.Minute\n" + } + ] + }, + { + "id": "CUTMARK-the-stop-is-judged-by-the-first-cause", + "why": "the retry chain hands up the cut with the strongest MONEY claim, which is deliberately the earlier paid break when the later one is free — so a run a person stopped arrives carrying a cause that is not «stopped». A guard reading that first cause decides a stop is not a stop and returns in silence: the money is settled, the position gets no row, and the export shows a gap nobody can explain. §4.2 forbids a hole with no mark at any moment", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestAStopBehindAPaidBreakStillMarksThePosition", + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\tif !errors.Is(err, context.Canceled) || !errors.As(err, &cut) {\n", + "replace": "\tif !errors.As(err, &cut) || cut.Cause != llm.CutByParent {\n" + } + ] + }, + { + "id": "CUTCOUNT-only-the-newest-cut-is-stamped", + "why": "the count is a property of the CHAIN, and chainError deliberately hands up an EARLIER cut when that is the one that owes money — carrying a number frozen at its birth. A chain that cut once and was then answered two 503s delivered three times and would report one, under-stating the very gap the ledger line exists to show; the later deliveries are not cuts, so nothing in the error tree knows about them", + "package": "./internal/llm/", + "battery": true, + "run": "TestAnOlderCutCarriesTheChainsFinalDeliveryCount", + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\tdefer func() {\n\t\t\tfor _, cc := range chainCuts {\n\t\t\t\tcc.Deliveries = askedToGenerate\n\t\t\t}\n\t\t}()\n", + "replace": "" + } + ] + }, + { + "id": "CUTCOUNT-an-undelivered-attempt-counts-as-asking", + "why": "an attempt whose request never left asked nobody and bought nothing; counting it says the engine bought a generation it did not, in the one line where the generated-versus-billed gap is published — and it is the direction D39.196 п.2а forbids", + "package": "./internal/llm/", + "battery": true, + "run": "TestAnUndeliveredAttemptIsNotCountedAsAsking", + "edits": [ + { + "file": "internal/llm/httpllm.go", + "find": "\t\tif deliveredAttempt(resp, err) {\n\t\t\taskedToGenerate++\n\t\t}\n", + "replace": "\t\taskedToGenerate++\n" + } + ] + }, + { + "id": "CUTROW-the-estimate-is-not-disclosed", + "why": "the ratified estimate is charged on ONE condition — that the row says it is an estimate (D39.230 п.1). A silent $0.001056 is indistinguishable from a provider-reported cost, and the owner's word does not apply to a row that does not disclose. The table of nine outcomes cannot see this: its column reads the CHECKPOINT's usage, not request_log.estimated", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestTheCutRowPublishesItsEstimateAndItsGap", + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\trl.Estimated, rl.EstTokens = cost > 0, r.estOutTokens(c.chunk.Text)\n", + "replace": "\trl.Estimated, rl.EstTokens = false, 0\n" + } + ] + }, + { + "id": "CUTROW-the-gap-is-not-published", + "why": "the provider was asked more than once and ONE estimate is booked for all of them; the row is the only place that under-count is published, and a silent gap is what row 360 was opened about", + "package": "./internal/pipeline/", + "battery": true, + "run": "TestTheCutRowPublishesItsEstimateAndItsGap", + "edits": [ + { + "file": "internal/pipeline/cutcall.go", + "find": "\t\trl.Err = cutErrLine(err, cut.Deliveries, cost)\n", + "replace": "" + } + ] } ] diff --git a/backend/configs/models.yaml b/backend/configs/models.yaml index 7bd24e2e..c27c041b 100644 --- a/backend/configs/models.yaml +++ b/backend/configs/models.yaml @@ -56,14 +56,44 @@ providers: # xai в ДАННЫЕ не вносил осознанно — это сняло бы grok-off-выключатель, чья цена (grok думает по # дефолту, additive) — решение владельца, а не сессии. echoes_when_thinking_off: true - timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60 } + # ⚠️ attempt_s — ПОЛ дедлайна, а не сам дедлайн: реальный считается от бюджета вызова + # (queue_slack_s + max_tokens/tok_s_floor, зажатый между attempt_s и attempt_max_s). Прежнее + # чтение «240 — это дедлайн» было верно ровно для ЧЕРНОВИКА и перенесено на редактора вслепую: + # 240 с — то, что вендорская формула 128 000 ток/час даёт для ~8.5k, а редактор бюджетируется на + # 16 000 с удвоением до 32 000, то есть на вызов, который в 240 с не уложится ни на какой скорости. + # + # tok_s_floor: 50 — ЗАМЕРЕНО по нашему же request_log, а не назначено. p10 скорости + # deepseek-v4-pro (самая медленная модель этого провайдера) = 51.40 ток/с на n=1745 строках + # 162 баз с непустыми completion_tokens и latency_ms; округление ВНИЗ до 50 объявлено здесь. + # Контроль: deepseek-v4-flash на тех же данных даёт p10 = 88.15 (n=6361), то есть пол держит обе. + # ⚠️ Замер берёт latency КОНЦА В КОНЕЦ, вместе с ожиданием в очереди, — значит он ЗАНИЖАЕТ + # реальную скорость генерации, и пол от него консервативен в правильную сторону. + # + # queue_slack_s: 600 — число ВЕНДОРА, не наше: DeepSeek документирует ранний 200 с пустыми + # строками, пока запрос ждёт планирования, и закрытие соединения, если инференс не начался за + # 10 минут (api-docs.deepseek.com/quick_start/rate_limit). Меньше ставить нельзя — это молча + # переигрывало бы принятый владельцем размен «ждать до ~20 минут» в сторону обрыва оплаченного. + # + # attempt_max_s: 1240 = 600 + 32000/50 — слак плюс максимальный грант С УДВОЕНИЕМ на полу. + timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60, tok_s_floor: 50, queue_slack_s: 600, attempt_max_s: 1240 } zai: # GLM, международный контур (docs.z.ai) kind: openai base_url: https://api.z.ai/api/paas/v4 api_key_env: ZAI_API_KEY reasoning: subset - timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60 } + # tok_s_floor: 35 — ЗАМЕРЕНО тем же прибором, что и deepseek: p10 скорости glm-5 = 37.51 ток/с на + # n=647 строках request_log (≥100, как требует правило вывода), округление ВНИЗ до 35 объявлено + # здесь. Ниже вендорского дефолта 35.5, то есть ставит вызову чуть БОЛЬШЕ времени, а не меньше. + # queue_slack_s НЕ ставится: документированного z.ai числа ожидания до инференса у меня нет, а + # гадать правило запрещает — слак остаётся нулевым, как и был. + # ⚠ ОТСТУПЛЕНИЕ, ОБЪЯВЛЕНО: пол замерен по glm-5, а НЕ по glm-5.1, которая тоже ходит через этого + # провайдера и наследует то же число без собственного замера. Отступление принято потому, что + # направление ошибки безопасно: 35 ниже вендорского дефолта 35.5, то есть даже неверный для 5.1 пол + # даёт вызову БОЛЬШЕ времени, чем дефолт, а не меньше — то есть режет не вызовы, а только запас. + # Убрать отступление можно одним способом: собрать n≥100 строк request_log по glm-5.1 и объявить её + # собственный p10 здесь. До тех пор число читать как «пол провайдера по измеренной модели». + timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60, tok_s_floor: 35 } kimi: # API-хост — api.moonshot.ai (intl, НЕ .cn). Веб-консоль переехала # platform.moonshot.ai → platform.kimi.ai, но API-эндпоинт остаётся @@ -73,7 +103,13 @@ providers: base_url: https://api.moonshot.ai/v1 api_key_env: KIMI_API_KEY reasoning: subset - timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60 } + # tok_s_floor НЕ ставится: строк этого провайдера в нашем request_log НОЛЬ (замер по 162 базам — + # deepseek 8106, glm 647, mistral 14, kimi 0), а назначать скорость без замера правило вывода + # запрещает. Дедлайн считается от вендорского дефолта 128 000 ток/час, который медленнее всего, + # что мы мерили, и потому только УДЛИНЯЕТ вызов; движок пишет об этом WARN на первом же вызове. + # attempt_max_s: 1200 — не замер, а ПОЛИТИКА: ратифицированный владельцем предел ожидания ~20 мин. + # Он ничего не режет — дефолтный пол на удвоенном гранте 16 000 даёт 900 с. + timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60, attempt_max_s: 1200 } xai: # Grok — главный тир канала B (18+) + судья 18+; из редакторских ролей СНЯТ (D30.1 — reasoning-off no-op). # ⚠️ КЛЮЧЕВАЯ ПОЛИТИКА (D27, супрсидит «чистый ключ» D19.4/D20.3): ЕДИНЫЙ XAI_API_KEY С data-sharing @@ -135,7 +171,10 @@ providers: # Поэтому склеиваем сами — это единственная форма, которая потерять не может. capabilities: system_messages: single - timeouts: { attempt_s: 300, max_attempts: 3, backoff_cap_s: 60 } # апекс думает дольше + # attempt_s 300 — апекс думает дольше; как и везде, это ПОЛ. tok_s_floor не ставится по той же + # причине, что у kimi: строк gemini в нашем request_log ноль. attempt_max_s — предел ожидания, + # ратифицированный владельцем (~20 мин); дефолтный пол на удвоенном гранте 16 000 даёт 900 с. + timeouts: { attempt_s: 300, max_attempts: 3, backoff_cap_s: 60, attempt_max_s: 1200 } openai: # OpenAI прямой ключ (D3: Anthropic убран, OpenAI ОСТАЁТСЯ). gpt-5-nano альт-черновик, # gpt-5-mini кандидат-редактор/second-opinion судья — НЕ в дефолтной цепочке Ф1 (слот под Ф2). diff --git a/backend/internal/config/models.go b/backend/internal/config/models.go index 161bbd6f..b1cd8c64 100644 --- a/backend/internal/config/models.go +++ b/backend/internal/config/models.go @@ -122,16 +122,36 @@ type ReasoningCapCfg struct { // Timeouts is the retry profile per provider (the per-provider profile from the // validation verdict; per-role overrides — Phase 1). type Timeouts struct { + // AttemptS is the FLOOR of one attempt's deadline, not the deadline itself: the real one is derived + // from the output budget the call carries (llm/attemptcut.go). Read as a fixed value it was wrong by + // construction for the editor — 240 s is what the vendor's own 128 000-tokens-per-hour figure gives + // for the DRAFT's ~8.5k budget, and the same number was carried to a stage budgeted at 16 000 with a + // doubling to 32 000, i.e. to a call that could not finish inside it at any speed the model holds. AttemptS int `yaml:"attempt_s"` MaxAttempts int `yaml:"max_attempts"` BackoffCapS int `yaml:"backoff_cap_s"` + // TokSFloor is the slowest generation speed this provider has been OBSERVED to hold — below the p10 + // of its own request_log, rounded down, and the rounding declared where it is set. Unset ⇒ the + // vendor default (128 000 tokens/hour), which is slower than any provider we have measured and + // therefore only ever grants a call MORE time than it needs. + TokSFloor float64 `yaml:"tok_s_floor"` + // QueueSlackS is how long the VENDOR documents a request may wait before generation starts. It is + // the vendor's number and not a guess: a slack smaller than what the vendor publishes silently + // re-decides how long we are willing to wait, in the direction of cutting calls we have paid for. + QueueSlackS int `yaml:"queue_slack_s"` + // AttemptMaxS caps the derived deadline — the longest a single call may hold a reservation. 0 = + // uncapped, i.e. the derivation stands on its own. + AttemptMaxS int `yaml:"attempt_max_s"` } func (t Timeouts) Profile() llm.RetryProfile { return llm.RetryProfile{ - AttemptTimeout: time.Duration(t.AttemptS) * time.Second, - MaxAttempts: t.MaxAttempts, - BackoffCap: time.Duration(t.BackoffCapS) * time.Second, + AttemptTimeout: time.Duration(t.AttemptS) * time.Second, + MaxAttempts: t.MaxAttempts, + BackoffCap: time.Duration(t.BackoffCapS) * time.Second, + TokensPerSecFloor: t.TokSFloor, + QueueSlack: time.Duration(t.QueueSlackS) * time.Second, + AttemptMax: time.Duration(t.AttemptMaxS) * time.Second, } } @@ -238,6 +258,7 @@ func LoadModels(path string) (*Models, error) { if p.Kind == "anthropic" && p.CacheTTL != "" && p.CacheTTL != "5m" && p.CacheTTL != "1h" { bad("provider %s: cache_ttl must be 5m|1h, got %q", name, p.CacheTTL) } + validateTimeouts(bad, name, p.Timeouts) if p.Kind == "local" && p.Model == "" { bad("provider %s: local kind requires model (its own tag)", name) } @@ -462,6 +483,17 @@ func (m *Models) ResolveCapability(modelName string) llm.Capability { return resolved } +// AttemptDeadline is how long ONE call to this model may run: the provider's retry profile applied to +// the call's own output budget (llm.RetryProfile.DeadlineFor). It resolves model→provider→timeouts the +// same way ResolveCapability and MinMaxTokens resolve their fields, so a caller that needs to SAY how +// long a call may wait reads the same answer the transport will act on rather than assembling one. +// +// An unknown model yields the zero profile, whose derivation still returns the vendor default — the +// same direction every fallback on this path takes: too long, never too short. +func (m *Models) AttemptDeadline(modelName string, maxTokens int) time.Duration { + return m.Providers[m.Models[modelName].Provider].Timeouts.Profile().DeadlineFor(maxTokens) +} + // thinkingControlExtraKeys are keys whose purpose is to toggle a provider's thinking // on the wire — top-level (GLM/DeepSeek {"thinking":…}, Qwen {"enable_thinking":…}, a // raw {"reasoning_effort":…}) OR nested (DeepSeek-V3.1+ disables via @@ -620,6 +652,37 @@ func containsLabel(set []string, want string) bool { // how "accepts none" is written), and a duplicate is a config smell worth naming rather than absorbing // silently. Labels are compared verbatim, so leading/trailing space would make two spellings of one // permission — rejected rather than trimmed, because a permission must not be guessed at. +// maxAttemptSeconds bounds every deadline knob a provider may set. It is a TYPO GUARD, not a policy: the +// owner ratified waiting up to about twenty minutes for one call, and this sits an order of magnitude +// above that so a legitimate configuration is never refused. What it refuses is a slipped digit — +// `queue_slack_s: 6000` reads as an hour and forty minutes of waiting nobody chose, and the only place +// that shows up is a run that looks hung. +const maxAttemptSeconds = 4 * 60 * 60 + +// validateTimeouts checks the deadline knobs AGAINST EACH OTHER, which is the part no per-field check +// can do. The derivation clamps the derived deadline up to `attempt_s` and then down to `attempt_max_s`, +// so a cap below the floor wins — and the floor's own doccomment, which calls it a FLOOR, quietly stops +// being true for every call that provider makes. That is a config a person can write in one keystroke +// and cannot see afterwards: nothing logs it, and the calls simply get less time than the file says. +func validateTimeouts(bad func(string, ...any), name string, t Timeouts) { + if t.AttemptS < 0 || t.AttemptS > maxAttemptSeconds { + bad("provider %s: attempt_s %d is out of range (0..%d seconds)", name, t.AttemptS, maxAttemptSeconds) + } + if t.QueueSlackS < 0 || t.QueueSlackS > maxAttemptSeconds { + bad("provider %s: queue_slack_s %d is out of range (0..%d seconds)", name, t.QueueSlackS, maxAttemptSeconds) + } + if t.AttemptMaxS < 0 || t.AttemptMaxS > maxAttemptSeconds { + bad("provider %s: attempt_max_s %d is out of range (0..%d seconds)", name, t.AttemptMaxS, maxAttemptSeconds) + } + if t.TokSFloor < 0 { + bad("provider %s: tok_s_floor %g is negative — a speed floor below zero grants a call infinite time", name, t.TokSFloor) + } + if t.AttemptMaxS > 0 && t.AttemptMaxS < t.AttemptS { + bad("provider %s: attempt_max_s %d is BELOW attempt_s %d — the cap would win and every call would "+ + "get less time than the declared floor, silently", name, t.AttemptMaxS, t.AttemptS) + } +} + func validateLabelSet(bad func(string, ...any), where string, set *[]string) { if set == nil { return diff --git a/backend/internal/config/models_catalog_test.go b/backend/internal/config/models_catalog_test.go index 568715bf..5bf79977 100644 --- a/backend/internal/config/models_catalog_test.go +++ b/backend/internal/config/models_catalog_test.go @@ -185,3 +185,166 @@ models: t.Fatalf("a declared `single` MUST move the snapshot bytes — that is what makes the flip a loud --resnapshot: %s", singleJSON) } } + +// TestAProviderThatWillWaitLongSaysSoInTheCatalog is the second half of the derived deadline, and it +// is the half that answers the standing review question — «does a provider the repository has never +// seen work without editing Go». +// +// It does, and this is the price: a provider whose models declare a big enough max_tokens FLOOR is +// declaring that its calls are big, and a big call under the vendor DEFAULT speed derives a deadline +// far past whatever attempt_s the config carries. That is not a failure — the derivation is +// deliberately slower than any provider we have measured, so it only ever grants MORE time — but it +// is a quarter of an hour of waiting that the config never mentions, and an operator who reads +// `attempt_s: 240` and watches a call run for fifteen minutes has been told something untrue by his +// own configuration. +// +// So the catalogue has to say ONE of two things about such a provider, and either is a line of DATA: +// - `tok_s_floor` — the speed it actually holds, measured from its own request_log; or +// - `attempt_max_s` — how long we are willing to wait for it, which is a policy and needs no measurement. +// +// ⛔ IT DOES NOT DEMAND THE MEASUREMENT. Requiring `tok_s_floor` would force a number to be invented +// for every provider nobody has run yet, and a fabricated floor is worse than an honest default: it +// would cut real calls we had already paid for. The alternative is the point. +func TestAProviderThatWillWaitLongSaysSoInTheCatalog(t *testing.T) { + m, err := LoadModels(shippedModelsYAML) + if err != nil { + t.Fatalf("load: %v", err) + } + // The largest budget one call to each provider can carry: its biggest declared floor, doubled once + // by a regeneration. It is a LOWER bound on what a real run may ask for (a long chunk derives more + // than the floor), which is the conservative direction for a gate: it can only under-report who + // needs a line, never demand one nobody needs. + biggest := map[string]int{} + for name, mod := range m.Models { + if floor := m.MinMaxTokens(name); floor*2 > biggest[mod.Provider] { + biggest[mod.Provider] = floor * 2 + } + } + checked, flagged := 0, 0 + for prov, grant := range biggest { + if grant == 0 { + continue // no model of this provider declares a floor: nothing to say + } + p := m.Providers[prov] + derived := p.Timeouts.Profile().DeadlineFor(grant) + configured := time.Duration(p.Timeouts.AttemptS) * time.Second + if derived <= configured { + continue // every call fits inside the deadline the config already names + } + checked++ + if p.Timeouts.TokSFloor <= 0 && p.Timeouts.AttemptMaxS <= 0 { + flagged++ + t.Errorf("provider %q can be asked for %d output tokens, which derives a %s deadline against "+ + "its configured attempt_s of %s — declare timeouts.tok_s_floor (measure it from this "+ + "provider's own request_log) or timeouts.attempt_max_s (how long we are willing to wait)", + prov, grant, derived, configured) + } + } + // The control beside the negative: «0 providers need a line» and «the loop never ran» print the + // same on a green test, and only one of them means anything. + t.Logf("providers in the catalogue: %d; with a declared token floor: %d; whose biggest call outgrows "+ + "its own attempt_s (and therefore must declare one of the two fields): %d; missing both: %d", + len(m.Providers), len(biggest), checked, flagged) + if checked == 0 { + t.Fatalf("no provider in the shipped catalogue outgrows its own attempt_s — this gate is asking a " + + "question of nothing, and would stay green through any regression in the derivation") + } +} + +// TestEveryDeclaredDeadlineKnobStaysDeclared pins the deadline data as data. Each of these numbers was +// PAID FOR — a speed floor is the p10 of that provider's own request_log over hundreds of rows, a queue +// slack is the vendor's published figure — and none of them is reachable by the gate above: it walks the +// LOADED struct, where a deleted line and a never-declared field are the same zero, and it skips a +// provider whose models declare no token floor at all. So deleting `tok_s_floor: 35` from zai passed +// every case in this package. +// +// The consequence of losing one is quiet by construction. A missing speed floor falls back to the vendor +// default, which is within two percent of zai's measured value — the run does not break, the measurement +// is simply gone, and the next person re-derives it from scratch or does without. +func TestEveryDeclaredDeadlineKnobStaysDeclared(t *testing.T) { + m, err := LoadModels(shippedModelsYAML) + if err != nil { + t.Fatalf("load: %v", err) + } + type knobs struct { + tokSFloor float64 + queueSlackS int + attemptMaxS int + } + // The catalogue as it ships. A change here is a change to a measured or vendor-published number and + // must be made deliberately, with the measurement said beside it in models.yaml. + want := map[string]knobs{ + "deepseek": {tokSFloor: 50, queueSlackS: 600, attemptMaxS: 1240}, + "zai": {tokSFloor: 35}, + "kimi": {attemptMaxS: 1200}, + "gemini": {attemptMaxS: 1200}, + } + declared := 0 + for name, p := range m.Providers { + got := knobs{p.Timeouts.TokSFloor, p.Timeouts.QueueSlackS, p.Timeouts.AttemptMaxS} + if got != (knobs{}) { + declared++ + } + w, pinned := want[name] + if !pinned { + if got != (knobs{}) { + t.Errorf("provider %q declares deadline knobs %+v that this pin does not know about — add "+ + "them here with the measurement that produced them, or the next edit loses them silently", name, got) + } + continue + } + if got != w { + t.Errorf("provider %q: deadline knobs moved from %+v to %+v. These are measured numbers (a "+ + "speed floor is the p10 of this provider's own request_log) — losing one returns the "+ + "provider to the vendor default without a word anywhere", name, w, got) + } + } + // The control beside the count: «every knob is where it was» and «the catalogue declares none» print + // the same on a green test. + if declared != len(want) { + t.Fatalf("premise broken: %d providers declare deadline knobs, the pin names %d — the two must be "+ + "the same set or one side is not being read", declared, len(want)) + } + t.Logf("providers in the catalogue: %d; declaring deadline knobs: %d, all pinned", len(m.Providers), declared) +} + +// TestACapBelowTheFloorIsRefusedAtLoad pins the one relation between the deadline knobs that no +// per-field check can see. The derivation clamps UP to `attempt_s` and then DOWN to `attempt_max_s`, so +// a cap below the floor wins: every call to that provider quietly gets less time than the file declares, +// and `attempt_s`'s own doccomment — which calls it a FLOOR — stops being true. Nothing logs it; the +// calls simply come back cut. +func TestACapBelowTheFloorIsRefusedAtLoad(t *testing.T) { + load := func(timeouts string) error { + t.Helper() + body := "prices_checked: " + time.Now().UTC().Format("2006-01-02") + ` +default_model: fake +providers: + p: { kind: openai, base_url: http://x, timeouts: ` + timeouts + ` } +models: + fake: { provider: p, price: { input_per_m: 1, output_per_m: 2 } } +` + _, err := LoadModels(writeTmp(t, filepath.Join(t.TempDir(), "models.yaml"), body)) + return err + } + // The control FIRST: a cap ABOVE the floor is an ordinary configuration and must load, or the check + // below would be satisfied by a loader that refuses everything. + if err := load("{ attempt_s: 240, max_attempts: 2, attempt_max_s: 1200 }"); err != nil { + t.Fatalf("a cap above the floor is a legitimate configuration and must load: %v", err) + } + if err := load("{ attempt_s: 240, max_attempts: 2 }"); err != nil { + t.Fatalf("an unset cap means «the derivation stands on its own» and must load: %v", err) + } + err := load("{ attempt_s: 240, max_attempts: 2, attempt_max_s: 120 }") + if err == nil { + t.Fatal("a cap BELOW the floor must be refused at load: the cap wins in the clamp, so every call " + + "gets less time than the declared floor and nothing anywhere says so") + } + if !strings.Contains(err.Error(), "attempt_max_s") || !strings.Contains(err.Error(), "attempt_s") { + t.Fatalf("the refusal must name BOTH knobs, or the operator cannot see which pair is wrong: %v", err) + } + // And the typo guard, which is the other half of the same field's danger: a slipped digit turns a + // ten-minute wait into an afternoon, and the only place it shows is a run that looks hung. + if err := load("{ attempt_s: 240, max_attempts: 2, queue_slack_s: 600000 }"); err == nil { + t.Fatal("a queue slack of a week must be refused as a typo — nobody chose to wait that long") + } +} diff --git a/backend/internal/llm/attemptcut.go b/backend/internal/llm/attemptcut.go new file mode 100644 index 00000000..3fd5dd29 --- /dev/null +++ b/backend/internal/llm/attemptcut.go @@ -0,0 +1,343 @@ +package llm + +import ( + "context" + "errors" + "fmt" + "net/http/httptrace" + "strings" + "sync/atomic" + "time" +) + +// attemptcut.go: what OUR OWN deadline does to a call, and the deadline itself. +// +// A provider that has accepted a request generates whether or not we are still listening. When the +// attempt deadline fires mid-generation the connection drops on our side, the provider never learns +// it, and the call is billed all the same — so «we stopped waiting» is a MONEY event, not a transport +// one. The transport used to lose it twice over: the body read's error went to `_`, and truncation was +// judged by SIZE, so a body our own deadline cut short (small) was indistinguishable from a whole one +// and went down the retryable «broken connection» branch that the same code's comment forbids for a +// billed 2xx («each retry is a new billed 2xx»). +// +// THE ONE FACT COMMON TO EVERY SUCH CASE IS THAT THE REQUEST WAS DELIVERED, and it is observable +// without a byte on the wire: net/http/httptrace reports when the request has been WRITTEN and when +// the first response byte arrived. +// +// ⚠ NO SURVEYED HARNESS DRAWS THE BOUNDARY HERE, and that is a fact rather than a claim of novelty: +// research/21 covers eleven of them and mentions httptrace nowhere (0 hits against 20 for openai-go). +// The reason is what they are: five are stream-only and four do both (§Q6), so «did anything arrive» is +// answered by the first event and the question never comes up; the remaining two are generic SDKs whose +// non-streaming answer is to REFUSE a long call outright. On a non-streaming path with no refusal to +// fall back on it has to be answered another way, and the stdlib already answers it — so the mechanism +// is stdlib rather than ours (research/21's own rule: an industrial primary source before a home-made one). +// +// Those two booleans separate the three cases that a status line cannot: +// +// delivered, headers still pending, our deadline WroteRequest=true GotFirstResponseByte=false +// TLS handshake hanging — NOT delivered WroteRequest=false GotFirstResponseByte=false +// early 200 with empty lines, our deadline WroteRequest=true GotFirstResponseByte=true +// +// The middle row is the ONLY one where «nothing was bought» is true by construction. On DeepSeek the +// first and third are the live ones: the vendor documents an early 200 with empty lines while a +// request waits to be scheduled, so a 200 there means ACCEPTED, not GENERATED, and the branch that +// reads «No 2xx ever arrived: nothing was billed» is unreachable on it — every self-cut hid under +// decode_error instead. + +// CutCause says WHO ended an attempt that had already been delivered. The three are dispositions, +// not shades of one error: our own deadline is not retried and is flagged, a run the operator +// stopped is re-done on resume at the same budget, and a broken connection gets one retry. +type CutCause string + +const ( + // CutBySelfDeadline: our per-attempt deadline fired while the provider was still working. + // Retrying buys the same generation a second time — the whole point of typing this. + CutBySelfDeadline CutCause = "attempt_timeout" + // CutByParent: the RUN ended (stop, Ctrl-C) under a call that had already gone out. The call was + // healthy; a human stopped it. + CutByParent CutCause = "cancelled" + // CutByConnection: the connection broke after delivery (unexpected EOF, an h2 stream reset) with + // both deadlines still alive. Transport-shaped, so it is worth exactly one retry. + CutByConnection CutCause = "connection_lost" +) + +// AttemptCutError is a DELIVERED request whose reply we did not receive whole. +// +// ⚠ It is deliberately NOT named for the timeout: two of its three causes are not one (a run somebody +// stopped, a connection that broke), and the runner routes money, flag and resume differently for each. +// A type named `AttemptTimeoutError` carrying `Cause: cancelled` would be a name lying about a cause, +// which is the thing the flag vocabulary forbids outright (D39.93 п.2). +// +// Delivered is the money boundary and is always true on a value that reaches a caller — the +// constructor refuses to build one otherwise, because «not delivered» is the case that must keep +// costing nothing. +type AttemptCutError struct { + Provider string + Cause CutCause + // Delivered: the request bytes reached the provider (httptrace WroteRequest, no write error). + Delivered bool + // AfterHeaders: the first BYTE of a response had arrived when the call was cut (httptrace's + // GotFirstResponseByte) — the early-200 case. + // + // ⚠ IT IS A BYTE, NOT A REPLY, and the distinction is the whole reason `Billable` is not this field. + // The first byte fires for a 1xx, for a header block that never terminated, for a broken proxy's + // garbage — none of which is a provider acknowledging anything. Read as «a response arrived» it + // tells an operator about a reply that did not exist; read as what it is, it is the evidence that + // something came back down the socket, and the money question is answered by `Billable`, which also + // requires a 2xx object in hand. + AfterHeaders bool + // BytesRead is how much of the body we did get. It is EVIDENCE, never the criterion: judging + // truncation by size is the defect this type replaces. + BytesRead int + // WhitespaceOnly says the bytes we got carry no content at all (DeepSeek's documented empty lines + // while a request waits for scheduling). It distinguishes «the provider was still queueing» from + // «the provider was mid-answer», which is the difference between a wait worth extending and a + // generation worth not buying twice. + WhitespaceOnly bool + Elapsed time.Duration + // Billable is the MONEY boundary, and it is deliberately NARROWER than Delivered. + // + // ⛔ TWO PREDICATES, BECAUSE THEY ANSWER TWO QUESTIONS. `WroteRequest` says our bytes left for the + // peer's TCP window; it says nothing about the APPLICATION behind it — a load balancer can accept + // while the backend never sees the request. Measured on a stopped run: 3 of 25 cancelled in-flight + // calls settled an estimate for a request no handler ever entered. On a non-streaming wire the one + // signal that a provider's application acknowledged the request is a 2xx response object reaching + // US, so that is what money is drawn on. Delivery still drives RETRY — a written request is not + // automatically re-sent — and that boundary is unchanged. + // + // The direction of what remains is the tolerable one: on a provider that HOLDS its headers, a + // pre-header self-cut books zero, i.e. an under-count. On DeepSeek nothing is lost at all — its 200 + // arrives on acceptance, so every self-cut there is post-header. + Billable bool + // Deliveries is how many times THIS request reached the provider inside one retry chain — counted by + // DELIVERY, not by cause, so a chain that was cut once and answered an undecodable 2xx once reports + // two. It is almost always 1; a broken connection is worth one retry, and then the provider has been + // asked to generate TWICE while the ledger books ONE estimate — the store cannot write two settles + // under one key. The number is carried rather than acted on: making the gap visible is this pack's + // business, deciding what it costs is the owner's. + Deliveries int + // Err is the transport error that ended the attempt. + Err error + // Parent is the parent context's own cause, set ONLY for CutByParent. It rides here so that + // errors.Is(err, context.Canceled) keeps deciding the process exit code while errors.As still + // finds this type: the exit contract and the money both need to be true of one error. + Parent error +} + +func (e *AttemptCutError) Error() string { + where := "before any response byte" + if e.AfterHeaders { + where = fmt.Sprintf("after %d body bytes", e.BytesRead) + if e.WhitespaceOnly { + where += " (whitespace only)" + } + } + return fmt.Sprintf("%s: delivered request cut by %s %s after %s: %v", + e.Provider, e.Cause, where, e.Elapsed.Round(time.Millisecond), e.Err) +} + +// Unwrap returns both truths of a cancelled attempt. Go's errors.Is/As walk every branch, so the +// caller that asks «did the run end» and the caller that asks «what did it interrupt» both get a +// straight answer from the same value. +func (e *AttemptCutError) Unwrap() []error { + if e.Parent != nil { + return []error{e.Err, e.Parent} + } + return []error{e.Err} +} + +// deliveryTrace records the two httptrace facts the money boundary is drawn from. The callbacks run +// on the transport's goroutine while the caller may already be reading the fields (a deadline fires +// concurrently with a write completing), so both are atomics rather than plain bools. +type deliveryTrace struct { + wrote atomic.Bool + firstByte atomic.Bool +} + +func (t *deliveryTrace) clientTrace() *httptrace.ClientTrace { + return &httptrace.ClientTrace{ + // info.Err non-nil means the write itself failed — the request did NOT reach the provider, + // and treating that as delivery would settle money for a call nobody received. + WroteRequest: func(info httptrace.WroteRequestInfo) { + if info.Err == nil { + t.wrote.Store(true) + } + }, + GotFirstResponseByte: func() { t.firstByte.Store(true) }, + } +} + +// delivered reports that the request reached the provider. `answered` says the transport handed us an +// actual 2xx response, and it is the second half of the evidence — needed, and needed CONDITIONALLY. +// +// ⛔ A RESPONSE BYTE IS DELIVERY EVIDENCE, BUT ONLY WHEN A REPLY ACTUALLY CAME BACK TO US. net/http does +// not wait for the write loop before handing back a response: Request.write fires WroteRequest from a +// deferred call on the writeLoop goroutine (net/http/request.go) while roundTrip returns as soon as the +// response channel fires (net/http/transport.go), and golang.org/x/net/http2 has the same shape. So a +// provider answering EARLY — precisely what DeepSeek documents doing while a request waits to be +// scheduled — can have its 200 overtake our own «the request was written» callback whenever the body is +// still draining. Read as «not delivered», that call books $0 and is retried: measured at three re-asks. +// +// ⛔⛔ AND THE OTHER READING COSTS MORE. Taking the response BYTE alone as proof, whether or not a reply +// reached us, pays for refusals: a provider that answers 401/403/413 and resets the connection while our +// body is still writing gives GotFirstResponseByte=true and a WRITE ERROR, so http.Client.Do returns the +// write failure and we never see the status at all. Measured on a copy of this tree: 22 refusals in 25 +// booked a paid cut, one of them $0.80 for a request the provider declined — and the delivered-cut retry +// sent the whole body a second time. When Do fails we hold no status line and cannot tell a refusal from +// an early success, so only the write counts; when Do SUCCEEDS with a 2xx, the reply is proof by itself. +func (t *deliveryTrace) delivered(answered bool) bool { + return t.wrote.Load() || (answered && t.firstByte.Load()) +} +func (t *deliveryTrace) afterHeaders() bool { return t.firstByte.Load() } + +// causeOf names who ended a delivered attempt. The order is the meaning: the PARENT is asked first, +// because a run that is ending has cancelled the attempt context too, and reading our own deadline +// first would file every stopped run as a self-cut and flag chunks nobody's provider misbehaved on. +func causeOf(ctx, attemptCtx context.Context) CutCause { + switch { + case ctx.Err() != nil: + return CutByParent + case errors.Is(attemptCtx.Err(), context.DeadlineExceeded): + return CutBySelfDeadline + default: + return CutByConnection + } +} + +// cutError builds the typed error for a delivered attempt, or NIL when the request never went out. +// The «not delivered» path is the one case where nothing was bought, and it must stay exactly what it +// was: a plain retryable transport failure that releases the reservation. Nil rather than «the error +// unchanged» so the caller branches on a value instead of on error identity — the shape that survives +// somebody wrapping the transport error one layer deeper. +// `answered` is true only where a 2xx response object is in hand — see delivered. +func (c *openAIClient) cutError(ctx, attemptCtx context.Context, tr *deliveryTrace, started time.Time, body []byte, err error, answered bool) *AttemptCutError { + if !tr.delivered(answered) { + return nil + } + cause := causeOf(ctx, attemptCtx) + cut := &AttemptCutError{ + Provider: c.name, Cause: cause, + Delivered: true, AfterHeaders: tr.afterHeaders(), + Billable: answered && tr.afterHeaders(), + BytesRead: len(body), WhitespaceOnly: len(body) == 0 || strings.TrimSpace(string(body)) == "", + Elapsed: time.Since(started), Err: err, + } + if cause == CutByParent { + cut.Parent = ctx.Err() + } + return cut +} + +// retryable is the retry half of the disposition, kept beside the causes so the two cannot drift. +// Only a broken connection is worth another call: our own deadline firing means the provider is STILL +// GENERATING what we just stopped listening to, and retrying buys that generation a second time — the +// defect this file exists to remove. A cancelled run has nothing to retry into. +func (e *AttemptCutError) retryable() bool { return e.Cause == CutByConnection } + +// --- the deadline itself (backlog row 360 point 5, row 369) --- + +// The industry reference for sizing a non-streaming call, and the source of the default rate: +// anthropic-sdk-go's CalculateNonStreamingTimeout budgets 1h · max_tokens / 128 000 and REFUSES a +// request whose expected time exceeds ten minutes («streaming is required»). Ours is the same +// arithmetic with the rate made per-provider data and the vendor's queue wait added, because we have +// no streaming path to fall back to and must wait instead of refusing (research/21 §Q6 + its 08.09 +// errata; openai-go's ResponseHeaderTimeout is the same idea applied to time-to-headers alone). +// +// It is the default because it is the one number that is not ours to invent — and it is the number +// `attempt_s: 240` was silently standing in for: 240 s at this rate is ~8.5k tokens, the DRAFT's +// budget, and the same 240 was carried to an editor budgeted at 16 000 with a doubling to 32 000. +const ( + vendorHourlyTokenBudget = 128000 + vendorBudgetWindow = time.Hour +) + +// defaultTokensPerSecFloor is DERIVED from the pair above and never typed as a decimal: writing +// 35.5 here would be a second carrier of a number the vendor states as 128 000 per hour, and the two +// would drift the day either moved. +func defaultTokensPerSecFloor() float64 { + return float64(vendorHourlyTokenBudget) / vendorBudgetWindow.Seconds() +} + +// deriveDeadline is the UNCLAMPED time this profile says a call for maxTokens needs: the vendor's +// documented wait before generation starts, plus the generation itself at the slowest speed the +// model has been observed to hold. It is a pure function of the profile and the budget — the budget +// the call actually carries, so an escalated attempt that doubled its max_tokens doubles its time +// instead of inheriting the budget of an attempt that asked for half as much. +// +// An unset or nonsensical floor falls back to the vendor default rather than to a division by zero +// or a zero deadline: a config that forgot the field must wait too long, never not at all. +func (p RetryProfile) deriveDeadline(maxTokens int) time.Duration { + floor := p.TokensPerSecFloor + if floor <= 0 { + floor = defaultTokensPerSecFloor() + } + if maxTokens < 0 { + maxTokens = 0 + } + return p.QueueSlack + time.Duration(float64(maxTokens)/floor*float64(time.Second)) +} + +// DeadlineFor clamps the derivation between the configured attempt_s and attempt_max_s. It is exported +// because it answers a question about a CONFIGURATION rather than about a call in flight — «how long +// will one call of this size wait under this profile» — and the catalogue gate over models.yaml has to +// ask it of a provider nobody has called yet. +// +// ⛔ attempt_s is the FLOOR, not the value. That is what makes this change safe to land on every +// existing config at once: a call whose derived time is shorter than the configured deadline keeps +// the configured one, so no provider loses a second it has today, and only calls that provably +// could not finish get more. attempt_max_s bounds the other end — an operator's ceiling on how long +// one call may hold a reservation — and is inert when unset. +func (p RetryProfile) DeadlineFor(maxTokens int) time.Duration { + d := p.deriveDeadline(maxTokens) + if d < p.AttemptTimeout { + d = p.AttemptTimeout + } + if p.AttemptMax > 0 && d > p.AttemptMax { + d = p.AttemptMax + } + return d +} + +// attemptDeadline is the client's own wrapper: the same clamp, plus the ONE notice an operator needs +// when a provider is running on the vendor default. Waiting a quarter of an hour for a call is a +// legitimate configuration — the owner ratified waiting up to ~20 minutes — but it must never be a +// surprise, and «why is nothing happening» is the pain this line answers before it is felt. +// +// The condition is structural rather than a threshold in tokens: it fires exactly when the derived +// time OVERRIDES the configured attempt_s, i.e. when this call is one the configured deadline could +// not have covered. A provider whose calls all fit inside its own attempt_s never sees it. +func (c *openAIClient) attemptDeadline(ctx context.Context, maxTokens int) time.Duration { + d := c.profile.DeadlineFor(maxTokens) + if c.profile.TokensPerSecFloor <= 0 && d > c.profile.AttemptTimeout { + c.floorWarned.Do(func() { + if c.log == nil { + return + } + c.log.WarnContext(ctx, "no measured generation speed for this provider; the call deadline is derived from the vendor default and is longer than the configured attempt_s — set timeouts.tok_s_floor from this provider's own request_log after the first run", + "provider", c.name, "default_tok_s", fmt.Sprintf("%.1f", defaultTokensPerSecFloor()), + "max_tokens", maxTokens, "derived_deadline", d.String(), "attempt_s", c.profile.AttemptTimeout.String()) + }) + } + return d +} + +// cancelledDuring is what a stopped run returns: BOTH the cancellation, which decides the process exit +// code, and whatever the attempt underneath it was — which is what decides the money. +// +// ⛔ IT IS errors.Join AND NOT A CHOICE BETWEEN THEM. Returning the attempt's error alone loses +// `errors.Is(err, context.Canceled)` for every attempt error that does not happen to wrap the +// cancellation — a stopped run would then leave with a foreign exit code. Returning the cancellation +// alone loses `errors.As`, and with it the money for a call already on the wire, which is the defect +// this whole file exists to remove. Join keeps both true of one value, and it does so whatever the +// attempt's cause was: an earlier form of this guard preserved only the errors that already carried the +// parent's own error, so a connection_lost or attempt_timeout whose run was stopped a moment later was +// still silently reduced to a bare cancellation. +func cancelledDuring(ctxErr, attemptErr error) error { + if attemptErr == nil { + return ctxErr + } + if errors.Is(attemptErr, ctxErr) { + return attemptErr // it already carries both; joining would only duplicate the sentence + } + return errors.Join(ctxErr, attemptErr) +} diff --git a/backend/internal/llm/attemptcut_test.go b/backend/internal/llm/attemptcut_test.go new file mode 100644 index 00000000..3677a69f --- /dev/null +++ b/backend/internal/llm/attemptcut_test.go @@ -0,0 +1,1330 @@ +package llm + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +// attemptcut_test.go: the two things the cut-call boundary rests on — that the per-attempt deadline is +// DERIVED from the budget rather than typed, and that a delivered request is told apart from an +// undelivered one by the transport rather than by the size of what came back. + +// draftGrant and editorGrant are the two real output budgets of the shipping pipeline: the draft's, and +// the editor's after one regeneration doubled it. Two, because a single grant cannot distinguish a +// formula from a constant that happens to match it. +const ( + draftGrant = 8496 + editorGrant = 32000 +) + +// vendorSeconds answers how long the vendor's own budgeting says it takes to emit n output tokens. It is +// deliberately the reciprocal ARRANGEMENT of the production formula — tokens × window ÷ budget, where +// deriveDeadline does tokens ÷ (budget ÷ window) — so a formula rewritten the wrong way round is caught. +// +// ⚠ IT PINS THE ARRANGEMENT AND NOT THE NUMBERS, and an earlier version of this comment claimed +// otherwise. Both sides read the SAME two package constants, so corrupting them moves both and the +// identity still holds: `vendorHourlyTokenBudget` was quadrupled and the whole battery stayed green. +// The numbers are pinned by TestTheVendorsPublishedPairIsWhatTheVendorPublishes, which quotes them +// against the source they came from; this function is the other half and neither replaces the other. +func vendorSeconds(n int) time.Duration { + return time.Duration(float64(n) * float64(vendorBudgetWindow) / float64(vendorHourlyTokenBudget)) +} + +// TestTheAttemptDeadlineIsDerivedFromTheBudget pins the FORMULA. It cannot be satisfied by any +// constant, because every expectation below is computed here from the vendor's two published numbers +// and the grant, and because the ratio assertion holds for a derivation and for nothing else. +// +// ⛔ WHY IT IS WRITTEN THIS WAY. The obvious version of this test asserts that the draft's grant +// derives to the deadline the config used to carry. That number is a PRODUCT of the formula, so +// writing it down turns the test into a pin on a literal — and the next person to see the arithmetic +// land a hair under a round number "fixes" the round number back in and the pin agrees. So: no +// expected duration appears in this file as a number at all, and TestTheDeadlineTestQuotesNoDeadline +// enforces that mechanically. +func TestTheAttemptDeadlineIsDerivedFromTheBudget(t *testing.T) { + // The claim this whole change rests on, stated as arithmetic: the deadline the shipping config + // carried is what the vendor's own rate gives for the DRAFT's budget, to within a second — which is + // why it was right there and wrong everywhere else. Asserted as agreement between two independently + // sourced numbers, one from the vendor and one from the config, with neither written down. + bare := RetryProfile{} + if got, want := bare.deriveDeadline(draftGrant), vendorSeconds(draftGrant); got != want { + t.Fatalf("draft grant %d derives to %s, the vendor's own rate says %s", draftGrant, got, want) + } + if got, want := bare.deriveDeadline(editorGrant), vendorSeconds(editorGrant); got != want { + t.Fatalf("editor grant %d derives to %s, the vendor's own rate says %s", editorGrant, got, want) + } + // The editor's doubled budget needs MORE than three times the draft's deadline. That is the whole + // finding: one attempt_s covered both, so the doubling axis was buying a budget the clock could + // never spend. The multiple is derived, not chosen — editorGrant/draftGrant. + if bare.deriveDeadline(editorGrant) <= 3*bare.deriveDeadline(draftGrant) { + t.Fatalf("the editor's doubled budget must need materially longer than the draft's: draft=%s editor=%s", + bare.deriveDeadline(draftGrant), bare.deriveDeadline(editorGrant)) + } + + // Doubling the budget doubles the GENERATION time and leaves the queue wait alone — the property + // that makes the slack a summand rather than a factor. True of a derivation, false of a constant, + // and false of any formula that amortized the wait over the budget. + withSlack := RetryProfile{TokensPerSecFloor: 50, QueueSlack: 7 * time.Minute} + n, twoN := 4000, 8000 + single := withSlack.deriveDeadline(n) - withSlack.QueueSlack + double := withSlack.deriveDeadline(twoN) - withSlack.QueueSlack + if double != 2*single { + t.Fatalf("doubling the budget must double the generation term and nothing else: %s → %s (want %s)", + single, double, 2*single) + } + if withSlack.deriveDeadline(n) != withSlack.QueueSlack+single { + t.Fatalf("the vendor's documented queue wait is added whole, not amortized") + } + + // A configured floor is USED. Without this the two arms above would both pass on a build that + // ignored the field and always took the default. + fast := RetryProfile{TokensPerSecFloor: 2 * (vendorHourlyTokenBudget / vendorBudgetWindow.Seconds())} + if got := fast.deriveDeadline(editorGrant); got != vendorSeconds(editorGrant)/2 { + t.Fatalf("a floor twice the vendor default must halve the derived time, got %s", got) + } +} + +// TestTheDeadlineClampsAtBothEnds pins that attempt_s is a FLOOR and attempt_max_s a ceiling — the +// property that lets this land on every shipping config at once, because no provider can lose a second +// it has today. +func TestTheDeadlineClampsAtBothEnds(t *testing.T) { + floor := 5 * time.Minute + p := RetryProfile{AttemptTimeout: floor, TokensPerSecFloor: 50} + // A call small enough to derive under the configured deadline keeps the configured one. + if got := p.DeadlineFor(1); got != floor { + t.Fatalf("attempt_s is the FLOOR: a tiny call must keep it, got %s want %s", got, floor) + } + // One big enough to derive over it gets the derived time — otherwise nothing about this change works. + if got := p.DeadlineFor(editorGrant); got != p.deriveDeadline(editorGrant) { + t.Fatalf("a call the configured deadline cannot cover must get the derived one, got %s", got) + } + // And attempt_max_s bounds it. + capped := p + capped.AttemptMax = floor + time.Minute + if got := capped.DeadlineFor(editorGrant); got != capped.AttemptMax { + t.Fatalf("attempt_max_s must bound the derivation, got %s want %s", got, capped.AttemptMax) + } + // An unset ceiling is inert rather than zero — a zero read as a ceiling would cut every call to nothing. + if got := p.DeadlineFor(editorGrant); got <= 0 { + t.Fatalf("an unset attempt_max_s must not act as a zero ceiling, got %s", got) + } +} + +// TestAnUnsetOrBrokenFloorFallsBackToTheVendorDefault: an empty field, a zero and a negative must all +// give the vendor rate. Not a division by zero, and — the one that would be silent — not a deadline of +// nothing, which would cut every call the instant it went out and bill every one of them as an estimate. +func TestAnUnsetOrBrokenFloorFallsBackToTheVendorDefault(t *testing.T) { + want := vendorSeconds(editorGrant) + for _, floor := range []float64{0, -1, -1e9} { + p := RetryProfile{TokensPerSecFloor: floor} + if got := p.deriveDeadline(editorGrant); got != want { + t.Fatalf("floor %v must fall back to the vendor rate: got %s want %s", floor, got, want) + } + if got := p.deriveDeadline(0); got != 0 && p.QueueSlack == 0 { + t.Fatalf("a zero budget derives a zero generation term, got %s", got) + } + } +} + +// TestTheDeadlineTestQuotesNoDeadline is the guard on the guard. It reads THIS FILE and asserts that +// the four numbers the derivation produces for the shipping grants appear nowhere in it — so a later +// session cannot "simplify" the arithmetic above into the constant it evaluates to and keep a green +// test that pins nothing. +// +// The control value is printed with the negative, because "no matches" and "the file was not read" are +// the same output and only one of them is a passing test. +func TestTheDeadlineTestQuotesNoDeadline(t *testing.T) { + src, err := os.ReadFile("attemptcut_test.go") + if err != nil { + t.Fatal(err) + } + // ⛔ THE BANNED LIST IS DERIVED, NOT TYPED, and the first version of this test failed against + // itself for exactly the reason the ban exists: writing the four numbers into the pattern put them + // in the file. Deriving them from the formula also means the ban follows the formula — change the + // vendor's published rate and this test bans the NEW products of it without anyone remembering to. + var banned []string + for _, grant := range []int{draftGrant, editorGrant} { + secs := int(vendorSeconds(grant).Round(time.Second).Seconds()) + // The neighbours too: a rounded-up «nicer» value is precisely the shape a later session would + // substitute for the derivation. + banned = append(banned, strconv.Itoa(secs-1), strconv.Itoa(secs), strconv.Itoa(secs+1)) + } + hits := regexp.MustCompile(`\b(`+strings.Join(banned, "|")+`)\b`).FindAllString(string(src), -1) + // The control: two numbers that ARE in the file, sought by the same instrument. Without it «no + // matches» and «the search never ran» print identically, and only one of them is a passing test. + control := regexp.MustCompile(`\b(`+strconv.Itoa(draftGrant)+`|`+strconv.Itoa(editorGrant)+`)\b`).FindAllString(string(src), -1) + t.Logf("read %d bytes of attemptcut_test.go; banned set %v → %d hit(s); control (the two grants) → %d hit(s)", + len(src), banned, len(hits), len(control)) + if len(control) < 2 { + t.Fatalf("the control literals are absent — this test is not reading what it thinks it is") + } + if len(hits) != 0 { + t.Fatalf("the derived deadlines must not appear as literals in this file; found %v — a pin on a "+ + "number the formula produced is a pin on nothing", hits) + } +} + +// --- the delivery boundary --- + +// hangingTLS is a listener that completes the TCP connection and then says nothing at all. A client +// speaking TLS to it blocks in the handshake, so the request is NEVER WRITTEN — the one shape in which +// «nothing was bought» is true by construction, and the one this fixture exists to keep distinct from +// every other failure. It is deliberately not an httptest server: an HTTP server that merely sleeps +// receives the request first, which is the opposite case. +func hangingTLS(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { + var held []net.Conn + for { + c, err := ln.Accept() + if err != nil { + for _, h := range held { + h.Close() + } + return + } + held = append(held, c) // accepted, never spoken to + select { + case <-done: + return + default: + } + } + }() + t.Cleanup(func() { close(done); ln.Close() }) + return "https://" + ln.Addr().String() +} + +func cutProfile(d time.Duration) RetryProfile { + // A floor high enough that the derivation never exceeds the deadline under test: these fixtures are + // about the cut, not about the arithmetic, and a derived deadline would make them slow. + return RetryProfile{AttemptTimeout: d, MaxAttempts: 3, BackoffBase: time.Millisecond, + BackoffCap: 5 * time.Millisecond, TokensPerSecFloor: 1e9} +} + +// holdUntilCut blocks a fixture's handler until the client walks away — and NEVER longer than a bound +// of its own. `<-r.Context().Done()` alone reads correct and deadlocks the fixture: httptest.Close +// waits for outstanding handlers, and a handler waiting for a disconnect the server has not noticed +// yet waits for the Close that is waiting for it. A hang has no colour — neither the battery nor a +// mutation reports one — so the bound is the difference between a red test and a silent one. +func holdUntilCut(r *http.Request) { + select { + case <-r.Context().Done(): + case <-time.After(3 * time.Second): + } +} + +// TestDeliveryIsWhatSeparatesAPurchaseFromAFailure walks the three rows of the boundary table on a +// real client: a request that went out and got no headers, one that went out and got an early 200 with +// nothing in it, and one that never went out at all. All three used to arrive at the same branch. +func TestDeliveryIsWhatSeparatesAPurchaseFromAFailure(t *testing.T) { + t.Run("delivered, headers never came, our deadline", func(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + holdUntilCut(r) // the provider has the request and is «generating» + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(150 * time.Millisecond)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("want an AttemptCutError, got %T %v", err, err) + } + if !cut.Delivered { + t.Fatalf("the server RECEIVED the request; Delivered must be true") + } + if cut.AfterHeaders { + t.Fatalf("no response byte ever arrived; AfterHeaders must be false") + } + if cut.Cause != CutBySelfDeadline { + t.Fatalf("our own deadline fired with the parent alive; cause = %s", cut.Cause) + } + // The other side of the count: one delivery reports one, or the field says nothing anywhere. + if cut.Deliveries != 1 { + t.Fatalf("this request reached the provider once; Deliveries=%d", cut.Deliveries) + } + // NOT retried: the provider is still generating what we just stopped listening to. + if got := calls.Load(); got != 1 { + t.Fatalf("the server must have been asked exactly once, got %d", got) + } + }) + + t.Run("delivered, early 200 with empty lines, our deadline", func(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte("\n")) // the vendor's documented «still queued» reply + w.(http.Flusher).Flush() + holdUntilCut(r) + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(150 * time.Millisecond)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("want an AttemptCutError, got %T %v", err, err) + } + if !cut.Delivered || !cut.AfterHeaders { + t.Fatalf("a 200 arrived over a delivered request: Delivered=%t AfterHeaders=%t", cut.Delivered, cut.AfterHeaders) + } + // The BOTH-SIDES half of the field: whitespace here, JSON in the next case. A field asserted on + // one side only says nothing about whether it is computed at all. + if !cut.WhitespaceOnly { + t.Fatalf("the body was a newline; WhitespaceOnly must be true (bytes read: %d)", cut.BytesRead) + } + if got := calls.Load(); got != 1 { + t.Fatalf("a 200 that carried nothing must not be re-bought, got %d calls", got) + } + }) + + t.Run("delivered, cut mid-JSON, our deadline", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"id":"x","choices":[{"message":{"content":"половина`)) + w.(http.Flusher).Flush() + holdUntilCut(r) + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(150 * time.Millisecond)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("want an AttemptCutError, got %T %v", err, err) + } + if cut.WhitespaceOnly { + t.Fatalf("the body carried a fragment of a real answer; WhitespaceOnly must be false") + } + if cut.BytesRead == 0 { + t.Fatalf("BytesRead must carry what did arrive") + } + }) + + t.Run("NOT delivered", func(t *testing.T) { + base := hangingTLS(t) + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: base, Profile: cutProfile(120 * time.Millisecond)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + if err == nil { + t.Fatal("want an error") + } + var cut *AttemptCutError + if errors.As(err, &cut) { + t.Fatalf("the request never went out — treating it as a delivered cut would settle money for "+ + "a call nobody received: %v", cut) + } + }) +} + +// TestACancelledRunKeepsBothTruthsInOneError is the exit-contract fixture, and both assertions live in +// ONE test on ONE error for a reason: they are the two halves of a single requirement. The process +// decides its exit code with errors.Is(err, context.Canceled), and the runner decides whether money is +// owed with errors.As(err, &AttemptCutError). Splitting them across two tests would let a change +// satisfy either one while breaking the other — a stop that exits with the wrong code, or a stop that +// silently loses the money for a call already on the wire. +func TestACancelledRunKeepsBothTruthsInOneError(t *testing.T) { + arrived := make(chan struct{}) + var once atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if once.CompareAndSwap(false, true) { + close(arrived) + } + holdUntilCut(r) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-arrived // cancel only AFTER the provider has the request: this is the delivered case + cancel() + }() + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(10 * time.Second)}, nil) + _, err := c.Complete(ctx, LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + if !errors.Is(err, context.Canceled) { + t.Fatalf("the exit contract reads errors.Is(err, context.Canceled); a stop that stops reporting "+ + "itself as one gives the run a foreign exit code. got %T %v", err, err) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("the same error must still say WHAT the stop interrupted, or the money for a delivered "+ + "call is lost with it. got %T %v", err, err) + } + if cut.Cause != CutByParent { + t.Fatalf("cause = %s, want %s", cut.Cause, CutByParent) + } + if !cut.Delivered { + t.Fatalf("the provider had the request when the run was stopped; Delivered must be true") + } +} + +// TestABrokenConnectionAfterDeliveryIsRetriedOnceAndOnlyOnce: the third cause. A socket that dies after +// the request went out is transport-shaped, so it is worth one more call — and exactly one, because the +// cap is keyed on DELIVERY rather than on a 2xx. Under the old key («did a 2xx arrive») a provider that +// answers 200 while still queueing could be re-asked under the full attempt budget. +func TestABrokenConnectionAfterDeliveryIsRetriedOnceAndOnlyOnce(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"id":"x","choices":[`)) + w.(http.Flusher).Flush() + // Drop the connection mid-body: the client's read fails with neither deadline expired. + hj, ok := w.(http.Hijacker) + if !ok { + t.Error("the fixture needs a hijackable response writer") + return + } + conn, _, err := hj.Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(5 * time.Second)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("want an AttemptCutError, got %T %v", err, err) + } + if cut.Cause != CutByConnection { + t.Fatalf("cause = %s, want %s", cut.Cause, CutByConnection) + } + if got := calls.Load(); got != 2 { + t.Fatalf("a delivered request whose socket broke is worth ONE retry and no more (max_attempts=3), got %d", got) + } + if !cut.Delivered { + t.Fatalf("the request reached the provider both times; Delivered must be true") + } + // ⛔ AND THE CALLER IS TOLD HOW MANY TIMES. The store books spend only through a checkpoint and both + // deliveries share one key, so ONE estimate covers both — an under-count, which is the ratified + // direction, but a SILENT one is what row 360 was opened about. The number is what makes it readable. + if cut.Deliveries != 2 { + t.Fatalf("the provider was asked twice and the ledger will book once; the error must carry the "+ + "count or the gap is invisible again. got Deliveries=%d", cut.Deliveries) + } +} + +// TestTheFloorWarningIsSaidOnceAndOnlyWhenItApplies: the operator notice that makes a new provider +// usable as DATA. It has to fire when the derived deadline overrides the configured one — and it has to +// stay silent otherwise, because a warning printed on every healthy call is a warning nobody reads. +func TestTheFloorWarningIsSaidOnceAndOnlyWhenItApplies(t *testing.T) { + fires := func(p RetryProfile, maxTokens int, calls int) int { + var seen atomic.Int32 + log := newCountingLogger(&seen, "no measured generation speed") + c := newOpenAIClient("p", "http://127.0.0.1:1", "", p, nil, nil, log) + for i := 0; i < calls; i++ { + c.attemptDeadline(context.Background(), maxTokens) + } + return int(seen.Load()) + } + // No floor, and a budget the configured deadline cannot cover: said, once, however many calls. + noFloor := RetryProfile{AttemptTimeout: time.Second} + if got := fires(noFloor, editorGrant, 5); got != 1 { + t.Fatalf("the notice must be said exactly once per client, got %d", got) + } + // No floor, but every call fits inside the configured deadline: nothing to warn about. + if got := fires(RetryProfile{AttemptTimeout: time.Hour}, editorGrant, 5); got != 0 { + t.Fatalf("a provider whose calls fit inside its own attempt_s must hear nothing, got %d", got) + } + // A measured floor is the answer to the notice, so it silences it. + if got := fires(RetryProfile{AttemptTimeout: time.Second, TokensPerSecFloor: 50}, editorGrant, 5); got != 0 { + t.Fatalf("a provider with a measured floor must hear nothing, got %d", got) + } +} + +// countingHandler counts log records whose message contains a substring. A slog.Handler rather than a +// buffer scan: asserting on a substring of a shared buffer is the shape that gives both false reds and +// quiet greens (D39.171), and the question here — «how many times was this record emitted» — is a count +// the handler can answer exactly. +type countingHandler struct { + n *atomic.Int32 + needle string +} + +func (h countingHandler) Enabled(context.Context, slog.Level) bool { return true } +func (h countingHandler) Handle(_ context.Context, r slog.Record) error { + if strings.Contains(r.Message, h.needle) { + h.n.Add(1) + } + return nil +} +func (h countingHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h countingHandler) WithGroup(string) slog.Handler { return h } + +func newCountingLogger(n *atomic.Int32, needle string) *slog.Logger { + return slog.New(countingHandler{n: n, needle: needle}) +} + +// TestAReplyThatOutrunsOurOwnWriteIsStillDelivered is the money leak an adversarial pass measured on +// this pack, reproduced as a pin. +// +// net/http hands back a response as soon as one arrives; WroteRequest fires later, from the write +// loop. So a provider that answers EARLY — DeepSeek documents doing exactly that while a request waits +// to be scheduled — can have its 200 overtake our own callback while the request body is still +// draining. Read as «not delivered», that call books $0 and is retried under the full attempt budget: +// the defect this whole file exists to remove, entering through the door meant to close it. +// +// The fixture forces the ordering by making the request big enough that the write cannot finish inside +// the socket buffer, and by answering before reading it. On loopback that needs megabytes; on a real +// network the window is the send buffer and the congestion window, i.e. orders of magnitude smaller. +func TestAReplyThatOutrunsOurOwnWriteIsStillDelivered(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + done := make(chan struct{}) + defer close(done) + go func() { + for { + c, aerr := ln.Accept() + if aerr != nil { + return + } + go func(c net.Conn) { + defer c.Close() + // Answer the moment the request LINE is in, promise a body far longer than what is + // sent — and then HOLD, reading nothing more. Two things follow, and both are needed: + // the client's own body write never finishes (nobody is draining it) and its read of + // the reply never finishes either, so what ends the call is OUR OWN deadline. That is + // what makes this deterministic. An earlier version closed the socket here instead, and + // under load the write failed first, leaving no reply to overtake anything — green + // alone, red inside a six-package baseline run. + br := bufio.NewReader(c) + if _, rerr := br.ReadString('\n'); rerr != nil { + return + } + fmt.Fprint(c, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 4096\r\n\r\n\n") + select { + case <-done: + case <-time.After(10 * time.Second): + } + }(c) + } + }() + + // A prompt big enough that the request body cannot be written into the socket buffer in one go, so + // the write is still outstanding when the reply lands. + huge := strings.Repeat("длинный исходный текст главы. ", 200000) + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: "http://" + ln.Addr().String(), Profile: cutProfile(700 * time.Millisecond)}, nil) + _, err = c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: huge}}, MaxTokens: 16}) + + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("a 200 came back over this request, so the provider HAS it and may bill for it; booking "+ + "$0 here is the leak this pack closes, arriving through its own door. got %T %v", err, err) + } + if !cut.Delivered { + t.Fatalf("Delivered must be true once a response byte has arrived: %+v", cut) + } + if !cut.AfterHeaders { + t.Fatalf("a 200 arrived; AfterHeaders must be true: %+v", cut) + } + if cut.Cause != CutBySelfDeadline { + t.Fatalf("our own deadline is what ended this call; cause = %s (%+v)", cut.Cause, cut) + } +} + +// TestAFailedWriteIsNotADelivery is the other side of the same bit, and it had NO pin at all until an +// adversarial pass removed the guard and watched both packages stay green. +// +// `WroteRequest` fires with an ERROR when the write itself failed — the request did not reach the +// provider, nothing was generated, and settling an estimate would charge a reader for a call nobody +// received. The fixture kills the socket without reading the body and without answering, so the +// callback fires with a write error and no response byte ever arrives. +func TestAFailedWriteIsNotADelivery(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + c, aerr := ln.Accept() + if aerr != nil { + return + } + c.Close() // accepted, then dropped: the body write fails and nothing comes back + } + }() + + huge := strings.Repeat("длинный исходный текст главы. ", 200000) + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: "http://" + ln.Addr().String(), Profile: cutProfile(3 * time.Second)}, nil) + _, err = c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: huge}}, MaxTokens: 16}) + + if err == nil { + t.Fatal("want an error") + } + var cut *AttemptCutError + if errors.As(err, &cut) { + t.Fatalf("the write itself failed and no reply ever came: the provider received nothing and owes "+ + "nothing, so this must NOT be a paid cut. got %+v", cut) + } +} + +// TestAStopDuringABackoffKeepsTheEvidence is the SECOND cancellation exit, and it was still throwing +// the evidence away after the first one stopped doing so. +// +// A run stopped during a retry backoff has an attempt behind it that may already have been delivered +// and billed. Returning a bare context error there leaves the runner nothing to settle and the chunk +// with no mark at all — the unexplained hole the whole cancelled-position machinery exists to prevent. +// The window is the entire sleep, up to a minute on the shipping config, and it opens exactly on the +// runs where a provider is flapping and an operator is therefore reaching for the stop. +// +// Both truths are asserted on one error for the same reason as the in-flight case: the exit code reads +// one and the money reads the other. +func TestAStopDuringABackoffKeepsTheEvidence(t *testing.T) { + arrived := make(chan struct{}) + var once atomic.Bool + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"id":"x","choices":[`) + w.(http.Flusher).Flush() + hj, ok := w.(http.Hijacker) + if !ok { + t.Error("the fixture needs a hijackable response writer") + return + } + conn, _, err := hj.Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() // a delivered request whose socket dies: retryable, so a backoff follows + if once.CompareAndSwap(false, true) { + close(arrived) + } + })) + defer srv.Close() + + // ⚠ THE STOP IS DELAYED, and the delay is what makes this test about the backoff at all. Cancelling + // the instant the handler closes the socket is a RACE with the loop's own bookkeeping: the second + // attempt often starts first and is itself cut, so the error under test becomes a `cancelled` from + // somewhere else entirely. Measured before the delay was added: 28 failures in 80 runs, every one of + // them the wrong attempt. The window here is half a second inside a three-second sleep, and the + // call-count assertion below says outright when the fixture missed it. + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-arrived + time.Sleep(500 * time.Millisecond) + cancel() + }() + p := cutProfile(5 * time.Second) + p.BackoffBase, p.BackoffCap = 3*time.Second, 3*time.Second + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: p}, nil) + _, err := c.Complete(ctx, LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + if n := calls.Load(); n != 1 { + t.Fatalf("the loop was meant to be ASLEEP before its retry when the stop landed; the server was "+ + "asked %d time(s), so this run measured a different moment", n) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("a stopped run must still leave as a cancellation, got %T %v", err, err) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("the delivered attempt behind the backoff must survive the stop, or its money is lost "+ + "and the chunk is left with no mark at all. got %T %v", err, err) + } + if cut.Cause != CutByConnection || !cut.Delivered { + t.Fatalf("the evidence must be the attempt's own, not a re-labelled cancellation: %+v", cut) + } +} + +// TestARefusalIsNotAPurchaseEvenWhenTheReplyOutrunsTheWrite is the regression the FIX for the early-200 +// leak introduced, and it is the more expensive of the two directions. +// +// A provider that REFUSES — 401, 403, 413, a quota 429 — and drops the connection while our request body +// is still writing gives GotFirstResponseByte=true and a WRITE error. http.Client.Do then returns the +// write failure, so the status line is never in our hands and the branch that reads it is never reached. +// Taking the response byte alone as proof of delivery paid an estimate for every one of those: measured +// at 22 refusals in 25, one of them $0.80 booked for a request the provider declined — and the +// delivered-cut retry sent the whole body to it a second time. +// +// So the reply counts as evidence only where a reply actually reached us. Here it did not. +func TestARefusalIsNotAPurchaseEvenWhenTheReplyOutrunsTheWrite(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + c, aerr := ln.Accept() + if aerr != nil { + return + } + go func(c net.Conn) { + // Refuse the moment the request line is in, then RESET — the body is still writing, so + // the client sees a write error and never gets a status line. + br := bufio.NewReader(c) + if _, rerr := br.ReadString('\n'); rerr != nil { + c.Close() + return + } + fmt.Fprint(c, "HTTP/1.1 401 Unauthorized\r\nContent-Length: 9\r\n\r\nno access") + if tc, ok := c.(*net.TCPConn); ok { + _ = tc.SetLinger(0) // RST rather than a graceful close + } + c.Close() + }(c) + } + }() + + huge := strings.Repeat("длинный исходный текст главы. ", 200000) + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: "http://" + ln.Addr().String(), Profile: cutProfile(2 * time.Second)}, nil) + _, err = c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: huge}}, MaxTokens: 16}) + + if err == nil { + t.Fatal("a refused request must not report success") + } + var cut *AttemptCutError + if errors.As(err, &cut) { + t.Fatalf("the provider REFUSED this request and generated nothing; booking an estimate for it "+ + "charges a reader for a call that never ran, and the delivered-cut retry sends the whole body "+ + "again. got %+v", cut) + } +} + +// TestARedirectIsNotADelivery: `Do` spans the WHOLE redirect chain and the delivery trace does not reset +// between its legs, so the first leg reaching a redirector sets WroteRequest for good — and a second leg +// whose connect is REFUSED still looked delivered. Measured through the ledger before the fix: +// $0.001056 booked for a `connect: connection refused` that never put a byte on any wire. +// +// The client now stops at the 3xx instead of chasing it, so a provider endpoint that redirects fails +// loud with its status rather than quietly costing money at the other end of the hop. +func TestARedirectIsNotADelivery(t *testing.T) { + // ⛔ EVERY CLIENT THIS PACKAGE BUILDS, and not just the cloud one. The guard first went only on + // keepAliveHTTPClient, and a mutation that removed it from the LOCAL client SURVIVED: this fixture + // exercised the cloud path alone, so a hole in the other constructor was invisible. They share + // attempt() and they share the delivery trace, so they must share the policy. + clients := map[string]func(base string) LLMClient{ + "cloud": func(base string) LLMClient { + return NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: base, Profile: cutProfile(2 * time.Second)}, nil) + }, + "local": func(base string) LLMClient { + return NewLocalClient(LocalConfig{BaseURL: base, Model: "m", Profile: cutProfile(2 * time.Second)}, nil) + }, + // The third, and it was missing while the comment above already said «every». It is a deprecated + // adapter with no live provider, but it builds its own client, it sends `x-api-key`, and + // net/http strips only Authorization across a host change — so its redirect carried the key to + // the target. A fixture that says «every» and builds two of three is the shape of green that + // let the local client keep following redirects. + "anthropic": func(base string) LLMClient { + return NewAnthropicClient(AnthropicConfig{BaseURL: base, APIKey: "k", Profile: cutProfile(2 * time.Second)}, nil) + }, + } + for name, build := range clients { + t.Run(name, func(t *testing.T) { + dead, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + target := dead.Addr().String() + dead.Close() // the control: this address refuses connections, so nothing can be delivered there + + var hops atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hops.Add(1) + http.Redirect(w, r, "http://"+target+"/chat/completions", http.StatusTemporaryRedirect) + })) + defer srv.Close() + + _, err = build(srv.URL).Complete(context.Background(), + LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + if err == nil { + t.Fatal("a provider endpoint that redirects must fail, not silently follow") + } + var cut *AttemptCutError + if errors.As(err, &cut) { + t.Fatalf("nothing was delivered anywhere: the redirector answered and the target refuses "+ + "connections, so booking money for this charges a reader for a call no endpoint "+ + "received. got %+v", cut) + } + // It fails as the status it is — the operator is told the base_url sends them somewhere else. + var hse *HTTPStatusError + if !errors.As(err, &hse) || hse.Status != http.StatusTemporaryRedirect { + t.Fatalf("the 3xx itself must reach the operator, got %T %v", err, err) + } + if n := hops.Load(); n == 0 { + t.Fatalf("the redirector was never asked — this fixture measured nothing (hops=%d)", n) + } + }) + } +} + +// TestADeliveredCutSurvivesATerminalStatusLaterInTheChain: only the LAST error used to leave the retry +// loop, so a chain that cut a delivered request and then met a terminal 4xx returned the status alone. +// The runner then read «the request never went out», released the reservation and booked $0 for a +// generation the provider had made — and left the position with no mark either. Reachable in the ordinary +// way: a 400/401/403/413 on the retry after a broken socket. +func TestADeliveredCutSurvivesATerminalStatusLaterInTheChain(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + // First: a delivered request whose socket dies mid-body — retryable, and BILLABLE. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"id":"x","choices":[`) + w.(http.Flusher).Flush() + hj, ok := w.(http.Hijacker) + if !ok { + t.Error("the fixture needs a hijackable response writer") + return + } + conn, _, herr := hj.Hijack() + if herr != nil { + t.Error(herr) + return + } + conn.Close() + return + } + // Then: a terminal status, which ends the chain and used to erase the first attempt. + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `{"error":"forbidden"}`) + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(3 * time.Second)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + if n := calls.Load(); n != 2 { + t.Fatalf("the fixture needs both attempts to happen, got %d — it is measuring a different chain", n) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("the first attempt was DELIVERED and cut, so it is owed whatever ended the chain "+ + "afterwards; losing it books $0 for a generation the provider made and leaves the position "+ + "unmarked. got %T %v", err, err) + } + if !cut.Delivered || cut.Cause != CutByConnection { + t.Fatalf("the surviving evidence must be the cut's own: %+v", cut) + } + // And the status that ENDED the chain is still there — a caller must be able to see both. + var hse *HTTPStatusError + if !errors.As(err, &hse) || hse.Status != http.StatusForbidden { + t.Fatalf("the terminal status must also reach the caller, got %T %v", err, err) + } +} + +// TestTheKeepalivePairSitsOnTheTransportTheCloudClientBuilds: the h2 keepalive bounds are worth their +// comment only if they sit on the transport the cloud client actually uses, and reading them back is +// the only way to know. tuneHTTP2 is that code path and it hands back what it set; asking +// ConfigureTransports a second time on an already-built client answers an error and no transport, so a +// check written that way asserts nothing at all. +func TestTheKeepalivePairSitsOnTheTransportTheCloudClientBuilds(t *testing.T) { + // The control comes first: both bounds are non-zero, so a build that wired NOTHING cannot pass the + // comparisons below by matching zero against zero. + if h2ReadIdleTimeout <= 0 || h2PingTimeout <= 0 { + t.Fatalf("a zero bound disables the keepalive silently: %s / %s", h2ReadIdleTimeout, h2PingTimeout) + } + // ⛔ THE CLIENT THE ENGINE ACTUALLY BUILDS, not a clone tuned beside it. Asking tuneHTTP2 on a fresh + // transport of the test's own would pass even if keepAliveHTTPClient stopped calling it at all — + // which is the shape the write bound's pin had, and it is why that pin survived its own mutation. + c, h2 := buildCloudClient() + if c == nil || h2 == nil { + t.Fatal("the cloud client could not be configured for h2 — this test measured nothing") + } + if c.Transport == nil { + t.Fatal("the cloud client has no transport at all — the bounds below would be read off nothing") + } + if h2.ReadIdleTimeout != h2ReadIdleTimeout { + t.Fatalf("the read-idle bound is not on the transport: got %s want %s", h2.ReadIdleTimeout, h2ReadIdleTimeout) + } + if h2.PingTimeout != h2PingTimeout { + t.Fatalf("the ping bound is not on the transport: got %s want %s", h2.PingTimeout, h2PingTimeout) + } +} + +// billableCut answers 200 and flushes the first bytes of a body, then kills the socket: the provider +// ACKNOWLEDGED the request, so the cut is one we are billed for. +func billableCut(t *testing.T, w http.ResponseWriter) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"id":"x","choices":[`) + w.(http.Flusher).Flush() + hijackKill(t, w) +} + +// freeCut kills the socket with no status line at all: the request was delivered, nothing came back, and +// nobody is owed for it. +func freeCut(t *testing.T, w http.ResponseWriter) { t.Helper(); hijackKill(t, w) } + +func hijackKill(t *testing.T, w http.ResponseWriter) { + t.Helper() + hj, ok := w.(http.Hijacker) + if !ok { + t.Error("the fixture needs a hijackable response writer") + return + } + conn, _, err := hj.Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() +} + +// TestAPaidCutSurvivesAPlainRetryableLaterInTheChain: a chain can be billed for an attempt that is not +// the attempt whose error ends it. Here the money is on attempt 1 and the chain ends on a 503 — an +// ordinary retryable that owes nobody anything. If the loop returns only what ended it, the engine +// settles $0 for a generation the provider made. +func TestAPaidCutSurvivesAPlainRetryableLaterInTheChain(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + billableCut(t, w) + return + } + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprint(w, `{"error":"busy"}`) + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(3 * time.Second)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + if n := calls.Load(); n < 2 { + t.Fatalf("the fixture needs both attempts to happen, got %d — it is measuring a different chain", n) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("attempt 1 was DELIVERED, ACKNOWLEDGED and cut — the chain owes money for it whatever "+ + "ended it later. Returning only the 503 books $0 for a generation the provider made. got %T %v", err, err) + } + if !cut.Billable { + t.Fatalf("the surviving cut must be the BILLABLE one: %+v", cut) + } +} + +// TestAFreeCutLaterDoesNotMaskThePaidCutEarlier is the same mismatch one turn harder, and it is the case +// the type test could not see. Both attempts end in a cut, so «the chain already ends carrying one» is +// true — and wrong: the ending cut was never acknowledged and costs nothing, while the earlier one was +// and does. Money, not type, decides which one the caller must be handed. +func TestAFreeCutLaterDoesNotMaskThePaidCutEarlier(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + billableCut(t, w) + return + } + freeCut(t, w) + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(3 * time.Second)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + if n := calls.Load(); n < 2 { + t.Fatalf("the fixture needs both attempts to happen, got %d", n) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("some cut must reach the caller, got %T %v", err, err) + } + if !cut.Billable { + t.Fatalf("the caller settles from ONE error, so it must be the one with the money on it: a later "+ + "cut nobody acknowledged erased an earlier one the provider did. got Billable=%t cause=%s", + cut.Billable, cut.Cause) + } +} + +// TestTheCancellationExitCarriesWhatTheChainOwes covers the loop's OTHER cancellation exit — the one +// taken right after an attempt rather than during a backoff (that one has its own fixture). A run +// stopped here has a paid cut behind it and an ordinary retryable in front of it, and returning only +// «cancelled + the 503» leaves the runner nothing to settle and the chunk with no mark. +func TestTheCancellationExitCarriesWhatTheChainOwes(t *testing.T) { + var calls atomic.Int32 + secondLanded := make(chan struct{}) + stopped := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + billableCut(t, w) + return + } + // ⛔ SYNCHRONISED ON THE STOP, NOT ON A CLOCK. The loop checks the context AFTER the attempt + // returns, so the stop has to be in place before this handler answers — and a sleep here would + // make the fixture measure whichever of the two won the race that day. + close(secondLanded) + <-stopped + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprint(w, `{"error":"busy"}`) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-secondLanded + cancel() + close(stopped) + }() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(3 * time.Second)}, nil) + _, err := c.Complete(ctx, LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + if n := calls.Load(); n < 2 { + t.Fatalf("the fixture needs the stop to land on the SECOND attempt, got %d calls", n) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("a stopped run must still read as cancelled — the exit code depends on it. got %T %v", err, err) + } + var cut *AttemptCutError + if !errors.As(err, &cut) || !cut.Billable { + t.Fatalf("the stop exit must carry the cut the chain owes money for; without it the runner settles "+ + "nothing and the position is left unmarked. got %T %v", err, err) + } +} + +// TestAPaidCutAfterAFreeOneIsTheOneTheCallerSettlesFrom is the accumulator's own case, and it is the +// order the «first cut wins» rule cannot serve. Attempt 1 is delivered and never acknowledged — nobody +// is owed for it. Attempt 2 IS acknowledged and cut, so it is the one the engine must settle. Keeping +// whichever cut came first hands the caller the free one and books $0 for a generation that was made. +func TestAPaidCutAfterAFreeOneIsTheOneTheCallerSettlesFrom(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + switch calls.Add(1) { + case 1: + freeCut(t, w) // delivered, never acknowledged — costs nothing + case 2: + billableCut(t, w) // acknowledged, then cut — this is the money + default: + w.WriteHeader(http.StatusForbidden) // unreachable under the cap; here so a policy change fails loudly + fmt.Fprint(w, `{"error":"forbidden"}`) + } + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(3 * time.Second)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + // TWO attempts, not three: only a connection cut is retryable and the delivered-cut cap makes the + // second one terminal, so the chain ends ON the paid cut. That is the premise this test rests on — + // if a policy change ever lets a third attempt happen, the count says so instead of the assertions + // quietly measuring a different chain. + if n := calls.Load(); n != 2 { + t.Fatalf("the fixture needs exactly two attempts, got %d — it is measuring a different chain", n) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("a cut must reach the caller, got %T %v", err, err) + } + if !cut.Billable { + t.Fatalf("the chain produced a free cut and then a PAID one; the caller settles from one error, so "+ + "it must be the paid one. Keeping whichever came first books $0 for a generation the provider "+ + "made. got Billable=%t cause=%s", cut.Billable, cut.Cause) + } +} + +// TestResponseBytesWithoutAReplyAreNotAPurchase pins the half of the money predicate that nothing else +// reaches: `answered`. A cut is billable only when the provider ACKNOWLEDGED the request with a reply — +// `Billable = answered && afterHeaders` — and the two halves fail apart. Every other fixture exercises +// the second: no response byte, so both are false and dropping the first changes nothing. +// +// This is the case where they disagree. The peer writes RESPONSE BYTES that are not a reply — a broken +// status line — so the delivery trace's first-byte flag is set while `Do` still fails and no 2xx object +// ever exists. Without `answered &&`, that reads as money owed, and the engine books an estimate for a +// call whose provider answered nothing. +func TestResponseBytesWithoutAReplyAreNotAPurchase(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + c, aerr := ln.Accept() + if aerr != nil { + return + } + go func(c net.Conn) { + defer c.Close() + br := bufio.NewReader(c) + if _, rerr := br.ReadString('\n'); rerr != nil { + return + } + // Bytes on the wire, and not a reply: the trace sees a first response byte, the parser + // never sees a status line. + fmt.Fprint(c, "NOT-HTTP garbage from a broken proxy\r\n") + }(c) + } + }() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: "http://" + ln.Addr().String(), Profile: cutProfile(2 * time.Second)}, nil) + _, err = c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + if err == nil { + t.Fatal("a reply that is not a reply must not report success") + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + // Not a failure of the property under test: the request WAS delivered, so it is a cut. If the + // engine ever stops classifying it as one, this fixture is measuring nothing and says so. + t.Fatalf("premise broken: the request was delivered and the peer answered garbage, so this is a "+ + "cut. got %T %v", err, err) + } + if !cut.AfterHeaders { + t.Fatalf("premise broken: the fixture must produce a FIRST RESPONSE BYTE, otherwise both halves of "+ + "the money predicate are false and this test cannot tell them apart: %+v", cut) + } + if cut.Billable { + t.Fatalf("response BYTES are not a reply. The provider acknowledged nothing, so nobody is owed for "+ + "this call; booking it charges a reader for a generation that was never made. %+v", cut) + } +} + +// TestTheDeliveryCountCountsDeliveriesNotCuts pins the number the operator reads. The ledger line the +// engine writes beside a cut says «the provider was asked N times; ONE estimate is booked for all of +// them» — it is the only place the gap between what was generated and what was billed becomes visible, +// and it was counting CUTS. A chain that was answered a 503 and then cut delivered twice and reported +// one. +func TestTheDeliveryCountCountsDeliveriesNotCuts(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) // delivered: the peer read the request to answer it + fmt.Fprint(w, `{"error":"busy"}`) + return + } + billableCut(t, w) + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(3 * time.Second)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + // THREE deliveries: the 503, the cut, and the one retry a delivered cut is worth. The count is the + // premise as well as the subject — if the retry policy ever changes, this says so instead of letting + // the assertion below quietly measure a different chain. + if n := calls.Load(); n != 3 { + t.Fatalf("the fixture needs three attempts — a 503, a cut, and the cut's one retry — got %d", n) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("premise broken: the chain must end on a cut for the count to be carried anywhere: %T %v", err, err) + } + if cut.Deliveries != 3 { + t.Fatalf("the request reached the provider THREE times — a 503 is an answer, so the peer read it — and "+ + "one estimate is booked for both. Counting only the cuts makes the operator's «asked N times» "+ + "line understate a mixed chain, which is the one line where that gap is visible. got %d", + cut.Deliveries) + } +} + +// TestTheStopExitStillReadsAsCancelledWhenTheAttemptDidNot closes a hole this pack's own pin had, and the +// hole is instructive: the fixture beside it drives the loop through the WIRE, and there a stop lands on +// a request in flight, so the attempt's own error is already a parent-cancelled cut carrying +// context.Canceled. Its «a stopped run must still read as cancelled» assertion was therefore satisfied by +// the cut's Parent no matter what the loop's exit did — and removing the exit's `cancelledDuring` +// survived the whole battery. +// +// The case that has nothing to lean on is the one an operator actually meets: an attempt fails with an +// ORDINARY error — a 503 — and the stop arrives immediately after, before the loop decides to retry. +// Nothing in that error knows about the context, so only the exit can carry the cancellation out; if it +// does not, `tmctl` exits 1 instead of 5 and the stream publishes `failed` where a person pressed stop. +// +// It drives retryLoop directly because the wire cannot produce this ordering on purpose: the window +// between an attempt returning and the loop reading the context is a few instructions wide, and a +// fixture that raced for it would be measuring the scheduler. +func TestTheStopExitStillReadsAsCancelledWhenTheAttemptDidNot(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + plain := &HTTPStatusError{Status: http.StatusServiceUnavailable, Body: "busy"} + calls := 0 + _, err := retryLoop(ctx, cutProfile(time.Second), "p", nil, func() (*openAIResponse, bool, error) { + calls++ + cancel() // the stop lands while this attempt is returning, not while it is in flight + return nil, true, plain + }) + if calls != 1 { + t.Fatalf("premise broken: the loop must take the stop exit on the FIRST attempt, got %d calls", calls) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("a stopped run must read as cancelled even when the attempt that was in flight failed of "+ + "its own accord: the exit code (5, not 1) and the `stopped` the stream publishes both hang off "+ + "this. The attempt's error knows nothing about the context, so only this exit can carry it. got %T %v", + err, err) + } + // The control: the attempt's own error must ALSO survive, or the exit would be trading one truth for + // the other — the operator needs to know a 503 is why the engine was retrying when the stop arrived. + if !errors.Is(err, plain) { + t.Fatalf("the attempt's own error must reach the caller beside the cancellation: %T %v", err, err) + } +} + +// TestTheVendorsPublishedPairIsWhatTheVendorPublishes pins the two numbers every derived deadline is +// built from, and it exists because the formula test beside it cannot: both of its sides read these same +// constants, so corrupting them moves the expectation with the result and the identity still holds. +// Measured on a copy: `vendorHourlyTokenBudget` quadrupled, and `internal/llm`, `internal/config` and the +// rest of the battery all stayed green. +// +// The direction is money and it is this pack's own subject. A budget four times too generous makes every +// derived deadline four times too SHORT, so calls the provider is still generating are cut by us — and +// since this pack, a self-cut that the provider acknowledged is PAID FOR. Five of the eight catalogued +// providers run on this default. +// +// ⚠ THESE ARE SOURCE NUMBERS, NOT PRODUCTS, which is why quoting them here does not violate +// TestTheDeadlineTestQuotesNoDeadline: that gate bans the DERIVED seconds, because a pin on a number the +// formula produced is a pin on nothing. A citation of the vendor's published pair is the opposite — it is +// the one place the arithmetic touches the outside world, and it belongs written down. +func TestTheVendorsPublishedPairIsWhatTheVendorPublishes(t *testing.T) { + // anthropic-sdk-go's CalculateNonStreamingTimeout budgets one hour per 128 000 output tokens. That + // pair is the vendor's, not ours: changing either is adopting a different vendor's arithmetic, and it + // must be a decision someone makes on purpose rather than an edit nothing notices. + if vendorHourlyTokenBudget != 128000 { + t.Fatalf("the vendor's published token budget is 128000 per window; this build says %d. Every "+ + "provider without its own measured floor derives its deadline from this number, and a budget "+ + "too generous makes the deadline too SHORT — cutting calls the provider is still generating, "+ + "which this pack now charges for", vendorHourlyTokenBudget) + } + if vendorBudgetWindow != time.Hour { + t.Fatalf("the vendor's published window is one hour; this build says %s", vendorBudgetWindow) + } + // The control: the pair is actually what the derivation uses, not two constants sitting beside it. + // Without this the assertions above would hold on a build that derived from something else entirely. + if got := (RetryProfile{}).deriveDeadline(vendorHourlyTokenBudget); got != vendorBudgetWindow { + t.Fatalf("a grant of exactly the vendor's hourly budget must derive to exactly its window — that "+ + "identity is what makes the two constants above the SOURCE of the derivation rather than "+ + "decoration beside it. got %s, want %s", got, vendorBudgetWindow) + } +} + +// TestAnOlderCutCarriesTheChainsFinalDeliveryCount is the count's other half, and the one a chain that +// ends on a cut cannot show. Here the money is on attempt 1 and the chain goes on WITHOUT cutting again: +// two 503s, then exhaustion. `chainError` hands up attempt 1's cut because it is the one that owes — and +// the number stamped on it when it was born knows about one delivery out of three. +// +// Nothing in the error tree can repair that: the later deliveries were answers, not cuts, so no cut in +// the tree has ever seen them. Only the chain's own counter has. +func TestAnOlderCutCarriesTheChainsFinalDeliveryCount(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + billableCut(t, w) + return + } + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprint(w, `{"error":"busy"}`) + })) + defer srv.Close() + + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: cutProfile(3 * time.Second)}, nil) + _, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16}) + + // The premise is the shape, not just the count: a cut FIRST, then deliveries that are not cuts. + if n := calls.Load(); n != 3 { + t.Fatalf("the fixture needs a cut and then two plain answers, got %d attempts", n) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("premise broken: the chain owes money for attempt 1 and must hand that cut up: %T %v", err, err) + } + if cut.Deliveries != 3 { + t.Fatalf("the provider was asked THREE times and one estimate covers all of them; the cut handed "+ + "up was born on attempt 1 and reports %d. The later deliveries were answers, not cuts, so no "+ + "cut in the error tree has ever seen them — only the chain's counter has, and the ledger line "+ + "built from this number under-states the gap it exists to show", cut.Deliveries) + } +} + +// TestAnUndeliveredAttemptIsNotCountedAsAsking pins the predicate the count rests on. Every other +// fixture here tells deliveries apart from CUTS; none tells them apart from ATTEMPTS, because none has +// an attempt that never reached the provider. So `askedToGenerate++` made unconditional survived +// everything: the number would say the provider was asked twice when it was asked once, in the one line +// where the generated-versus-billed gap is published. +func TestAnUndeliveredAttemptIsNotCountedAsAsking(t *testing.T) { + var conns atomic.Int32 + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + c, aerr := ln.Accept() + if aerr != nil { + return + } + go func(c net.Conn) { + defer c.Close() + if conns.Add(1) == 1 { + // The first attempt never reaches the provider: accepted and dropped WITHOUT reading, + // so the huge body's write fails partway and nothing was asked of anybody. A reset + // instead of this plain close would let the body land in the socket buffer first, and + // the trace would then be right to call it delivered — which is a different case. + return + } + // The second DOES reach it: the whole request is READ before anything is answered, so + // the client's write completes and the delivery trace says so. Answering early — the + // shape of the branch above — would make this attempt a failed write too, and the + // fixture would then be comparing two undelivered attempts. + br := bufio.NewReader(c) + req, rerr := http.ReadRequest(br) + if rerr != nil { + return + } + _, _ = io.Copy(io.Discard, req.Body) + req.Body.Close() + // Acknowledged, and then cut: one delivery, and money. + fmt.Fprint(c, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"id\":\"x\",\"choices\":[") + if tc, ok := c.(*net.TCPConn); ok { + _ = tc.SetLinger(0) + } + }(c) + } + }() + + huge := strings.Repeat("длинный исходный текст главы. ", 200000) + c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: "http://" + ln.Addr().String(), Profile: cutProfile(3 * time.Second)}, nil) + _, err = c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: huge}}, MaxTokens: 16}) + + // THREE attempts, TWO deliveries: the first never leaves, the second is delivered and cut, and a + // delivered cut is worth one retry — which delivers again and hits the cap. The count is the premise + // as well as the subject: if the retry policy moves, this says so instead of the assertion below + // quietly measuring a different chain. + if n := conns.Load(); n != 3 { + t.Fatalf("the fixture needs three attempts — one that never delivers, then a cut and its one "+ + "retry — got %d", n) + } + var cut *AttemptCutError + if !errors.As(err, &cut) { + t.Fatalf("premise broken: the chain must end on the delivered attempt's cut: %T %v", err, err) + } + if cut.Deliveries != 2 { + t.Fatalf("the provider was asked TWICE across three attempts: the first request never left, so "+ + "nobody was asked and nobody generated. Counting it says the engine bought three generations "+ + "where it bought two, in the one line where that gap is published. got %d", cut.Deliveries) + } +} diff --git a/backend/internal/llm/httpllm.go b/backend/internal/llm/httpllm.go index 650aadf4..8c7e4172 100644 --- a/backend/internal/llm/httpllm.go +++ b/backend/internal/llm/httpllm.go @@ -10,8 +10,10 @@ import ( "log/slog" "math/rand" "net/http" + "net/http/httptrace" "strconv" "strings" + "sync" "time" "golang.org/x/net/http2" @@ -43,10 +45,23 @@ import ( // defaults; the profile comes from models.yaml per provider (per-role // overrides — Phase 1). type RetryProfile struct { - AttemptTimeout time.Duration // deadline for ONE HTTP attempt + AttemptTimeout time.Duration // FLOOR for one HTTP attempt's deadline (see deadlineFor) MaxAttempts int BackoffBase time.Duration // first backoff; doubles per attempt BackoffCap time.Duration + // The three fields below turn the per-attempt deadline from a constant into a function of the + // budget the call carries (attemptcut.go). All three are optional and all three are DATA: a + // provider the repository has never seen gets a working deadline from the vendor default and a + // startup warning naming what to measure, with no Go edit. + // + // TokensPerSecFloor is the slowest generation speed this provider has been OBSERVED to hold — + // below the p10 of its own request_log, rounded down. Unset ⇒ the vendor default. + TokensPerSecFloor float64 + // QueueSlack is how long the VENDOR documents a request may wait before generation starts. It is + // added whole rather than amortized: the wait is not proportional to the budget. + QueueSlack time.Duration + // AttemptMax bounds the derived deadline — the longest one call may hold a reservation. 0 = unbounded. + AttemptMax time.Duration } func (p RetryProfile) withDefaults() RetryProfile { @@ -90,14 +105,50 @@ const ( // reliability bonus, never a hard dependency. The local provider passes its OWN // no-proxy client (httpc != nil), so it is untouched — localhost needs no h2 keepalive. func keepAliveHTTPClient() *http.Client { - base := http.DefaultTransport.(*http.Transport).Clone() - if h2, err := http2.ConfigureTransports(base); err == nil && h2 != nil { - h2.ReadIdleTimeout = h2ReadIdleTimeout - h2.PingTimeout = h2PingTimeout - } - return &http.Client{Transport: base} + c, _ := buildCloudClient() + return c } +// buildCloudClient is the construction itself, handing back BOTH the client and the h2 transport it +// installed the bounds on. The second return is what makes the bounds checkable at all: reading them off +// a finished client is impossible (ConfigureTransports answers an error the second time), so a check +// written against a client would have to re-derive them on a transport of its own and would then be +// asserting about a transport nobody uses. +func buildCloudClient() (*http.Client, *http2.Transport) { + base := http.DefaultTransport.(*http.Transport).Clone() + h2 := tuneHTTP2(base) + return &http.Client{Transport: base, CheckRedirect: doNotFollowRedirects}, h2 +} + +// tuneHTTP2 installs the keepalive pair and RETURNS the h2 transport it configured, so a test can read +// what was actually set rather than re-deriving it. Asking ConfigureTransports a second time answers an +// error and no transport, so a check written against an already-built client asserts nothing at all. +// nil on the (unexpected) configure error: keepalive is a reliability bonus, never a hard dependency. +func tuneHTTP2(base *http.Transport) *http2.Transport { + h2, err := http2.ConfigureTransports(base) + if err != nil || h2 == nil { + return nil + } + h2.ReadIdleTimeout = h2ReadIdleTimeout + h2.PingTimeout = h2PingTimeout + return h2 +} + +// doNotFollowRedirects stops the client at a 3xx instead of chasing it, so the redirect surfaces as the +// terminal non-2xx it is and the operator is told the base_url is wrong. +// +// ⛔ IT IS A MONEY GUARD BEFORE IT IS A HYGIENE ONE. `Do` spans the WHOLE redirect chain, and the +// delivery trace does not reset between its legs: the first leg reaching a redirector sets WroteRequest +// (and GotFirstResponseByte) for good, so a second leg whose connect is REFUSED still looked delivered. +// Measured through the ledger: $0.001056 booked for a `connect: connection refused` that never put a byte +// on any wire, with AfterHeaders true and zero bytes from the target. A stale `http://` or a normalised +// slash in base_url is enough to trigger it. +// +// The second reason is the one a security review would raise first: net/http drops Authorization only +// across a host change it considers unsafe, and a provider endpoint that redirects is a misconfiguration +// in every case — there is no shape in which silently following one is what an operator wanted. +func doNotFollowRedirects(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + // maxResponseBytes caps one completion body read. A translated chapter chunk // is ~10–50 KiB; 16 MiB leaves two orders of magnitude of headroom while // keeping a misbehaving endpoint from exhausting memory. @@ -126,6 +177,13 @@ const maxRetryAfterWait = 5 * time.Minute func retryLoop[T any](ctx context.Context, profile RetryProfile, name string, log *slog.Logger, attempt func() (T, bool, error)) (T, error) { var zero T var lastErr error + // ⛔ A DELIVERED CUT MUST OUTLIVE THE ATTEMPT THAT MADE IT. Only the LAST error left this loop, so a + // chain that cut a delivered request and then met a terminal 4xx — a 400/401/403/413 on the retry, + // entirely reachable — returned the status alone: the runner saw «the request never went out», gave + // the reservation back and booked $0 for a generation the provider had made, leaving the position + // with no mark either. The first cut is kept and joined onto whatever ends the chain, so the money + // and the reason a caller finally reports are both true of one error. + var owedCut error for att := 0; att < profile.MaxAttempts; att++ { if log != nil { log.DebugContext(ctx, name+" attempt start", "attempt", att+1, "max", profile.MaxAttempts, @@ -136,11 +194,12 @@ func retryLoop[T any](ctx context.Context, profile RetryProfile, name string, lo return resp, nil } lastErr = err + owedCut = moreOwed(owedCut, err) if ctx.Err() != nil { - return zero, ctx.Err() + return zero, chainError(cancelledDuring(ctx.Err(), err), owedCut) } if !retryable { - return zero, err + return zero, chainError(err, owedCut) } if att+1 >= profile.MaxAttempts { break // attempts exhausted — no retry remains, no sleep @@ -152,11 +211,72 @@ func retryLoop[T any](ctx context.Context, profile RetryProfile, name string, lo } select { case <-ctx.Done(): - return zero, ctx.Err() + // ⛔ THE SECOND CANCELLATION EXIT, and it used to throw the evidence away where the first one + // no longer does. A run stopped DURING a backoff has an attempt behind it that may already + // have been delivered and billed — a cut connection, an undecodable 2xx — and returning a + // bare ctx.Err() here left the runner with nothing to settle and the chunk with no mark at + // all. Measured: a delivered connection_lost plus a stop inside the backoff booked $0 and + // wrote no chunk_status row. The window is the whole sleep, up to a minute on the shipping + // config, and it opens precisely on the runs where a provider is flapping and an operator is + // therefore reaching for the stop. + // + // ⚠ THIS CLOSES THE MONEY HALF ONLY. What leaves here carries the cut with the strongest + // money claim, which is deliberately NOT the stop when an earlier paid break outranks it — so + // the runner's mark cannot be decided by this error's first cause. The mark asks its own + // question (pipeline's recordCancelledStage: was the run stopped, and did ANY cut deliver), + // and a guard that read the first cause instead left the position with no row at all. + return zero, chainError(cancelledDuring(ctx.Err(), lastErr), owedCut) case <-time.After(backoff): } } - return zero, fmt.Errorf("%s: exhausted %d attempts: %w", name, profile.MaxAttempts, lastErr) + return zero, chainError(fmt.Errorf("%s: exhausted %d attempts: %w", name, profile.MaxAttempts, lastErr), owedCut) +} + +// ⛔ THE ERROR IS A SCALAR AND THE CHAIN IS A LIST, and every defect this pair exists to stop came from +// that mismatch. A retry chain can deliver, and be billed for, an attempt that is NOT the attempt whose +// error ends it: a cut, then a 503, then a stop. Whatever one error the loop finally returns is the ONLY +// thing the caller settles from — so it must carry the cut the chain owes money for, from every exit, +// not from the two that happened to be written with it in mind. +// +// moreOwed is the accumulator, and it keeps the FIRST cut deliberately rather than ranking them here. +// +// ⚠ RANKING BELONGS AT THE EXIT, and putting it here as well would be a guard that cannot fire. A chain +// holds at most TWO cuts: only CutByConnection is retryable (AttemptCutError.retryable), and the +// delivered-cut cap makes the second one terminal (`retryable && deliveredCutSeen > 1` → not retryable). +// So whenever two cuts exist, the second one IS the error ending the chain and is in `chainError`'s hand +// already; whenever only one exists, there is nothing to rank. A billable-beats-free branch here would +// read like a money guard and never execute — the shape this pack has spent a shift removing. +func moreOwed(kept, candidate error) error { + var cc *AttemptCutError + if kept != nil || !errors.As(candidate, &cc) { + return kept // already holding one, or not a cut at all — a 503 owes nobody anything + } + return candidate +} + +// chainError attaches what the chain owes to whatever error ends it. It returns `final` untouched when +// nothing is owed, when the owed cut IS what ended it (joining an error to itself prints the sentence +// twice), or when `final` already carries a cut with a money claim at least as strong. +// +// ⚠ THE TEST IS THE MONEY, NOT THE TYPE. The version this replaces asked `errors.As(final, &cut)` and +// returned early on any cut at all — true while money was drawn on `Delivered`, false the moment it was +// drawn on `Billable`: a later FREE cut then erased an earlier PAID one and the engine booked $0. +func chainError(final, owed error) error { + if owed == nil || errors.Is(final, owed) { + return final + } + var fc *AttemptCutError + if errors.As(final, &fc) { + var oc *AttemptCutError + if errors.As(owed, &oc) && oc.Billable && !fc.Billable { + // ⛔ THE OWED CUT GOES FIRST, and the order is the whole assertion. `errors.As` hands back the + // FIRST match it meets walking the tree, so joining the free cut ahead of the paid one leaves + // the caller settling from the free one — the very masking this branch exists to undo. + return errors.Join(owed, final) + } + return final + } + return errors.Join(final, owed) } // nextBackoff computes the wait before attempt att+1 (0-based att just failed): @@ -212,6 +332,10 @@ type openAIClient struct { profile RetryProfile headers map[string]string // extra static headers (provider-specific), may be nil log *slog.Logger + // floorWarned fires the «this provider has no measured speed» notice ONCE per client rather than + // once per call: the fact is about the configuration, and a wave of forty chunks would otherwise + // print it forty times and teach the operator to scroll past it. + floorWarned sync.Once } // newOpenAIClient builds the shared transport. httpc may be nil (default @@ -350,8 +474,49 @@ func (c *openAIClient) complete(ctx context.Context, reqBody openAIRequest) (*op return nil, err } billedDecodeSeen := 0 + deliveredCutSeen := 0 + // ⛔ TWO DIFFERENT COUNTS, and they were one field. `deliveredCutSeen` is a RETRY CAP keyed on cuts. + // `askedToGenerate` is what the operator is told — how many times this request reached the provider + // inside one chain — and a chain delivers in more ways than by being cut: an undecodable 2xx that + // already billed, a terminal 4xx, a 503. Counting only the cuts made the ledger line understate a + // mixed chain while calling itself «the provider was asked N times». + askedToGenerate := 0 + // ⛔ EVERY CUT THE CHAIN MADE IS RE-STAMPED, not only the newest. The count is a property of the + // CHAIN, and chainError deliberately hands up an EARLIER cut when that is the one that owes money — + // carrying a number frozen at the moment it was born. A chain that cut once and was then answered + // two 503s delivered three times and reported one, under-stating the very gap the line exists to + // show. The later deliveries are not cuts, so nothing in the error tree knows about them; only this + // counter does. + var chainCuts []*AttemptCutError return retryLoop(ctx, c.profile, c.name, c.log, func() (*openAIResponse, bool, error) { - resp, retryable, err := c.attempt(ctx, payload) + resp, retryable, err := c.attempt(ctx, payload, reqBody.maxTokens) + if deliveredAttempt(resp, err) { + askedToGenerate++ + } + defer func() { + for _, cc := range chainCuts { + cc.Deliveries = askedToGenerate + } + }() + // Cap a DELIVERED cut's re-calls at ONE, for the same reason and by the same shape as the + // billed-decode cap above — but keyed on DELIVERY rather than on a 2xx. That is the whole + // correction: on a provider that answers 200 while the request is still queued (DeepSeek + // documents exactly that), «did a 2xx arrive» says nothing about whether a generation was + // bought, while «did the request go out» says it exactly. A broken connection after delivery + // is worth one more call; a second is a dead provider, not a flaky socket. + if err != nil { + var cut *AttemptCutError + if errors.As(err, &cut) { + deliveredCutSeen++ + // The provider has now been asked to generate this many times, and the caller is told: + // one settle will cover all of them, because the store writes spend only through a + // checkpoint and they share one key. + chainCuts = append(chainCuts, cut) + if retryable && deliveredCutSeen > 1 { + return resp, false, err + } + } + } // Cap billed-decode re-bills at ONE (research/21 §1.18в): an undecodable 2xx // has ALREADY billed, so re-running it under the full MaxAttempts turns a // provider emitting 2xx garbage into a paid retry STORM (grok-build's @@ -371,15 +536,44 @@ func (c *openAIClient) complete(ctx context.Context, reqBody openAIRequest) (*op }) } +// deliveredAttempt reports whether ONE attempt's request reached the provider — the question the +// operator's «asked N times» line answers, and a different question from «did it end in a cut». +// +// A status of any kind is delivery by definition: the peer had to read the request to answer it. So is a +// 2xx whose body would not parse — that one has already billed. A cut says so itself. What is NOT a +// delivery is everything that failed before the bytes left: a refused connect, a DNS failure, a write +// that died mid-body. +func deliveredAttempt(resp *openAIResponse, err error) bool { + if err == nil { + return resp != nil + } + var cut *AttemptCutError + if errors.As(err, &cut) { + return cut.Delivered + } + var hse *HTTPStatusError + var bde *BilledDecodeError + return errors.As(err, &hse) || errors.As(err, &bde) +} + // attempt performs one HTTP call. Returns retryable=true for 429/5xx and // network errors, false for other non-2xx (terminal 4xx). The per-attempt -// deadline bounds a single hung connection; the overall per-request deadline -// (set by the caller via ctx) bounds the whole retry loop. -func (c *openAIClient) attempt(ctx context.Context, payload []byte) (*openAIResponse, bool, error) { - attemptCtx, cancel := context.WithTimeout(ctx, c.profile.AttemptTimeout) +// deadline is derived from THIS call's output budget (attemptcut.go); the +// overall per-request deadline (set by the caller via ctx) bounds the whole retry loop. +// +// maxTokens is the budget the body already carries. It is passed rather than re-parsed so the +// deadline and the request can never describe different calls. +func (c *openAIClient) attempt(ctx context.Context, payload []byte, maxTokens int) (*openAIResponse, bool, error) { + deadline := c.attemptDeadline(ctx, maxTokens) + attemptCtx, cancel := context.WithTimeout(ctx, deadline) defer cancel() - req, err := http.NewRequestWithContext(attemptCtx, http.MethodPost, c.base+"/chat/completions", bytes.NewReader(payload)) + // The delivery facts are collected by the transport itself: a request is DELIVERED once its bytes + // are written, which is knowable before any reply exists and is the boundary the money is drawn on. + var tr deliveryTrace + started := time.Now() + req, err := http.NewRequestWithContext(httptrace.WithClientTrace(attemptCtx, tr.clientTrace()), + http.MethodPost, c.base+"/chat/completions", bytes.NewReader(payload)) if err != nil { return nil, false, err } @@ -395,18 +589,26 @@ func (c *openAIClient) attempt(ctx context.Context, payload []byte) (*openAIResp resp, err := c.http.Do(req) if err != nil { - // Network error / timeout — retryable (unless the parent ctx is done). A - // per-attempt deadline is annotated with the configured timeout: the bare - // «context deadline exceeded» doesn't tell the operator WHOSE deadline it is — - // the attempt timeout (cured by timeouts.attempt_s) or the whole run cancelled. + // A DELIVERED request that never answered: the provider has it and is (or was) working, so + // this is a money event and carries its cause. An UNDELIVERED one stays exactly what it was — + // a plain retryable transport failure whose reservation is released, the one case where + // «nothing was bought» is true by construction. + if cut := c.cutError(ctx, attemptCtx, &tr, started, nil, err, false); cut != nil { + return nil, cut.retryable(), cut + } + // The bare «context deadline exceeded» doesn't tell the operator WHOSE deadline it is — + // this call's own (derived from its budget) or the whole run cancelled. if errors.Is(attemptCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil { - err = fmt.Errorf("attempt timed out after %s (timeouts.attempt_s): %w", c.profile.AttemptTimeout, err) + err = fmt.Errorf("attempt timed out after %s (derived from max_tokens=%d; timeouts.attempt_s is its floor) before the request was delivered: %w", deadline, maxTokens, err) } return nil, ctx.Err() == nil, err } defer resp.Body.Close() - // Read one byte past the limit to DISTINGUISH truncation from a whole body. - data, _ := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + // Read one byte past the limit to DISTINGUISH truncation from a whole body — for the >16 MiB case + // alone. Whether the read COMPLETED is a separate question with a separate answer, and dropping + // readErr here is what made a body our own deadline cut short (small) indistinguishable from a + // whole one, so it went down the retryable branch and bought the same generation twice. + data, readErr := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) truncated := len(data) > maxResponseBytes if truncated { data = data[:maxResponseBytes] @@ -414,15 +616,35 @@ func (c *openAIClient) attempt(ctx context.Context, payload []byte) (*openAIResp obs.LogLLMExchange(ctx, c.log, c.name, payload, resp.StatusCode, data) + // ⛔ THE STATUS LINE IS ASKED BEFORE THE READ ERROR, and the order is the money. A non-2xx says the + // provider refused or failed — nothing was generated and nothing is owed — so a body cut short + // under it is a detail of a failure, not a purchase. Reading them the other way round would settle + // an estimate for every 4xx whose tiny body happened to land on the deadline. The partial body is + // still what the retry/terminal split reads (a truncated marker just fails to match and the status + // stays retryable, the conservative direction), and the read error rides in the message so a + // misclassified 429 leaves a trace instead of none. if resp.StatusCode < 200 || resp.StatusCode >= 300 { retryable := retryableStatus(resp.StatusCode, data) e := &HTTPStatusError{Provider: c.name, Status: resp.StatusCode, Body: snippet(data)} + if readErr != nil { + e.Body = snippet(data) + fmt.Sprintf(" [body read incomplete: %v]", readErr) + } if retryable { e.RetryAfter = parseRetryAfter(resp.Header) // only a retryable status will honour it } return nil, retryable, e } + if readErr != nil { + // A 2xx whose body we did not receive whole. Headers had arrived, so the request was written + // by definition — but ask the trace rather than assume it, and let an undelivered + // impossibility fall through to the old shape instead of settling money on a deduction. + if cut := c.cutError(ctx, attemptCtx, &tr, started, data, readErr, true); cut != nil { + return nil, cut.retryable(), cut + } + return nil, ctx.Err() == nil, readErr + } + var out openAIResponse if err := json.Unmarshal(data, &out); err != nil { // A 2xx with an unreadable body: the provider has ALREADY charged. We type it diff --git a/backend/internal/llm/provider_anthropic.go b/backend/internal/llm/provider_anthropic.go index 60472af2..5c6dd993 100644 --- a/backend/internal/llm/provider_anthropic.go +++ b/backend/internal/llm/provider_anthropic.go @@ -73,9 +73,16 @@ func NewAnthropicClient(cfg AnthropicConfig, logger *slog.Logger) LLMClient { base = "https://api.anthropic.com" } return &anthropicClient{ - base: base, - key: cfg.APIKey, - http: &http.Client{}, + base: base, + key: cfg.APIKey, + // ⚠ THE ONE THING THIS DEPRECATED ADAPTER GETS FROM THE CUT-CALL PACK, and it is here because it + // is a CREDENTIAL leak rather than adapter work: this client sends `x-api-key`, and net/http + // strips only Authorization/Cookie/WWW-Authenticate across a host change — so a 3xx from a + // mistyped base_url carried the key to a host nobody chose. Measured: the redirect target + // received the key and the call returned nil error. The rest of the pack's money boundary is + // deliberately NOT here (this adapter has no delivery trace and a constant deadline); a key + // walking off is not a scope question. + http: &http.Client{CheckRedirect: doNotFollowRedirects}, profile: cfg.Profile.withDefaults(), cacheTTL: cfg.CacheTTL, log: logger, diff --git a/backend/internal/llm/provider_local.go b/backend/internal/llm/provider_local.go index b90025d1..4d0de6d2 100644 --- a/backend/internal/llm/provider_local.go +++ b/backend/internal/llm/provider_local.go @@ -45,8 +45,16 @@ type LocalConfig struct { // NoProxyClient is an http.Client that bypasses any environment proxy. // Exported so the failover prober uses the same transport discipline. +// +// ⚠ IT REFUSES REDIRECTS FOR THE SAME REASON THE CLOUD CLIENT DOES, and it needed saying separately: +// the guard first went only on keepAliveHTTPClient, whose comment says the local provider «passes its +// OWN no-proxy client, so it is untouched» — true of keepalive, and read as permission for redirects +// too. `Do` spans a whole redirect chain without resetting the delivery trace, so a leg whose connect +// is refused still looks delivered. On the local stand the money is $0 today only because the local +// model is priced at zero; the BEHAVIOUR is wrong either way — a mistyped base_url would reach an +// operator as a flapping socket instead of as the 3xx it is. func NoProxyClient() *http.Client { - return &http.Client{Transport: &http.Transport{Proxy: nil}} + return &http.Client{Transport: &http.Transport{Proxy: nil}, CheckRedirect: doNotFollowRedirects} } type localClient struct { diff --git a/backend/internal/pipeline/burnedpregates_test.go b/backend/internal/pipeline/burnedpregates_test.go new file mode 100644 index 00000000..20077a06 --- /dev/null +++ b/backend/internal/pipeline/burnedpregates_test.go @@ -0,0 +1,1193 @@ +package pipeline + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + + "textmachine/backend/internal/chunk" + "textmachine/backend/internal/config" + "textmachine/backend/internal/llm" + "textmachine/backend/internal/obs" + "textmachine/backend/internal/store" + "textmachine/backend/internal/terminology" +) + +// burnedpregates_test.go: THE PAID-CALL CONTRACT read before the funnel — who asks «is this call already +// paid for», what a BURNED key does to their answer, and what the answer is allowed to decide. +// +// A burned checkpoint records money and NO result (a delivered call the engine cut short), so runAttempt +// cannot replay it — it walks past and buys the work again. A pre-gate that reads such a row as «already +// paid» therefore does not skip a purchase, it lets one through: the fresh call happens with the gate's +// own budget check jumped over. The bank's probe learned this when cut calls started settling at the +// batch key; these two gates ask the same question about repair and about the escalation hop. +// +// Each pair is deliberate. The first fixture holds the property (a burned key must not buy), the second +// holds its opposite (a real key must still replay for free) — because a predicate that simply answered +// «not paid» to everything would pass the first alone while turning the free-replay contract off. + +// gateServer is this file's own provider stub. It exists rather than reusing cutServer because these +// fixtures need to tell the REPAIR call apart from every other one — the whole assertion is «how many +// repair calls were bought» — and because a handler that must both read the body and then hold the +// connection cannot share a helper that drains the body before it is handed the request. +type gateServer struct { + mu sync.Mutex + // counted is how many calls of the kind THIS fixture is about have landed — repair calls for + // newGateServer, hop calls for newHopServer. One field, because a fixture watches one gate. + counted int + srv *httptest.Server +} + +// newGateServer answers every non-repair call with a draft carrying the defect the repair class detects, +// and hands the repair call to `onRepair` — with its 1-based ordinal, so a fixture can make the first +// call behave differently from the ones that follow it. +func newGateServer(t *testing.T, onRepair func(i int, w http.ResponseWriter, req *http.Request)) *gateServer { + t.Helper() + gs := &gateServer{} + gs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, _ := io.ReadAll(req.Body) + if !isRepairBody(string(body)) { + answerJSON(w, "Он прождал полчаса и вошёл внутрь.") + return + } + gs.mu.Lock() + gs.counted++ + i := gs.counted + gs.mu.Unlock() + onRepair(i, w, req) + })) + t.Cleanup(gs.srv.Close) + return gs +} + +func (g *gateServer) repairCalls() int { g.mu.Lock(); defer g.mu.Unlock(); return g.counted } + +// newHopServer is the escalation twin of newGateServer: every primary-model call is answered with a +// draft, and only the call addressed to the FALLBACK model — the hop — is counted and handed to the +// fixture. Counting by model rather than by ordinal matters: the primary's own attempts share the +// connection and would otherwise be indistinguishable from the hop in the tally. +func newHopServer(t *testing.T, onHop func(i int, w http.ResponseWriter, req *http.Request)) *gateServer { + t.Helper() + gs := &gateServer{} + gs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, _ := io.ReadAll(req.Body) + if !isHopBody(string(body)) { + answerJSON(w, "ЧЕРНОВИК ПЕРЕВОДА.") + return + } + gs.mu.Lock() + gs.counted++ + i := gs.counted + gs.mu.Unlock() + onHop(i, w, req) + })) + t.Cleanup(gs.srv.Close) + return gs +} + +func (g *gateServer) hopCalls() int { return g.repairCalls() } + +// answerJSON writes one ordinary finished reply. +func answerJSON(w http.ResponseWriter, text string) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%q},"finish_reason":"stop"}], + "usage":{"prompt_tokens":1000,"completion_tokens":500}}`, text) +} + +// repairBudgetLine is the pipeline.yaml line the fixtures rewrite to exhaust the repair sub-budget +// between the two runs. It is matched exactly so a change to repairGateYAML breaks this loudly instead +// of silently leaving the budget wide open and the assertions vacuous. +const repairBudgetLine = " budget_usd: 1.0\n" + +// exhaustRepairBudget rewrites the fixture's repair budget to a value no call can fit under, so the ONLY +// thing that can still produce a repair call is a pre-gate answering «already paid». +func exhaustRepairBudget(t *testing.T, bookPath string) { + t.Helper() + p := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml") + raw, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), repairBudgetLine) { + t.Fatalf("premise broken: %s no longer carries %q, so this fixture would run with an unbounded "+ + "repair budget and prove nothing", p, strings.TrimSpace(repairBudgetLine)) + } + writeFile(t, p, strings.Replace(string(raw), repairBudgetLine, " budget_usd: 0.0000001\n", 1)) +} + +// assertRepairBudgetExhausted fails LOUDLY when the fixture's own premise is not in place. Without it a +// broken exhaust-helper leaves the burned fixtures accusing the pre-gate of buying past the sub-budget +// when the budget was simply never exhausted — the right colour under the wrong text, which the next +// shift reads as a money defect and goes to fix in repair.go. +func assertRepairBudgetExhausted(t *testing.T, r *Runner, spent float64) { + t.Helper() + if b := r.Pipeline.Gates.Repair.BudgetUSD; spent <= b { + t.Fatalf("premise broken: the repair sub-budget is NOT exhausted (spent %.6f, budget %.6f), so "+ + "nothing below is measuring what it claims", spent, b) + } +} + +// repairJob sets up the snapshot and job a direct maybeRepair call needs, and returns the edit stage it +// runs over. maybeRepair is called directly rather than through TranslateBook because the property is +// about ONE gate's decision, and a whole-book run would reach it through a resume path that has its own +// short-circuits. +func repairJob(t *testing.T, r *Runner) (config.Stage, chunk.Chunk, string, *store.Job) { + t.Helper() + st := r.Pipeline.Stages[len(r.Pipeline.Stages)-1] + if st.Name != "edit" { + t.Fatalf("premise broken: the repair gate runs on the SHIPPING stage; the fixture's last stage is %q", st.Name) + } + ch := chunk.Chunk{Chapter: 1, ChunkIdx: 0, Text: "他等了半个时辰。"} + const snapID = "snapshot-under-test" + if err := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); err != nil { + t.Fatal(err) + } + job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID) + if err != nil { + t.Fatal(err) + } + // The waves build the provider clients eagerly before any worker runs; a direct call into one stage + // has to do the same or the first request finds no client and never reaches the wire — which would + // leave every «no call was made» assertion below true for the wrong reason. + if err := r.buildClients(); err != nil { + t.Fatal(err) + } + return st, ch, snapID, job +} + +// TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget is the money property of repair.go's pre-gate. +// +// The sequence is the production one, not a hand-written row: the first run's repair call IS delivered +// and IS cut (the operator stops the run while it is in flight), so the engine settles it at the repair +// request's own key — money, no text. The second run then finds that key with the sub-budget already +// exhausted. `repairCheckpointExists` answering «paid» skips the budget comparison entirely and walks +// into runRepairAttempt, which cannot replay a burned row and buys the repair again — a fresh paid call +// the only budget bounding repair spend never saw. +func TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget(t *testing.T) { + // ⛔ THE BURN MUST BE A BILLABLE ONE, and how it is produced is the fixture's only delicate part. Our + // money predicate charges a cut call only once the provider has ACKNOWLEDGED it with a reply, so a + // server that merely holds a silent connection leaves a $0 row — real, but not the row this gate is + // about. The handler therefore answers 200, flushes the first bytes of a body, and kills the socket: + // money owed for a generation, no result to show for it. + // + // The cause is the CONNECTION rather than a stop button on purpose. An externally-timed cancel has to + // be fired after the client has parsed the response headers, and the handler cannot know when that + // happened — flushing only pushes bytes at the socket. Measured on the first draft of this fixture: + // 2 runs in 5 cancelled early enough that the call read as unanswered and settled for $0, which would + // have made the assertions below pass while measuring nothing. A server-driven drop needs no + // synchronisation at all, and `connection_lost` burns the key exactly the same way. + // + // Only the FIRST run's calls are dropped. A later one is answered, so that if the gate does buy a + // repair it cannot afford, the fixture fails on the MONEY assertion — «a call was bought outside the + // sub-budget» — rather than on whatever the transport happened to say about the second drop. + dropUntil := 0 + srv := newGateServer(t, func(i int, w http.ResponseWriter, _ *http.Request) { + if dropUntil == 0 || i <= dropUntil { + dropAfterHeaders(t, w) + return + } + answerJSON(w, "Он прождал час и вошёл внутрь.") + }) + bookPath := setupRepairProject(t, srv.srv.URL) + + // Run 1: the repair call is cut in flight, so its key holds money and no result. + r1 := newRunner(t, bookPath) + st, ch, snapID, job := repairJob(t, r1) + const drafted = "Он прождал полчаса и вошёл внутрь." + _, _, mrErr := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil) + if mrErr == nil { + t.Fatal("premise broken: the repair call was supposed to be cut in flight, and it returned cleanly") + } + burned, err := r1.Store.RepairSpentUSD(r1.Book.BookID) + if err != nil { + t.Fatal(err) + } + if burned <= 0 { + t.Fatalf("premise broken: a cut repair call must be SETTLED at the repair key — that row is the "+ + "whole subject of this fixture. repair spend %.6f", burned) + } + // The count is recorded, not asserted to be one: a lost connection is retried once inside the + // transport, so run 1 legitimately asks twice under ONE request key and settles ONE row. What the + // second run must add is zero, and that is the assertion below. + repairsAfterRun1 := srv.repairCalls() + if repairsAfterRun1 == 0 { + t.Fatal("premise broken: run 1 never reached the provider, so there is no burned key to test") + } + dropUntil = repairsAfterRun1 // from here on the provider answers; only the money may still object + r1.Close() + + // Run 2: the sub-budget can no longer fit a call. Nothing may buy one. + exhaustRepairBudget(t, bookPath) + r2 := newRunner(t, bookPath) + defer r2.Close() + st2, ch2, snapID2, job2 := repairJob(t, r2) + assertRepairBudgetExhausted(t, r2, burned) + if _, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil); err != nil { + t.Fatalf("a repair the budget cannot afford is SKIPPED, not an error: %v", err) + } + if got := srv.repairCalls(); got != repairsAfterRun1 { + t.Fatalf("a burned key is money with NO result: runAttempt cannot replay it and buys the repair "+ + "again, so a pre-gate answering «already paid» lets that purchase jump the sub-budget check. "+ + "want %d repair calls, got %d — one of them was bought outside gates.repair.budget_usd", + repairsAfterRun1, got) + } + spent, err := r2.Store.RepairSpentUSD(r2.Book.BookID) + if err != nil { + t.Fatal(err) + } + if spent != burned { + t.Fatalf("the exhausted sub-budget still moved: %.6f → %.6f", burned, spent) + } +} + +// TestARealRepairKeyStillReplaysFreeOnAnExhaustedBudget is the other half, and it is what stops the fix +// above from being «answer no to everything». An ANSWERED repair key must replay for $0 even when the +// sub-budget is gone: the money was already spent, the bytes already exist, and re-deciding them would +// make a resumed run ship different text than the run that paid for it. +func TestARealRepairKeyStillReplaysFreeOnAnExhaustedBudget(t *testing.T) { + srv := newGateServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { + answerJSON(w, "Он прождал час и вошёл внутрь.") + }) + bookPath := setupRepairProject(t, srv.srv.URL) + + r1 := newRunner(t, bookPath) + st, ch, snapID, job := repairJob(t, r1) + const drafted = "Он прождал полчаса и вошёл внутрь." + fixed, _, err := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(fixed, "час") || strings.Contains(fixed, "полчаса") { + t.Fatalf("premise broken: run 1 must have APPLIED a repair, got %q", fixed) + } + paidSpend, err := r1.Store.RepairSpentUSD(r1.Book.BookID) + if err != nil || paidSpend <= 0 { + t.Fatalf("premise broken: the answered repair must be booked, got %.6f %v", paidSpend, err) + } + repairsAfterRun1 := srv.repairCalls() + r1.Close() + + exhaustRepairBudget(t, bookPath) + r2 := newRunner(t, bookPath) + defer r2.Close() + st2, ch2, snapID2, job2 := repairJob(t, r2) + assertRepairBudgetExhausted(t, r2, paidSpend) + again, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil) + if err != nil { + t.Fatal(err) + } + if again != fixed { + t.Fatalf("an already-paid repair must re-serve the SAME bytes:\n first: %q\nsecond: %q", fixed, again) + } + if got := srv.repairCalls(); got != repairsAfterRun1 { + t.Fatalf("an already-paid repair replays for $0 and asks nobody: want %d repair calls, got %d", + repairsAfterRun1, got) + } + if spent, serr := r2.Store.RepairSpentUSD(r2.Book.BookID); serr != nil || spent != paidSpend { + t.Fatalf("a free replay must not move the sub-budget spend: %.6f → %.6f (%v)", paidSpend, spent, serr) + } +} + +// exhaustEscalationBudget rewrites the fixture's escalation budget below what run 1 already spent, so +// escalationBudgetRemains() answers «no» and the ONLY thing that can still buy a hop is the idempotency +// probe answering «already paid». +func exhaustEscalationBudget(t *testing.T, bookPath string) { + t.Helper() + p := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml") + raw, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + i := strings.Index(string(raw), "escalation: { budget_usd:") + if i < 0 { + t.Fatalf("premise broken: %s carries no escalation budget line, so this fixture would run with the "+ + "gate wide open and prove nothing", p) + } + j := strings.Index(string(raw)[i:], "\n") + writeFile(t, p, string(raw)[:i]+"escalation: { budget_usd: 0.0000001 }"+string(raw)[i+j:]) +} + +// isHopBody reports whether a mock request is the ESCALATION hop: it is the only call addressed to the +// fallback model, which the wire body names. +func isHopBody(body string) bool { return strings.Contains(body, `"model":"fake-fallback"`) } + +// escalationJob prepares a direct maybeEscalate call over the draft stage, which is the one carrying +// `escalate_to` in the fixture. +func escalationJob(t *testing.T, r *Runner) (config.Stage, chunk.Chunk, string, *store.Job) { + t.Helper() + st := r.Pipeline.Stages[0] + if st.ResolvedHop == "" { + t.Fatalf("premise broken: stage %q has no resolved hop, so maybeEscalate returns before it decides "+ + "anything and every assertion below would be vacuous", st.Name) + } + ch := chunk.Chunk{Chapter: 1, ChunkIdx: 0, Text: "静かな図書館の朝。"} + const snapID = "snapshot-under-test" + if err := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); err != nil { + t.Fatal(err) + } + job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID) + if err != nil { + t.Fatal(err) + } + if err := r.buildClients(); err != nil { + t.Fatal(err) + } + return st, ch, snapID, job +} + +// hopMessages is the prompt both runs hand maybeEscalate. It is a constant of the fixture because the +// hop's request hash is derived from it: two runs that differed here would address two different keys +// and the second would find nothing, passing for the wrong reason. +var hopMessages = []llm.Message{{Role: "user", Content: "静かな図書館の朝。"}} + +const hopBaseMaxTokens = 512 + +// TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget is the money property of escalation.go's +// idempotency probe, and it has a second half the repair gate does not: the fresh-hop branch is also +// where escMu is taken. A burned key read as «already paid» skips BOTH — the budget check and the +// serialisation — so the hop it then buys is unbounded AND unserialised against the other draft workers. +func TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget(t *testing.T) { + dropUntil := 0 + srv := newHopServer(t, func(i int, w http.ResponseWriter, _ *http.Request) { + if dropUntil == 0 || i <= dropUntil { + dropAfterHeaders(t, w) // delivered, acknowledged, then the socket dies: money, no result + return + } + answerJSON(w, "Тихое утро в библиотеке.") + }) + bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0) + + r1 := newRunner(t, bookPath) + st, ch, snapID, job := escalationJob(t, r1) + primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}} + if _, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false); err == nil { + t.Fatal("premise broken: the hop was supposed to be cut by the connection, and it returned cleanly") + } + burned, err := r1.Store.EscalationSpentUSD(r1.Book.BookID) + if err != nil { + t.Fatal(err) + } + if burned <= 0 { + t.Fatalf("premise broken: a cut hop must be SETTLED at the hop key — that row is the subject of "+ + "this fixture. escalation spend %.6f", burned) + } + hopsAfterRun1 := srv.hopCalls() + if hopsAfterRun1 == 0 { + t.Fatal("premise broken: run 1 never reached the provider, so there is no burned key to test") + } + dropUntil = hopsAfterRun1 + r1.Close() + + exhaustEscalationBudget(t, bookPath) + r2 := newRunner(t, bookPath) + defer r2.Close() + st2, ch2, snapID2, job2 := escalationJob(t, r2) + if remains, rerr := r2.escalationBudgetRemains(); rerr != nil || remains { + t.Fatalf("premise broken: the escalation budget must be exhausted for run 2, remains=%t err=%v", remains, rerr) + } + if _, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false); err != nil { + t.Fatalf("a hop the budget cannot afford is SKIPPED, not an error: %v", err) + } + if got := srv.hopCalls(); got != hopsAfterRun1 { + t.Fatalf("a burned key is money with NO result: runAttempt cannot replay it and buys the hop again, "+ + "so a probe answering «already paid» sends that purchase past escalation.budget_usd AND past "+ + "escMu. want %d hop calls, got %d", hopsAfterRun1, got) + } + if spent, serr := r2.Store.EscalationSpentUSD(r2.Book.BookID); serr != nil || spent != burned { + t.Fatalf("the exhausted escalation budget still moved: %.6f → %.6f (%v)", burned, spent, serr) + } +} + +// TestARealHopKeyStillReplaysFreeOnAnExhaustedBudget is the other half. escalation.go's doccomment +// promises a paid hop replays «regardless of the budget» — a crash between the hop's settle and the +// chunk_status write must not discard a paid, successful translation and flip the verdict OK→flagged. +// Teaching the probe about burned keys must not cost that promise. +func TestARealHopKeyStillReplaysFreeOnAnExhaustedBudget(t *testing.T) { + srv := newHopServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { + answerJSON(w, "Тихое утро в библиотеке.") + }) + bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0) + + r1 := newRunner(t, bookPath) + st, ch, snapID, job := escalationJob(t, r1) + primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}} + out1, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false) + if err != nil { + t.Fatal(err) + } + if !out1.attempted { + t.Fatal("premise broken: run 1 must have made the hop") + } + paid, err := r1.Store.EscalationSpentUSD(r1.Book.BookID) + if err != nil || paid <= 0 { + t.Fatalf("premise broken: the answered hop must be booked, got %.6f %v", paid, err) + } + hopsAfterRun1 := srv.hopCalls() + r1.Close() + + exhaustEscalationBudget(t, bookPath) + r2 := newRunner(t, bookPath) + defer r2.Close() + st2, ch2, snapID2, job2 := escalationJob(t, r2) + if remains, rerr := r2.escalationBudgetRemains(); rerr != nil || remains { + t.Fatalf("premise broken: the escalation budget must be exhausted for run 2, remains=%t err=%v", remains, rerr) + } + out2, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false) + if err != nil { + t.Fatal(err) + } + if !out2.attempted { + t.Fatal("an already-paid hop must still be REPLAYED on an exhausted budget: discarding it flips a " + + "paid, successful translation to flagged on every resume") + } + if out2.fb.text != out1.fb.text { + t.Fatalf("the replayed hop must re-serve the SAME bytes:\n first: %q\nsecond: %q", out1.fb.text, out2.fb.text) + } + if got := srv.hopCalls(); got != hopsAfterRun1 { + t.Fatalf("an already-paid hop replays for $0 and asks nobody: want %d hop calls, got %d", hopsAfterRun1, got) + } + if spent, serr := r2.Store.EscalationSpentUSD(r2.Book.BookID); serr != nil || spent != paid { + t.Fatalf("a free replay must not move the escalation spend: %.6f → %.6f (%v)", paid, spent, serr) + } +} + +// TestAPaidHopBehindABurnedKeyStillReplaysFree is the case both halves of the pair above miss, and it is +// the one that makes the probe's SHAPE matter rather than its answer. +// +// The funnel does not stop at a burned key — it walks to the next attempt index at the same budget and +// buys there. So after a stopped run the position looks like this: attempt 0 burned, attempt 1 PAID and +// answered. A probe that asks about a FIXED index sees only the burn, answers «not paid», and hands the +// decision to the budget — which, being exhausted, discards a translation that was already bought. The +// probe must ask what the FUNNEL will ask: walk the burns, then look. +// +// Three runs, because the state needs two of them to exist: run 1 burns attempt 0, run 2 buys the answer +// at attempt 1, run 3 arrives with no budget left and must still serve it for $0. +func TestAPaidHopBehindABurnedKeyStillReplaysFree(t *testing.T) { + dropFirst := true + srv := newHopServer(t, func(i int, w http.ResponseWriter, _ *http.Request) { + if dropFirst { + dropAfterHeaders(t, w) // run 1: money, no result — the key at attempt 0 is burned + return + } + answerJSON(w, "Тихое утро в библиотеке.") + }) + bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0) + primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}} + + r1 := newRunner(t, bookPath) + st, ch, snapID, job := escalationJob(t, r1) + if _, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false); err == nil { + t.Fatal("premise broken: run 1's hop was supposed to be cut by the connection") + } + r1.Close() + + // Run 2: the budget is still there, so the funnel walks past the burn and buys the answer one index up. + dropFirst = false + r2 := newRunner(t, bookPath) + st2, ch2, snapID2, job2 := escalationJob(t, r2) + out2, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false) + if err != nil { + t.Fatal(err) + } + if !out2.attempted || out2.fb.text == "" { + t.Fatalf("premise broken: run 2 must have BOUGHT the hop past the burn, got attempted=%t text=%q", out2.attempted, out2.fb.text) + } + paid, err := r2.Store.EscalationSpentUSD(r2.Book.BookID) + if err != nil || paid <= 0 { + t.Fatalf("premise broken: run 2's hop must be booked, got %.6f %v", paid, err) + } + hopsAfterRun2 := srv.hopCalls() + r2.Close() + + // Run 3: no budget left. The hop was PAID and ANSWERED — discarding it now throws away bought work + // and flips the unit's verdict on every later resume. + exhaustEscalationBudget(t, bookPath) + r3 := newRunner(t, bookPath) + defer r3.Close() + st3, ch3, snapID3, job3 := escalationJob(t, r3) + if remains, rerr := r3.escalationBudgetRemains(); rerr != nil || remains { + t.Fatalf("premise broken: the escalation budget must be exhausted for run 3, remains=%t err=%v", remains, rerr) + } + out3, err := r3.maybeEscalate(context.Background(), st3, snapID3, ch3, job3, hopBaseMaxTokens, hopMessages, primary, false) + if err != nil { + t.Fatal(err) + } + if !out3.attempted || out3.fb.text != out2.fb.text { + t.Fatalf("A PAID, SUCCESSFUL HOP WAS DISCARDED because a BURNED key sits in front of it. The probe "+ + "asks a fixed attempt index; the funnel walks burns and answers one index up, so «is this paid» "+ + "must be asked the funnel's way. run 2 bought %q, run 3 served attempted=%t %q", + out2.fb.text, out3.attempted, out3.fb.text) + } + // The control: serving it must have cost nothing — otherwise this test would pass by re-buying. + if got := srv.hopCalls(); got != hopsAfterRun2 { + t.Fatalf("the replay must ask nobody: want %d hop calls, got %d", hopsAfterRun2, got) + } + if spent, serr := r3.Store.EscalationSpentUSD(r3.Book.BookID); serr != nil || spent != paid { + t.Fatalf("a free replay must not move the escalation spend: %.6f → %.6f (%v)", paid, spent, serr) + } +} + +// TestAPaidRepairBehindABurnedKeyStillReplaysFree is the repair twin of the hop case above. Its own +// contract is stated at repair.go's budget block: an already-paid repair replays regardless of the +// remaining budget, because a crash between the paid call and the chunk_status write would otherwise +// make the resumed run ship DIFFERENT bytes than the run that paid for them. +func TestAPaidRepairBehindABurnedKeyStillReplaysFree(t *testing.T) { + dropFirst := true + srv := newGateServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { + if dropFirst { + dropAfterHeaders(t, w) // run 1: the repair key at the candidate's own index is burned + return + } + answerJSON(w, "Он прождал час и вошёл внутрь.") + }) + bookPath := setupRepairProject(t, srv.srv.URL) + const drafted = "Он прождал полчаса и вошёл внутрь." + + r1 := newRunner(t, bookPath) + st, ch, snapID, job := repairJob(t, r1) + if _, _, err := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil); err == nil { + t.Fatal("premise broken: run 1's repair was supposed to be cut by the connection") + } + r1.Close() + + dropFirst = false + r2 := newRunner(t, bookPath) + st2, ch2, snapID2, job2 := repairJob(t, r2) + fixed, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(fixed, "час") || strings.Contains(fixed, "полчаса") { + t.Fatalf("premise broken: run 2 must have BOUGHT the repair past the burn, got %q", fixed) + } + paid, err := r2.Store.RepairSpentUSD(r2.Book.BookID) + if err != nil || paid <= 0 { + t.Fatalf("premise broken: run 2's repair must be booked, got %.6f %v", paid, err) + } + repairsAfterRun2 := srv.repairCalls() + r2.Close() + + exhaustRepairBudget(t, bookPath) + r3 := newRunner(t, bookPath) + defer r3.Close() + st3, ch3, snapID3, job3 := repairJob(t, r3) + assertRepairBudgetExhausted(t, r3, paid) + again, _, err := r3.maybeRepair(context.Background(), st3, snapID3, ch3, job3, drafted, drafted, nil) + if err != nil { + t.Fatal(err) + } + if again != fixed { + t.Fatalf("A PAID REPAIR WAS DISCARDED because a BURNED key sits in front of it, so the resumed run "+ + "ships DIFFERENT bytes than the run that paid:\n paid for: %q\n now ships: %q", fixed, again) + } + if got := srv.repairCalls(); got != repairsAfterRun2 { + t.Fatalf("the replay must ask nobody: want %d repair calls, got %d", repairsAfterRun2, got) + } + if spent, serr := r3.Store.RepairSpentUSD(r3.Book.BookID); serr != nil || spent != paid { + t.Fatalf("a free replay must not move the sub-budget spend: %.6f → %.6f (%v)", paid, spent, serr) + } +} + +// TestTheBankProbeFindsThePaidBatchBehindABurnedKey is the third member of the family, pinned directly +// rather than end-to-end: the bank probe is a pure function of the store, and the two runs the other two +// fixtures need exist here only to create rows. What it holds is the same contract — a burn in front of +// an answer must not hide the answer. +func TestTheBankProbeFindsThePaidBatchBehindABurnedKey(t *testing.T) { + dir := t.TempDir() + srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { answerWhole(w) }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + r, err := NewRunner(bookPath, obs.NewLogger()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + st := r.Pipeline.Stages[0] + ch := chunk.Chunk{Chapter: 0, ChunkIdx: 0} + msgs := []llm.Message{{Role: "user", Content: "термины"}} + const snapID = "snapshot-under-test" + if serr := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); serr != nil { + t.Fatal(serr) + } + job, jerr := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID) + if jerr != nil { + t.Fatal(jerr) + } + _, maxTokens := r.bankCallBudget(st.Model, msgs) + hashAt := func(attempt int) string { + return RequestHash(r.attemptRequest(st, st.Model, snapID, ch, attempt, maxTokens, msgs)) + } + write := func(hash, finish, text string) { + t.Helper() + resv, verdict, rerr := r.Store.Reserve(r.Book.BookID, 0.001, store.Ceilings{BookUSD: 100, DayUSD: 100}) + if rerr != nil || verdict != store.ReserveOK { + t.Fatalf("reserve: %v %v", verdict, rerr) + } + if serr := r.Store.SettleWithCheckpoint(resv, 0.001, store.Checkpoint{ + RequestHash: hash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: 0, + Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: st.Model, + ResponseText: text, UsageJSON: "{}", CostUSD: 0.001, FinishReason: finish, + }, nil); serr != nil { + t.Fatalf("settle: %v", serr) + } + } + + // The control first: with only the burn there, the batch is NOT paid — the same assertion the probe's + // own fixture makes, repeated here so a probe that answered «paid» to everything would fail before + // reaching the case this test is about. + write(hashAt(0), cancelledFinish, "") + if paid, perr := r.bankCheckpointExists(st, snapID, ch, msgs); perr != nil || paid { + t.Fatalf("a lone burned key is not a paid batch: paid=%t err=%v", paid, perr) + } + // And now the answer the funnel bought one index up, exactly where a stopped run leaves it. + write(hashAt(1), "stop", "термин\tterm") + if paid, perr := r.bankCheckpointExists(st, snapID, ch, msgs); perr != nil || !paid { + t.Fatalf("the batch WAS bought and answered at the index the funnel walked to; a probe that only "+ + "looks at the starting key hides it, and the role sub-budget then refuses to serve work it "+ + "already paid for. paid=%t err=%v", paid, perr) + } +} + +// TestAnUnaffordableBatchDoesNotTakeThePaidBatchesBehindIt pins the admission SHAPE of the bank pass. +// +// Admission is not a prefix. An already-paid batch costs nothing and is admitted whatever the budget +// says — the loop's own comment promises exactly that — so a batch the budget refuses can sit in FRONT of +// batches that are free to serve. While the decision was a prefix bound, one refused batch took every +// paid batch behind it: their replay was $0, their result was already bought, and dropping them bought +// nothing while losing a consolidated bank. +func TestAnUnaffordableBatchDoesNotTakeThePaidBatchesBehindIt(t *testing.T) { + dir := t.TempDir() + srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { answerWhole(w) }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + r, err := NewRunner(bookPath, obs.NewLogger()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + const snapID = "snapshot-under-test" + if serr := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); serr != nil { + t.Fatal(serr) + } + st := r.bankStage(roleTerminologist) + job, jerr := r.Store.EnsureJob(r.Book.BookID, 0, st.Name, snapID) + if jerr != nil { + t.Fatal(jerr) + } + // Two batches. The messages function is the fixture's own, so the request identity below is built the + // same way the pass builds it — re-deriving it any other way would address a different key and this + // test would pass by measuring nothing. + batches := [][]terminology.Candidate{{{Key: "甲", Src: "甲"}}, {{Key: "乙", Src: "乙"}}} + msgsOf := func(b []terminology.Candidate) ([]llm.Message, error) { + return []llm.Message{{Role: "user", Content: "batch " + b[0].Key}}, nil + } + plan := bankRolePlan{role: roleTerminologist, budgetUSD: 0.0000001, messages: msgsOf} + + // Batch 1 is ALREADY PAID: a real answered checkpoint at its own key. Batch 0 is not, and the budget + // cannot fit it — so batch 0 is refused and batch 1 must still be served for $0. + msgs1, _ := msgsOf(batches[1]) + _, maxTokens := r.bankCallBudget(st.Model, msgs1) + hash1 := RequestHash(r.attemptRequest(st, st.Model, snapID, chunk.Chunk{Chapter: 0, ChunkIdx: 1}, 0, maxTokens, msgs1)) + resv, verdict, rerr := r.Store.Reserve(r.Book.BookID, 0.001, store.Ceilings{BookUSD: 100, DayUSD: 100}) + if rerr != nil || verdict != store.ReserveOK { + t.Fatalf("reserve: %v %v", verdict, rerr) + } + if serr := r.Store.SettleWithCheckpoint(resv, 0.001, store.Checkpoint{ + RequestHash: hash1, JobID: job.ID, ChunkIdx: 1, Attempt: 0, + Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: st.Model, + ResponseText: "乙\tvtoroy", UsageJSON: "{}", CostUSD: 0.001, FinishReason: "stop", + }, nil); serr != nil { + t.Fatalf("settle: %v", serr) + } + if err := r.buildClients(); err != nil { + t.Fatal(err) + } + callsBefore := srv.calls() + + run, err := r.runBankRoleBatches(context.Background(), snapID, plan, batches, "render") + if err != nil { + t.Fatal(err) + } + // The premise: batch 0 really was refused, or the assertion below would hold for the wrong reason. + if run.ran[0] { + t.Fatalf("premise broken: batch 0 was supposed to be unaffordable under a budget of %g", plan.budgetUSD) + } + if !run.ran[1] || run.texts[1] == "" { + t.Fatalf("an ALREADY-PAID batch sitting behind a refused one was dropped. Its replay costs $0 and "+ + "its result was already bought, so refusing it buys nothing and loses a consolidated bank. "+ + "ran=%v texts=%q dropped=%d", run.ran, run.texts, run.dropped) + } + // And the control: serving it asked the provider nothing. + if got := srv.calls(); got != callsBefore { + t.Fatalf("a paid batch must replay for $0: the provider was asked %d times", got-callsBefore) + } + if run.dropped != 1 { + t.Fatalf("exactly one batch was refused, so the report must say one: dropped=%d", run.dropped) + } +} + +// assertRowsMatchTheLedger holds the invariant this family is about: a position's chunk_status row +// records what that position COST, and the checkpoints of that position are what it cost. The two are +// written by different code on different exits — the funnel, the escalation hop, the repair sub-step, +// the cut settle — and every one of them can walk out early. A row below its own ledger is money the +// book spent and the projection a person decides on does not show. +// +// Bank roles are excluded because they write no chunk_status at all (synthetic stage, chapter 0): they +// are a different accounting axis, and counting them here would compare a row against a sum that +// includes rows it never claimed. +func assertRowsMatchTheLedger(t *testing.T, bookPath string) { + t.Helper() + m := readMoney(t, bookPath) + type key struct { + ch, chunk int + stage string + } + ledger := map[key]float64{} + bankRoles := map[string]bool{roleTerminologist: true, roleClassifier: true} + counted := 0 + for _, u := range m.checkpoints { + if bankRoles[u.Role] { + continue + } + ledger[key{u.Chapter, u.ChunkIdx, u.Stage}] += u.CostUSD + counted++ + } + // The control: the instrument must have been given something to compare. «No mismatch» over an empty + // ledger and «no mismatch» over a real one read identically in a pass. + if counted == 0 || len(m.statuses) == 0 { + t.Fatalf("premise broken: nothing to compare — %d checkpoints outside the bank roles, %d status rows", + counted, len(m.statuses)) + } + const cent = 0.0000005 // half a micro-dollar: the columns are float64 sums of the same values + for _, cs := range m.statuses { + // A row that cost money must say how many attempts bought it. `attempts=0` beside a non-zero + // cost is a row that reports work nobody did — measured at 0 attempts against $0.001056 — and it + // is the number an operator reads to tell «one call was cut» from «the position never started». + if cs.CostUSD > cent && cs.Attempts == 0 { + t.Fatalf("chunk_status ch%d/chunk%d/%s cost %.6f and reports attempts=0: the count is written "+ + "after the error check, so a position that PAID reads as one that never started", + cs.Chapter, cs.ChunkIdx, cs.Stage, cs.CostUSD) + } + want := ledger[key{cs.Chapter, cs.ChunkIdx, cs.Stage}] + if diff := cs.CostUSD - want; diff > cent || diff < -cent { + t.Fatalf("chunk_status ch%d/chunk%d/%s says it cost %.6f while its own checkpoints add up to "+ + "%.6f. The row is what the projection reads; money that reaches the ledger and not the row "+ + "is money the book spent and nobody is shown.", + cs.Chapter, cs.ChunkIdx, cs.Stage, cs.CostUSD, want) + } + } + t.Logf("ledger invariant held over %d rows against %d checkpoints", len(m.statuses), counted) +} + +// TestAStoppedRunLeavesEveryRowMatchingItsOwnLedger is the family's contract, pinned end to end: the +// position's money comes from settles written by different code on different exits, and the row that +// records it is written by a third piece on an error exit. +// +// ⚠ THE HOP'S MONEY IS MADE INEVITABLE RATHER THAN RACED FOR, and the first draft of this fixture got +// that wrong. It stopped the run over a hop held on the wire and relied on the stop landing AFTER the +// client had parsed the hop's headers — only an acknowledged call is billable. Measured on the mutated +// tree, where it must fail every time: **2 reds in 8 runs**. The other six the stop won the race, the cut +// settled for $0, and the row matched the ledger at nothing to nothing — a pin that passed by measuring +// an empty scenario. +// +// So the money is put there BEFORE the race can matter: run 1 burns the hop's key (two drops — one is +// retried inside the transport), and run 2's funnel walks that burn and carries its cost whatever +// happens to the fresh call. Whether the stop beats the headers or not, `esc.fb.cumCost` is non-zero, +// and the only question left is the one being asked — does that money reach the row. +func TestAStoppedRunLeavesEveryRowMatchingItsOwnLedger(t *testing.T) { + dir := t.TempDir() + var hopCalls atomic.Int32 + hopHeld := make(chan struct{}) + var once sync.Once + var srv *cutServer + srv = newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) { + if strings.Contains(lastBody(srv), `"fake-hop"`) { + if hopCalls.Add(1) <= 2 { + dropAfterHeaders(t, w) // run 1: two drops burn the hop's key — money, no result + return + } + once.Do(func() { close(hopHeld) }) // run 2: held, and the run is stopped over it + hold(req) + return + } + // The primary draft: a content filter — deterministic and escalatable, so a hop follows. + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"нет"},"finish_reason":"content_filter"}], + "usage":{"prompt_tokens":100,"completion_tokens":10}}`) + }) + bookPath := setupHopProject(t, dir, srv.srv.URL) + + if err := runOnce(t, context.Background(), bookPath); err == nil { + t.Fatal("premise broken: run 1's hop was supposed to end on a delivered cut and burn its key") + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-hopHeld + cancel() + }() + _ = runOnce(t, ctx, bookPath) + + // The premise, and it is what the first draft lacked: there IS hop money on this position, so the + // comparison below is not nothing against nothing. + m := readMoney(t, bookPath) + hopUSD := 0.0 + for _, u := range m.checkpoints { + if u.Escalation { + hopUSD += u.CostUSD + } + } + if hopUSD <= 0 { + t.Fatalf("premise broken: the hop cost nothing, so this fixture is comparing an empty position. "+ + "checkpoints=%d", len(m.checkpoints)) + } + assertRowsMatchTheLedger(t, bookPath) +} + +// TestABurnedKeysMoneyReachesTheRow is the other half of the same invariant, on the ordinary path. The +// funnel walks past a burned key and buys the work again one index up; the row must carry BOTH — the +// money that bought nothing and the money that bought the text. The walk's total was being overwritten +// by the fresh call's cost rather than added to it, so a position that was cut and then paid for read as +// if only the second call had happened. +func TestABurnedKeysMoneyReachesTheRow(t *testing.T) { + dir := t.TempDir() + var calls atomic.Int32 + srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { + // TWO drops, because one is not a burn: a lost connection is retried inside the transport, so a + // single drop followed by an answer never reaches the store as a cut. The delivered-cut cap makes + // the SECOND one terminal — that is what settles the key as money with no result. + if calls.Add(1) <= 2 { + dropAfterHeaders(t, w) // acknowledged, then cut + return + } + answerWhole(w) + }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + + // Run 1 ends on the cut and leaves the burned key behind; run 2 walks past it and buys one index up. + if err := runOnce(t, context.Background(), bookPath); err == nil { + t.Fatal("premise broken: run 1 was supposed to end on a delivered cut") + } + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("run 2 must complete: the burn is re-asked at the SAME budget one index up: %v", err) + } + m := readMoney(t, bookPath) + burned := 0 + for _, u := range m.checkpoints { + if u.CostUSD > 0 && strings.TrimSpace(u.UsageJSON) == "{}" { + burned++ + } + } + if burned == 0 { + t.Fatalf("premise broken: no burned checkpoint was written, so this fixture is measuring an "+ + "ordinary run. checkpoints=%d", len(m.checkpoints)) + } + assertRowsMatchTheLedger(t, bookPath) +} + +// TestAStoppedRunCarriesTheRepairsMoneyToTheRow is the repair twin of the hop case. The repair sub-step +// runs INSIDE the shipping stage and returns its money together with its error, and the row that records +// the position is written on that error exit — so the same ordering defect lives here, and the lens that +// found it measured a ledger of 0.001303 against rows adding to 0.000240. +// +// The money is made inevitable the same way: run 1 burns the repair key, so run 2's walk carries that +// cost whatever the stop does to the fresh call. +func TestAStoppedRunCarriesTheRepairsMoneyToTheRow(t *testing.T) { + var repairCalls atomic.Int32 + repairHeld := make(chan struct{}) + var once sync.Once + var srv *cutServer + srv = newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) { + if isRepairBody(lastBody(srv)) { + if repairCalls.Add(1) <= 2 { + dropAfterHeaders(t, w) // run 1: two drops burn the repair's key + return + } + once.Do(func() { close(repairHeld) }) + hold(req) + return + } + if isEditBody(lastBody(srv)) { + answerJSON(w, "Он прождал полчаса и вошёл внутрь.") // carries the defect the repair class detects + return + } + answerJSON(w, "ЧЕРНОВИК ПЕРЕВОДА.") + }) + bookPath := setupRepairProject(t, srv.srv.URL) + + if err := runOnce(t, context.Background(), bookPath); err == nil { + t.Fatal("premise broken: run 1's repair was supposed to end on a delivered cut and burn its key") + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-repairHeld + cancel() + }() + _ = runOnce(t, ctx, bookPath) + + m := readMoney(t, bookPath) + repairUSD := 0.0 + for _, u := range m.checkpoints { + if u.Role == roleRepair { + repairUSD += u.CostUSD + } + } + if repairUSD <= 0 { + t.Fatalf("premise broken: the repair cost nothing, so this fixture is comparing an empty position. "+ + "checkpoints=%d", len(m.checkpoints)) + } + assertRowsMatchTheLedger(t, bookPath) +} + +// TestAStoppedPositionIsNotADecidedUnit pins the fourth carrier of the same question. A `cancelled` row +// records a position the engine was stopped over — the work was never done and the resume re-does it and +// pays again — so it decides nothing about the unit. Reading it as decided puts the unit into the +// extrapolation's denominator, and the projection then shows a book further along and cheaper than it +// is; the re-bill consent threshold, min($0.50, 5% × projected), shrinks with it. +// +// Both directions, because a predicate that answered «not decided» to everything would pass the first +// half alone and quietly turn the flagged state off for every real defect. +func TestAStoppedPositionIsNotADecidedUnit(t *testing.T) { + rows := func(reason FlagReason) []store.ChunkStatus { + return []store.ChunkStatus{ + {Chapter: 1, ChunkIdx: 0, Stage: "draft", Disposition: string(DispOK), CostUSD: 0.001}, + {Chapter: 1, ChunkIdx: 0, Stage: "edit", Disposition: string(DispFlagged), + FlagReason: string(reason), CostUSD: 0.002}, + } + } + stopped := resolveChunkState(rows(FlagCancelled), 2) + if stopped.State == ChunkFlagged { + t.Fatalf("a stopped position decides nothing: the work was never done and the resume re-does it "+ + "and pays again. Counting it as a decided unit puts it in the projection's denominator, so one "+ + "stop makes the book look further along and cheaper than it is. got state=%v reason=%q", + stopped.State, stopped.Reason) + } + if stopped.State != ChunkInProgress { + t.Fatalf("a stopped position is still IN PROGRESS — that is what it is. got %v", stopped.State) + } + // The historical money is a fact and stays on the resolution whatever the state says. + if stopped.CostUSD <= 0 { + t.Fatalf("the money already spent on the position must still be reported: %.6f", stopped.CostUSD) + } + // The control: an ordinary flag still decides the unit, or this change has simply switched the + // flagged state off. + flagged := resolveChunkState(rows(FlagLength), 2) + if flagged.State != ChunkFlagged || flagged.Reason != string(FlagLength) { + t.Fatalf("a real content flag must still decide the unit: state=%v reason=%q", flagged.State, flagged.Reason) + } +} + +// TestAStopOnAStagesFirstCallStillReportsAnAttempt is the count's own case. Every other fixture here has +// a call that succeeded before the stop, so the loop had already recorded a number; this one is stopped +// on the stage's FIRST call, which is where `attempts=0` beside real money was measured. +// +// The money is made inevitable the same way as the rest: run 1 burns the key, so run 2's walk carries a +// cost whatever the stop does to the fresh call. +func TestAStopOnAStagesFirstCallStillReportsAnAttempt(t *testing.T) { + dir := t.TempDir() + var calls atomic.Int32 + held := make(chan struct{}) + var once sync.Once + srv := newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) { + if calls.Add(1) <= 2 { + dropAfterHeaders(t, w) // run 1: two drops burn the first key + return + } + once.Do(func() { close(held) }) // run 2: held, and the run is stopped over it + hold(req) + }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + + if err := runOnce(t, context.Background(), bookPath); err == nil { + t.Fatal("premise broken: run 1 was supposed to end on a delivered cut and burn the key") + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-held + cancel() + }() + _ = runOnce(t, ctx, bookPath) + + m := readMoney(t, bookPath) + marked := false + for _, cs := range m.statuses { + if cs.FlagReason == string(FlagCancelled) { + marked = true + if cs.CostUSD <= 0 { + t.Fatalf("premise broken: the stopped position carries no money, so the count below proves "+ + "nothing. row=%+v", cs) + } + if cs.Attempts == 0 { + t.Fatalf("the position PAID %.6f and reports attempts=0 — the stop landed on the stage's "+ + "first call, and the count is written after the error check, so a position that was "+ + "cut reads as one that never started", cs.CostUSD) + } + } + } + if !marked { + t.Fatalf("premise broken: no cancelled row was written, so this fixture measured nothing. rows=%d", + len(m.statuses)) + } + assertRowsMatchTheLedger(t, bookPath) +} + +// TestTheAskedNTimesNoteReachesTheReader pins the operator sentence at its source. Its two properties +// are invisible from the call site and both were wrong: the note sat behind a transport error longer +// than the column's own bound, and it claimed an estimate was booked on chains where nothing was. +func TestTheAskedNTimesNoteReachesTheReader(t *testing.T) { + long := errors.New(strings.Repeat("провайдер разорвал соединение на середине тела ответа. ", 6)) + + paid := cutErrLine(long, 2, 0.001056) + if !strings.HasPrefix(paid, "[the provider was asked 2 times;") { + t.Fatalf("the note must come FIRST — the column keeps 120 bytes and the error in front of it is "+ + "longer than that, so a note at the end reaches nobody. got %q", paid) + } + if !strings.Contains(paid, "ONE estimate is booked") { + t.Fatalf("a chain that WAS billed must say one estimate covers it: %q", paid) + } + free := cutErrLine(long, 3, 0) + if strings.Contains(free, "ONE estimate is booked") { + t.Fatalf("nothing was booked for this chain, so the line must not claim an estimate was: %q", free) + } + if !strings.Contains(free, "NOTHING is booked") { + t.Fatalf("the line must say what actually happened to the money: %q", free) + } + // The control: the error itself is still carried, or the note would have replaced the diagnosis + // instead of leading it. + if !strings.Contains(paid, "разорвал соединение") { + t.Fatalf("the transport error must still be in the line: %q", paid) + } +} + +// TestAStopBehindAPaidBreakStillMarksThePosition is the hole §4.2 forbids «at any moment», and it opens +// exactly where two correct mechanisms meet. The retry chain hands up the cut with the strongest MONEY +// claim, which is deliberately the earlier `connection_lost` when the later one is free — so a run a +// person stopped arrives at the mark carrying a cause that is not «stopped». The guard read that first +// cut's cause, decided this was not a stop, and returned in silence: money settled, no row, and the +// export shows a gap nobody can explain. +// +// The money is not what is lost here and the fixture says so: the ledger stays whole and the resume +// re-does the position correctly. What is lost is the position's visible mark — which is the one thing +// the ratified estimate was conditioned on. +func TestAStopBehindAPaidBreakStillMarksThePosition(t *testing.T) { + dir := t.TempDir() + var calls atomic.Int32 + held := make(chan struct{}) + var once sync.Once + srv := newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) { + if calls.Add(1) == 1 { + // Attempt 1: acknowledged, then the socket dies — retryable, and the money of the chain. + dropAfterHeaders(t, w) + return + } + once.Do(func() { close(held) }) // attempt 2 is in flight when the person presses stop + hold(req) + }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-held + cancel() + }() + _ = runOnce(t, ctx, bookPath) + + m := readMoney(t, bookPath) + // The premise, and it is the whole point: the chain really did carry a PAID break rather than the + // stop. Without it the fixture would pass on the ordinary «stopped over the first call» path, which + // was never broken. + paid := 0.0 + for _, u := range m.checkpoints { + paid += u.CostUSD + } + if paid <= 0 { + t.Fatalf("premise broken: the chain must have settled a PAID break before the stop, else this "+ + "fixture measures the case that already worked. checkpoints=%d", len(m.checkpoints)) + } + marked := false + for _, cs := range m.statuses { + if cs.FlagReason == string(FlagCancelled) { + marked = true + } + } + if !marked { + t.Fatalf("a person stopped this run and its money is settled, but the position carries NO mark: "+ + "the guard read the cause of whichever cut came first — a paid `connection_lost` the chain "+ + "kept for its money — and decided a stop was not a stop. §4.2 forbids a hole with no mark at "+ + "any moment. rows=%d, settled=%.6f", len(m.statuses), paid) + } + assertRowsMatchTheLedger(t, bookPath) +} + +// TestTheCutRowPublishesItsEstimateAndItsGap pins the two carriers of DISCLOSURE on a cut row, and both +// were held by nothing. The ratified estimate is charged on one condition — that the row says it is an +// estimate (D39.230 п.1) — so these are not decoration: they are what the owner's word rests on. +// +// ⚠ WHY THE TABLE OF NINE OUTCOMES DID NOT COVER THEM. Its `wantEstimated` column reads the CHECKPOINT's +// usage through estimatedSpend, not `request_log.estimated`. In every fixture the two are true together, +// so the assertion could not tell them apart — the «two different quantities agreed» shape. This one +// reads the row it is talking about. +func TestTheCutRowPublishesItsEstimateAndItsGap(t *testing.T) { + dir := t.TempDir() + var calls atomic.Int32 + srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { + // Two drops: one is retried inside the transport, the second is terminal — so the chain delivered + // TWICE and one estimate covers both, which is exactly the gap the row must publish. + if calls.Add(1) <= 2 { + dropAfterHeaders(t, w) + return + } + answerWhole(w) + }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + if err := runOnce(t, context.Background(), bookPath); err == nil { + t.Fatal("premise broken: the run was supposed to end on a delivered cut") + } + + r, err := NewReadOnlyRunner(bookPath, obs.NewLogger()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + rows, err := r.Store.RequestLogRows(r.Book.BookID) + if err != nil { + t.Fatal(err) + } + var cutRows int + for _, row := range rows { + if row.CostUSD <= 0 { + continue + } + cutRows++ + if row.Estimated != 1 { + t.Fatalf("a row whose cost is a RESERVATION ESTIMATE must say so: the owner's word charges it "+ + "on that condition, and a silent $%.6f is indistinguishable from a provider-reported one. "+ + "row finish=%q estimated=%d", row.CostUSD, row.FinishReason, row.Estimated) + } + if row.EstTokens <= 0 { + t.Fatalf("an estimated row publishes the token figure the estimate was built from; 0 leaves "+ + "the reader with a number and no way to weigh it. row finish=%q", row.FinishReason) + } + if !strings.Contains(row.Err, "asked 2 times") { + t.Fatalf("the provider was asked twice and ONE estimate is booked for both — the row is the "+ + "only place that gap is published, and it says %q", row.Err) + } + } + // The control: there WAS a paid row to inspect. «No violations» over an empty set and over a real one + // print the same on a green test. + if cutRows == 0 { + t.Fatalf("premise broken: no paid row was written, so nothing above was checked. rows=%d", len(rows)) + } +} diff --git a/backend/internal/pipeline/cutcall.go b/backend/internal/pipeline/cutcall.go new file mode 100644 index 00000000..3671585a --- /dev/null +++ b/backend/internal/pipeline/cutcall.go @@ -0,0 +1,255 @@ +package pipeline + +import ( + "context" + "errors" + "fmt" + "time" + + "textmachine/backend/internal/chunk" + "textmachine/backend/internal/config" + "textmachine/backend/internal/llm" + "textmachine/backend/internal/store" +) + +// cutcall.go: the money and the disposition of a call THE ENGINE ITSELF cut short (backlog row 360; +// the owner's word on the amount is D39.230 п.1). The transport half — how a cut is detected and told +// apart from an undelivered request — lives in internal/llm/attemptcut.go; this file decides what it +// COSTS and what happens to the chunk. +// +// A delivered request is generated whether or not we are still on the line. Three things end our side +// of it, they share the money, and they differ in everything else: +// +// attempt_timeout our deadline fired mid-generation. 1 call, flagged, NOT retried. Resume: 0 calls. +// cancelled a person stopped the run. 1 call, marked `cancelled`. Resume: 1 call, SAME budget. +// connection_lost the socket broke after delivery. 2 calls, then an infra pause. Resume: 1 call, SAME budget. +// +// Until this file existed all three arrived at a branch commented «No 2xx ever arrived: nothing was +// billed», which released the reservation and booked ZERO. On the provider that ships today that +// comment is false by construction: DeepSeek answers 200 with empty lines while a request waits to be +// scheduled, so a status line says nothing about whether a generation was bought. + +// cutCall is the identity of the call being settled — the same tuple runAttempt already holds, passed +// as one value because thirteen positional arguments is how a chapter index and a chunk index end up +// swapped without the compiler noticing. +type cutCall struct { + stage config.Stage + chunk chunk.Chunk + job *store.Job + model string + reqHash string + resv store.Reservation + estimate float64 + attempt int + escalation bool +} + +// cancelledPosition is the chunk×stage a stopped run was working on. It is a struct for the reason +// cutCall is: `snapID` and `contentHash` are adjacent strings, and a swap between them compiles, +// writes a row addressed to nothing, and is found by nobody. +type cancelledPosition struct { + stage config.Stage + chunk chunk.Chunk + snapshotID string + contentHash string + cumCostUSD float64 + attempts int +} + +// settleUSDForCutCall is the ONE place that answers «what does a call we cut short cost». +// +// The answer is the reservation's own estimate (D39.230 п.1). Nobody knows what the provider actually +// billed, and whether it bills a cut call at all is deliberately OPEN — the paid balance probe that +// would answer it was declined — so the estimate is an upper bound under the assumption that the +// vendor charges for what it generated. Booking zero understates the book against its own ceiling and +// hands the platform a margin it cannot measure. +// +// It is a function and not a YAML field because a money policy behind a config knob can be changed by +// an operator who is not deciding a money policy, silently, between two runs of one book — leaving the +// ledger with two answers to one question and nothing on the row saying which. +func settleUSDForCutCall(estimate float64) float64 { return estimate } + +// settleCutCall books the money for a delivered call the engine cut short, records the row, and +// returns the disposition that cause deserves. +// +// The money is written FIRST, before anything decides what happens to the chunk: the provider's side +// of the transaction already happened, and a process that dies between here and the verdict must leave +// the spend recorded. The checkpoint is what makes booking it possible at all — the store cannot write +// spend without one — and for two of the three causes it is also what stops the next run from buying +// the same call again. +func (r *Runner) settleCutCall(ctx context.Context, c cutCall, cut *llm.AttemptCutError, err error, att stageAttempt) (stageAttempt, error) { + // ⛔ THE MONEY IS DRAWN ON THE PROVIDER'S OWN ACKNOWLEDGEMENT, not on our write. A request whose + // bytes entered the peer's TCP window may never reach the application behind it — a load balancer + // accepts, the backend never sees it — and settling for that charges a reader for a call nobody + // ran, which is the one direction the canon forbids (D39.196 п.2а). A 2xx object in our hands is + // the acknowledgement; without it the same checkpoint is written for ZERO. + // + // The row is identical either way: same key, same class, same flag, same resume. Only the number + // and the `estimated` mark move, so an operator sees the class and the reader pays only for what a + // provider confirmed taking. + cost := 0.0 + if cut.Billable { + cost = settleUSDForCutCall(c.estimate) + } + finish := string(cut.Cause) + if serr := r.Store.SettleWithCheckpoint(c.resv, cost, store.Checkpoint{ + RequestHash: c.reqHash, JobID: c.job.ID, ChunkIdx: c.chunk.ChunkIdx, Attempt: c.attempt, + Stage: c.stage.Name, Role: c.stage.Role, ModelRequested: c.model, ModelActual: c.model, + ResponseText: "", UsageJSON: "{}", CostUSD: cost, FinishReason: finish, + Escalation: c.escalation, + }, r.events.spendLine()); serr != nil { + // The rule of every settle on this path: money state falling behind reality is a loud infra + // fault, never something to continue on top of. + return att, fmt.Errorf("pipeline: settle after a %s cut a delivered call: %w", finish, serr) + } + r.events.flush() + att.finish = finish + // `att` arrives carrying the money of any burned keys runAttempt walked over, and that money is + // this chunk's just as much as this call's — dropping it would under-state a position that was cut + // twice. `runCost` takes only what THIS run bought. + att.cumCost, att.runCost = att.cumCost+cost, cost + + rl := r.baseRequestLog(c.stage, c.chunk, c.model, c.reqHash) + rl.CostUSD, rl.LatencyMS, rl.FinishReason = cost, att.latency, finish + rl.Degraded, rl.Err, rl.OK = finish, err.Error(), false + if cut.Deliveries > 1 { + // ⚠ THE LEDGER IS SHORT BY DESIGN HERE, AND THE ROW SAYS SO. The provider was asked to generate + // more than once inside one retry chain, and the store books spend only through a checkpoint — + // they share one key, so one settle covers all of them. Under-counting is the ratified + // direction (D39.196 п.2а: the deploy absorbs it, the user's balance is never overstated), so + // this pack does NOT quietly multiply the charge; it makes the gap readable instead, because a + // silent under-count is what row 360 was opened about. + // ⛔ THE NOTE GOES FIRST, and that is not cosmetics. The column an operator reads is bounded — + // `errTail` keeps 120 bytes — and the transport error in front of it is routinely longer than + // that, so a note appended to the end reached nobody, ever. It also says what was ACTUALLY + // booked: on a cut the provider never acknowledged, the answer is nothing, and a line claiming + // «one estimate is booked» beside a $0 row is a sentence that has to be disbelieved to be used. + rl.Err = cutErrLine(err, cut.Deliveries, cost) + } + // «Never write a silent $0 usage; estimate and FLAG it» is the one discipline every surveyed + // harness converges on (research/21 §5.3: crush `EstimatedUsage`, goose `CostSource`). The flag is + // what the operator's legend, the machine-readable pair beside committed_usd and the platform's + // «≥ X, up to Y» all read. + rl.Estimated, rl.EstTokens = cost > 0, r.estOutTokens(c.chunk.Text) + r.Store.LogRequest(ctx, r.Log, rl) + + r.Log.WarnContext(ctx, "we cut a delivered call; the reservation estimate is charged as an ESTIMATE only when the provider had acknowledged it with a reply", + "stage", c.stage.Name, "chapter", c.chunk.Chapter, "chunk", c.chunk.ChunkIdx, "attempt", c.attempt, + "cause", finish, "after_headers", cut.AfterHeaders, "bytes_read", cut.BytesRead, + "whitespace_only", cut.WhitespaceOnly, "elapsed", cut.Elapsed.Round(time.Millisecond).String(), + "deliveries", cut.Deliveries, "estimate_usd", fmt.Sprintf("%.6f", cost)) + + if cut.Cause == llm.CutBySelfDeadline { + // Our deadline, the run is healthy: a disposition, not an infra failure. Nothing retries it — + // a retry asks the provider to generate, and bill, the very thing it is generating right now. + att.cls = classification{FlagAttemptTimeout, cut.Error()} + r.setJobStatus(ctx, c.job.ID, "done") + return att, nil + } + // A stopped run and a dead socket both end this stage. The money above is recorded either way; the + // error travels so the run pauses instead of walking the rest of the book against a dead provider. + r.setJobStatus(ctx, c.job.ID, "failed") + return att, fmt.Errorf("pipeline: stage %s call (ch%d/chunk%d, model %s): %w", + c.stage.Name, c.chunk.Chapter, c.chunk.ChunkIdx, c.model, err) +} + +// cutErrLine is the sentence an operator reads when one retry chain asked the provider to generate more +// than once. It is a named function rather than a Sprintf at the call site because both of its +// properties are load-bearing and neither is visible from there. +// +// ⛔ THE NOTE COMES FIRST. The column that prints this is bounded (cmd/tmctl's errTail keeps 120 bytes) +// and the transport error is routinely longer, so a note appended at the end reached nobody — while +// being the only place the gap between what the provider generated and what the ledger booked is +// visible at all. +// +// ⛔ AND IT SAYS WHAT WAS ACTUALLY BOOKED. On a cut the provider never acknowledged, the answer is +// nothing; «one estimate is booked» printed beside a $0 row is a sentence a reader has to disbelieve +// before they can use it. +func cutErrLine(err error, deliveries int, cost float64) string { + booked := "ONE estimate is booked for all of them" + if cost <= 0 { + booked = "NOTHING is booked: the provider acknowledged none of them" + } + return fmt.Sprintf("[the provider was asked %d times; %s] %s", deliveries, booked, err.Error()) +} + +// recordCancelledStage marks a position whose call a HUMAN stopped, so the stop leaves no unexplained +// gap. It never changes the outcome it is called on: the run is ending, and this says what it was +// doing when it did. +// +// The mark carries its own reason. The call was healthy — the stop button ended it — so +// `flagged(cancelled)` reads as «stopped; the resume re-does it», while a shared flag would send an +// operator hunting for a defect that is not there and no mark at all would let the chapter export a +// hole nobody knows about. A flag lying about its cause is forbidden in its own right (D39.93 п.2). +// +// A write failure is logged, not returned: the caller is already returning the error that stopped the +// run, and replacing it with a bookkeeping failure would hide why the run stopped. +func (r *Runner) recordCancelledStage(ctx context.Context, p cancelledPosition, err error) { + // ⛔ THE STOP IS ASKED OF THE ERROR AS A WHOLE, and the delivery of ANY cut in it — not of whichever + // cut `errors.As` happens to reach first. A retry chain hands up the cut with the strongest money + // claim (llm's chainError), which is deliberately the EARLIER one when a later free cut would mask + // it — so a run stopped over an attempt behind a paid `connection_lost` arrives here carrying that + // cause, and a guard reading only the first cut's cause decided «this is not a stop» and returned in + // silence. The money was settled, the position got no row, and the export showed an unexplained gap: + // the hole §4.2 forbids «at any moment». + // ⚠ «A cut» IS «a delivered cut», by construction and in one place: cutError returns NIL when the + // request never went out, and the single construction of the type sets Delivered true. So one + // errors.As answers both halves, and a walk over the error tree looking for a delivered one would be + // asking a question that cannot come back different. + var cut *llm.AttemptCutError + if !errors.Is(err, context.Canceled) || !errors.As(err, &cut) { + return + } + if uerr := r.Store.UpsertChunkStatus(store.ChunkStatus{ + BookID: r.Book.BookID, Chapter: p.chunk.Chapter, ChunkIdx: p.chunk.ChunkIdx, Stage: p.stage.Name, + SnapshotID: p.snapshotID, ContentHash: p.contentHash, + Disposition: string(DispFlagged), FlagReason: string(FlagCancelled), + Attempts: p.attempts, FinalHash: "", CostUSD: p.cumCostUSD, + Detail: "the run was stopped while this call was in flight; it was paid for at the reservation estimate and the resume re-does it on the same budget", + }); uerr != nil { + r.Log.ErrorContext(ctx, "could not mark the stopped position; its money is recorded but the chunk will read as never started", + "stage", p.stage.Name, "chapter", p.chunk.Chapter, "chunk", p.chunk.ChunkIdx, "err", uerr) + return + } + r.Log.WarnContext(ctx, "the run was stopped over a call that had already gone out; the position is marked cancelled and the resume re-does it on the same budget", + "stage", p.stage.Name, "chapter", p.chunk.Chapter, "chunk", p.chunk.ChunkIdx, "cost_usd", fmt.Sprintf("%.6f", p.cumCostUSD)) +} + +// paidAfterBurns answers «was this call already paid for AND answered», asked THE WAY THE FUNNEL WILL +// ASK IT. It is the single definition of that question for every pre-gate that decides money before a +// call, and it exists because asking it any other way has now been wrong in both directions. +// +// ⛔ A FIXED ATTEMPT INDEX CANNOT ANSWER IT. runAttempt does not stop at a burned key: it walks to the +// next index at the SAME budget and buys there, so after a stopped run a position reads «attempt 0 +// burned, attempt 1 paid and answered». A probe that looks only at the starting index sees the burn and +// says «not paid» — and the caller, finding no budget left, discards a translation that was already +// bought. A probe that ignores burns says «paid» — and the caller skips its budget check while the +// funnel goes and buys the work again. Both were measured, one after the other, on this very code. +// +// So the probe walks exactly as the funnel walks (stagerun.go, the burn loop) and answers about the key +// the funnel will actually use. Two questions asked one way cannot disagree; that is the whole point of +// this function existing rather than three call sites each getting the walk right. +func (r *Runner) paidAfterBurns(st config.Stage, model, snapID string, ch chunk.Chunk, attempt, maxTokens int, msgs []llm.Message) (bool, error) { + for { + cp, err := r.Store.GetCheckpoint(RequestHash(r.attemptRequest(st, model, snapID, ch, attempt, maxTokens, msgs))) + if err != nil { + return false, err + } + if cp == nil { + return false, nil // the walk ran out of keys: nothing here is paid for + } + if !burnedByCut(cp) { + return true, nil // an answer — this is what the funnel would serve for $0 + } + attempt++ + } +} + +// resolvedForResume says a stored disposition is an ANSWER the resume may serve without calling +// anybody. Everything terminal is; `cancelled` is the one row that is not, because it records a stop +// rather than a verdict and the work behind it was never done. Reading it as terminal degenerates the +// whole construction into its opposite — never re-doing anything that was interrupted — which is the +// failure mode this design is most at risk of, since it would look perfectly green. +func resolvedForResume(cs *store.ChunkStatus) bool { + return FlagReason(cs.FlagReason) != FlagCancelled +} diff --git a/backend/internal/pipeline/cutcall_test.go b/backend/internal/pipeline/cutcall_test.go new file mode 100644 index 00000000..626433c7 --- /dev/null +++ b/backend/internal/pipeline/cutcall_test.go @@ -0,0 +1,1460 @@ +package pipeline + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "textmachine/backend/internal/chunk" + "textmachine/backend/internal/llm" + "textmachine/backend/internal/obs" + "textmachine/backend/internal/store" + + _ "modernc.org/sqlite" +) + +// cutcall_test.go: the nine outcomes of a call, walked on a real provider socket, with the CALL COUNT +// taken from the server and a paired assertion about what the RESUME does. +// +// ⛔ WHY THE COUNT COMES FROM THE SERVER. A client-side counter counts intentions; only the server +// knows how many generations were actually asked for, and «how many times did we pay for this» is the +// whole question. Every count below is `srv.calls()`. +// +// ⛔ AND WHY THREE «BEFORE HEADERS» ROWS COST NOTHING. The money is drawn on the provider's own +// acknowledgement — a 2xx object in our hands — and not on our write: bytes entering the peer's TCP +// window say nothing about the application behind it, and settling for that charges a reader for a call +// nobody ran. Those rows keep their class, their flag and their resume; only the number is zero. On the +// shipping provider nothing is lost by it, because DeepSeek answers 200 on acceptance, so every self-cut +// there is post-header and paid. +// +// ⛔ AND WHY EVERY LOST-BUT-DELIVERED ROW HAS A RESUME ASSERTION. This design can fail in two +// directions and only one of them is loud. Re-buying a call we already paid for is the defect it +// fixes; NEVER re-doing an interrupted call is the same construction failing the other way, and it +// would leave every test here green while quietly turning a stopped run into permanent data loss. So +// each row states both: what the first run cost, and what the second run calls. + +// --- the fixture provider --- + +// cutServer is an OpenAI-compatible endpoint whose behaviour is chosen per REQUEST NUMBER, so one +// fixture can cut the first call and answer the second — which is exactly the shape a resume test needs. +type cutServer struct { + mu sync.Mutex + n int + bodies []string + behave func(i int, w http.ResponseWriter, r *http.Request) // i is 1-based + arrived chan struct{} // closed when the FIRST request lands + once sync.Once + srv *httptest.Server +} + +func newCutServer(t *testing.T, behave func(i int, w http.ResponseWriter, r *http.Request)) *cutServer { + t.Helper() + cs := &cutServer{behave: behave, arrived: make(chan struct{})} + cs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + cs.mu.Lock() + cs.n++ + i := cs.n + cs.bodies = append(cs.bodies, string(body)) + cs.mu.Unlock() + cs.once.Do(func() { close(cs.arrived) }) + cs.behave(i, w, r) + })) + t.Cleanup(cs.srv.Close) + return cs +} + +func (c *cutServer) calls() int { c.mu.Lock(); defer c.mu.Unlock(); return c.n } +func (c *cutServer) allBodies() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.bodies...) +} + +// hold blocks the handler until the client walks away — with a bound of its own, because +// httptest.Close waits for outstanding handlers and a handler waiting for a disconnect nobody has +// noticed yet deadlocks the test. A hang has no colour: neither the battery nor a mutation reports one. +func hold(r *http.Request) { + select { + case <-r.Context().Done(): + case <-time.After(4 * time.Second): + } +} + +func answerWhole(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"Тихое утро."},"finish_reason":"stop"}], + "usage":{"prompt_tokens":1000,"completion_tokens":500}}`) +} + +// dropAfterHeaders sends a 2xx and part of a body, then kills the socket: a connection that broke with +// the provider mid-answer and both deadlines still alive. +func dropAfterHeaders(t *testing.T, w http.ResponseWriter) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"id":"fake","choices":[`) + w.(http.Flusher).Flush() + hijackClose(t, w) +} + +// dropBeforeHeaders kills the socket with no status line at all — the request was delivered, nothing +// came back. +func dropBeforeHeaders(t *testing.T, w http.ResponseWriter) { + t.Helper() + hijackClose(t, w) +} + +func hijackClose(t *testing.T, w http.ResponseWriter) { + t.Helper() + hj, ok := w.(http.Hijacker) + if !ok { + t.Error("the fixture needs a hijackable response writer") + return + } + conn, _, err := hj.Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() +} + +// early200 is the vendor-documented «accepted, still queued» reply: a 200 whose body is empty lines. +func early200(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "\n") + w.(http.Flusher).Flush() + hold(r) +} + +// deadTLS is a socket that completes the TCP connection and then says nothing. A client speaking TLS +// to it blocks in the handshake, so the request is NEVER WRITTEN. +// +// ⛔ THIS IS THE FIXTURE THE WHOLE FILE IS MOST AT RISK FROM. «Delivered» and «not delivered» are one +// bit apart, and the easy way to build the second is an HTTP handler that sleeps — which RECEIVES the +// request first and is therefore the FIRST case wearing the second's name. Every row here would then +// pass while the boundary they all stand on was never tested at all. +func deadTLS(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { + var held []net.Conn + for { + c, err := ln.Accept() + if err != nil { + for _, h := range held { + h.Close() + } + return + } + held = append(held, c) + } + }() + t.Cleanup(func() { ln.Close() }) + return "https://" + ln.Addr().String() +} + +// --- the project --- + +// setupCutProject writes a ONE-STAGE book so a call count is unambiguous: every request the server +// sees is this chunk's draft, and nothing else is buying anything. +// +// tok_s_floor is set absurdly high on purpose. These rows are about the CUT, not about the deadline +// arithmetic — the derivation is pinned in internal/llm — and left at the vendor default every fixture +// would wait its derived deadline instead of the second attempt_s asks for. +func setupCutProject(t *testing.T, dir, providerURL string) string { + t.Helper() + writeFile(t, filepath.Join(dir, "prompts", "translator.md"), + "Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}") + writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(` +prices_checked: %q +default_model: fake-model +providers: + fake: + kind: openai + base_url: %q + timeouts: { attempt_s: 1, max_attempts: 3, backoff_cap_s: 1, tok_s_floor: 1000000 } +models: + fake-model: + provider: fake + price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 } +`, time.Now().UTC().Format("2006-01-02"), providerURL)) + writeFile(t, filepath.Join(dir, "pipeline.yaml"), ` +core: C1 +version: 1 +defaults: { max_output_ratio: 2.0, min_max_tokens: 512 } +retries: { regenerate_before_escalate: 0 } +stages: + - { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" } +`) + writeFile(t, filepath.Join(dir, "source.txt"), "静かな朝。") + writeFile(t, filepath.Join(dir, "book.yaml"), ` +book_id: test-book +title: Тест +source_lang: ja +target_lang: ru +genre: ранобэ +audience: тест +venuti: 0.5 +honorifics: keep +transcription: polivanov +footnotes: minimal +pipeline: pipeline.yaml +models: models.yaml +source_file: source.txt +ceilings: { book_usd: 5.0, day_usd: 10.0 } +`) + return filepath.Join(dir, "book.yaml") +} + +// runOnce opens a runner over the project and translates, returning the error verbatim (a cut run +// FAILS on purpose for two of the causes, and swallowing that would hide the pause). +func runOnce(t *testing.T, ctx context.Context, bookPath string) error { + t.Helper() + r, err := NewRunner(bookPath, obs.NewLogger()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + _, terr := r.TranslateBook(ctx) + return terr +} + +// money reads the ledger for the book — the raw spend row and the checkpoints behind it, never an +// aggregate somebody else derived. +type money struct { + committed, reserved float64 + checkpoints []store.CheckpointUsage + statuses []store.ChunkStatus + estRows int + estUSD float64 +} + +func readMoney(t *testing.T, bookPath string) money { + t.Helper() + r, err := NewReadOnlyRunner(bookPath, obs.NewLogger()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + var m money + if m.committed, m.reserved, err = r.Store.SpentUSD(r.Book.BookID); err != nil { + t.Fatal(err) + } + if m.checkpoints, err = r.Store.CheckpointUsageForBook(r.Book.BookID); err != nil { + t.Fatal(err) + } + if m.statuses, err = r.Store.ChunkStatusesForBook(r.Book.BookID); err != nil { + t.Fatal(err) + } + m.estRows, m.estUSD = estimatedSpend(m.checkpoints, m.committed) + return m +} + +// --- the nine --- + +// cutRow is one outcome: how the provider misbehaves, what the FIRST run must cost and mark, and what +// the RESUME must call. +type cutRow struct { + name string + // behave drives the first run. Request numbering is per-server, so `i` is how many calls this + // fixture has already taken. + behave func(t *testing.T, i int, w http.ResponseWriter, r *http.Request) + // cancelOnArrival stops the run as soon as the provider has the request — the «a person pressed + // stop» cause, which cannot be produced by a handler. + cancelOnArrival bool + // notDelivered points the first run at a socket that never reads the request. + notDelivered bool + + wantFirstCalls int // asked of the SERVER + wantPaid bool // did the first run book money for it + // wantNoStatus: an infra pause resolves nothing and must leave NO chunk_status row. It is its own + // field rather than an empty wantFlag: reasonOK is itself the empty string, so overloading the zero + // value would make «the chunk shipped» and «the chunk was never marked» the same expectation — and + // the ok row would then pass while asserting the opposite of what it means. + wantNoStatus bool + wantFlag FlagReason + wantDisp Disposition + wantEstimated bool // is the money an ESTIMATE — asserted on BOTH sides, so the flag cannot be constant + wantResumeCall int // fresh calls the SECOND run makes + // wantSameBudget: the re-done call must carry the max_tokens of the one that was cut — a doubling + // here would buy twice the call for a health nobody lost. + wantSameBudget bool +} + +func TestTheNineOutcomesOfACall(t *testing.T) { + rows := []cutRow{{ + name: "after 2xx · whole body, JSON ok", + behave: func(_ *testing.T, _ int, w http.ResponseWriter, _ *http.Request) { + answerWhole(w) + }, + wantFirstCalls: 1, wantPaid: true, wantFlag: reasonOK, wantDisp: DispOK, wantEstimated: false, wantResumeCall: 0, + }, { + name: "after 2xx · whole body, JSON broken", + behave: func(_ *testing.T, i int, w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"fake","choices":`) // complete read, undecodable content + }, + // The billed-decode cap: one re-bill and no more. Unchanged by this pack — its read error is + // nil, so it is the SECOND row of the table and not a cut at all. + wantFirstCalls: 2, wantPaid: true, wantFlag: FlagDecodeError, wantDisp: DispFlagged, wantEstimated: true, wantResumeCall: 0, + }, { + name: "after 2xx · self-cut (early 200, then our deadline)", + behave: func(_ *testing.T, _ int, w http.ResponseWriter, r *http.Request) { + early200(w, r) + }, + wantFirstCalls: 1, wantPaid: true, wantFlag: FlagAttemptTimeout, wantDisp: DispFlagged, + wantEstimated: true, wantResumeCall: 0, + }, { + name: "after 2xx · cancelled by the operator", + behave: func(_ *testing.T, _ int, w http.ResponseWriter, r *http.Request) { + early200(w, r) + }, + cancelOnArrival: true, + wantFirstCalls: 1, wantPaid: true, wantFlag: FlagCancelled, wantDisp: DispFlagged, + wantEstimated: true, wantResumeCall: 1, wantSameBudget: true, + }, { + name: "after 2xx · connection lost", + behave: func(t *testing.T, _ int, w http.ResponseWriter, _ *http.Request) { + dropAfterHeaders(t, w) + }, + // One retry, keyed on DELIVERY rather than on a 2xx, then an infra pause: no chunk_status row. + wantFirstCalls: 2, wantPaid: true, wantNoStatus: true, wantEstimated: true, wantResumeCall: 1, wantSameBudget: true, + }, { + name: "before headers · self-cut (nothing ever came back)", + behave: func(_ *testing.T, _ int, w http.ResponseWriter, r *http.Request) { + hold(r) // delivered; not one response byte + }, + wantFirstCalls: 1, wantPaid: false, wantFlag: FlagAttemptTimeout, wantDisp: DispFlagged, + wantEstimated: false, wantResumeCall: 0, + }, { + name: "before headers · cancelled by the operator", + behave: func(_ *testing.T, _ int, w http.ResponseWriter, r *http.Request) { + hold(r) + }, + cancelOnArrival: true, + wantFirstCalls: 1, wantPaid: false, wantFlag: FlagCancelled, wantDisp: DispFlagged, + wantEstimated: false, wantResumeCall: 1, wantSameBudget: true, + }, { + name: "before headers · connection lost", + behave: func(t *testing.T, _ int, w http.ResponseWriter, _ *http.Request) { + dropBeforeHeaders(t, w) + }, + wantFirstCalls: 2, wantPaid: false, wantNoStatus: true, wantEstimated: false, wantResumeCall: 1, wantSameBudget: true, + }, { + name: "NOT delivered · the request never went out", + notDelivered: true, + // Every attempt of the retry chain, because nothing was bought and there is nothing to protect. + wantFirstCalls: 0, wantPaid: false, wantNoStatus: true, wantEstimated: false, wantResumeCall: 1, + }} + + if len(rows) != 9 { + t.Fatalf("the table is the contract: five outcomes after a 2xx and four before headers, got %d rows", len(rows)) + } + + for _, row := range rows { + t.Run(row.name, func(t *testing.T) { + dir := t.TempDir() + // The SECOND run always gets a healthy provider: the question a resume answers is «what does + // it call», and a fixture that kept misbehaving would answer «it fails again» instead. + // ⚠ ATOMIC, and not a plain bool. The handler runs on the server's goroutine and may still be + // parked in `hold` when the test flips this — a data race the detector reports as a FAILED + // TEST with no assertion in it, which is the most confusing kind of red there is. + var resumed atomic.Bool + srv := newCutServer(t, func(i int, w http.ResponseWriter, r *http.Request) { + if resumed.Load() { + answerWhole(w) + return + } + row.behave(t, i, w, r) + }) + base := srv.srv.URL + if row.notDelivered { + base = deadTLS(t) + } + bookPath := setupCutProject(t, dir, base) + + // --- the first run --- + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if row.cancelOnArrival { + go func() { + <-srv.arrived + // The request is with the provider; this is the delivered-and-stopped case. + time.Sleep(50 * time.Millisecond) + cancel() + }() + } + firstErr := runOnce(t, ctx, bookPath) + firstCalls := srv.calls() + if firstCalls != row.wantFirstCalls { + t.Fatalf("the SERVER was asked %d time(s), want %d (first run err: %v)", firstCalls, row.wantFirstCalls, firstErr) + } + + m := readMoney(t, bookPath) + assertCutMoney(t, row, m) + + // --- the resume, on the same store and the same snapshot --- + resumed.Store(true) + if row.notDelivered { + // Only the endpoint moves, and base_url is not a snapshot input (buildSnapshotID folds + // the provider's temperature/max_tokens/model and the resolved capability, never the + // address) — so this is a RESUME and not a re-pin. The call count below is the proof: + // a moved snapshot would re-run everything, not one stage. + setupCutProject(t, dir, srv.srv.URL) + } + before := srv.calls() + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("the resume must complete: %v", err) + } + gotResume := srv.calls() - before + if gotResume != row.wantResumeCall { + t.Fatalf("the resume asked the server %d time(s), want %d — a cut call must be re-done "+ + "exactly once and a resolved one not at all", gotResume, row.wantResumeCall) + } + if row.wantSameBudget { + assertSameBudget(t, srv.allBodies(), before) + } + }) + } +} + +// assertCutMoney is the money half of every row, read from the RAW ledger. +func assertCutMoney(t *testing.T, row cutRow, m money) { + t.Helper() + if row.wantPaid { + if m.committed <= 0 { + t.Fatalf("a DELIVERED request was booked at $0 — that is the whole defect: committed=%.8f", m.committed) + } + } else if m.committed != 0 { + t.Fatalf("nothing was delivered, so nothing may be booked: committed=%.8f", m.committed) + } + // A reservation that was neither settled nor released silently tightens the book's own ceiling. + if m.reserved != 0 { + t.Fatalf("the reservation must be resolved either way, got reserved=%.8f", m.reserved) + } + // ESTIMATED, asserted on BOTH sides. The positive half is the disclosure the owner's word came + // with; the negative half is what keeps it from being a constant — a build that marked every paid + // row estimated would satisfy the positive half of all eight rows and say nothing. + if row.wantEstimated { + if m.estRows == 0 { + t.Fatalf("money booked without a token count must publish as ESTIMATED; estimatedSpend saw %d row(s)", m.estRows) + } + if diff := m.estUSD - m.committed; diff > 1e-12 || diff < -1e-12 { + t.Fatalf("all of this book's spend is estimated, so the published estimate must equal committed: est=%.8f committed=%.8f", + m.estUSD, m.committed) + } + } else if m.estRows != 0 { + t.Fatalf("a call the provider reported usage for is not an estimate; got %d estimated row(s) worth $%.8f", + m.estRows, m.estUSD) + } + // The disposition. + if row.wantNoStatus { + if len(m.statuses) != 0 { + t.Fatalf("an infra pause resolves nothing, so it must leave no chunk_status row; got %d: %+v", len(m.statuses), m.statuses) + } + return + } + if len(m.statuses) != 1 { + t.Fatalf("want exactly one chunk_status row, got %d", len(m.statuses)) + } + cs := m.statuses[0] + if FlagReason(cs.FlagReason) != row.wantFlag { + t.Fatalf("flag_reason = %q, want %q — a mark that names the wrong cause sends a person hunting "+ + "for a defect that is not there", cs.FlagReason, row.wantFlag) + } + if Disposition(cs.Disposition) != row.wantDisp { + t.Fatalf("disposition = %q, want %q", cs.Disposition, row.wantDisp) + } +} + +// assertSameBudget proves the re-done call asked for the SAME output budget as the one that was cut — +// the reason the attempt index and the doubling count had to come apart. `from` is how many requests +// the server had taken before the resume. +func assertSameBudget(t *testing.T, bodies []string, from int) { + t.Helper() + if from == 0 || len(bodies) <= from { + t.Fatalf("need a cut request and a re-done one to compare, have %d bodies with %d before the resume", len(bodies), from) + } + first := maxTokensOfBody(t, bodies[0]) + redone := maxTokensOfBody(t, bodies[from]) + if first != redone { + t.Fatalf("the re-done call must carry the budget the cut one was granted: cut asked for %d, "+ + "the re-do asked for %d — a doubling here buys twice the call for a health nobody lost", + first, redone) + } +} + +func maxTokensOfBody(t *testing.T, body string) int { + t.Helper() + var b struct { + MaxTokens int `json:"max_tokens"` + } + if err := json.Unmarshal([]byte(body), &b); err != nil { + t.Fatalf("unreadable recorded request body: %v", err) + } + if b.MaxTokens == 0 { + t.Fatalf("recorded body carries no max_tokens: %s", body) + } + return b.MaxTokens +} + +// TestSettleUSDForCutCallIsTheReservationEstimate pins the money POLICY on its own, because it is the +// one number in this pack that came from the owner rather than from the code, and it must be readable +// as a decision rather than inferred from a chain of fixtures. +func TestSettleUSDForCutCallIsTheReservationEstimate(t *testing.T) { + for _, est := range []float64{0, 0.000001, 0.13, 12.5} { + if got := settleUSDForCutCall(est); got != est { + t.Fatalf("a call we cut is settled at the reservation estimate (D39.230 п.1): estimate %v → %v", est, got) + } + } +} + +// TestAfterASelfCutTheResumeCallsNobody asserts ONE thing, and the narrowness is the design. +// +// The third leak this pack closes is not the retry — it is that the old zero branch wrote no +// checkpoint at all, so a resume walked back to the provider and BOUGHT THE CALL AGAIN through a +// different door. Removing the retry without the checkpoint would have closed the class halfway. +// +// ⛔ IT CARRIES NO MONEY ASSERTION ON PURPOSE. Break the checkpoint write and a test that also checks +// the ledger goes red about the money first, and the reader concludes the mutation was caught by the +// money pin — a right verdict for the wrong reason, which is a hole rather than a catch. Here the only +// sentence a failure can print is about a fresh call. +func TestAfterASelfCutTheResumeCallsNobody(t *testing.T) { + dir := t.TempDir() + var resumed atomic.Bool + srv := newCutServer(t, func(_ int, w http.ResponseWriter, r *http.Request) { + if resumed.Load() { + answerWhole(w) + return + } + hold(r) // delivered, and we walk away + }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("a self-cut is a disposition, not an infra failure: %v", err) + } + before := srv.calls() + resumed.Store(true) + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("the resume must complete: %v", err) + } + if fresh := srv.calls() - before; fresh != 0 { + t.Fatalf("the resume made %d fresh provider call(s) after a self-cut — the generation we already "+ + "paid for is being bought a second time through the resume door", fresh) + } +} + +// TestANonTwoHundredIsNotAPurchase pins the ORDER inside the transport: the status line is read before +// the body's read error. A 4xx/5xx says the provider refused or failed — nothing was generated and +// nothing is owed — so a body cut short underneath one is a detail of a failure, not a purchase. Read +// the other way round, every terminal 4xx whose tiny body happened to land on the deadline would settle +// an estimate, and the money boundary would leak through the one door that is supposed to be free. +func TestANonTwoHundredIsNotAPurchase(t *testing.T) { + dir := t.TempDir() + srv := newCutServer(t, func(_ int, w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error":{"message":"upstream b`) // body never finishes + w.(http.Flusher).Flush() + hold(r) + }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + + if err := runOnce(t, context.Background(), bookPath); err == nil { + t.Fatal("a provider failing every attempt must surface as an infra failure") + } + m := readMoney(t, bookPath) + if m.committed != 0 { + t.Fatalf("a non-2xx generated nothing and owes nothing; committed=%.8f", m.committed) + } + if m.reserved != 0 { + t.Fatalf("the reservation of a failed call must be given back; reserved=%.8f", m.reserved) + } + if len(m.checkpoints) != 0 { + t.Fatalf("nothing was bought, so nothing may be checkpointed; got %d", len(m.checkpoints)) + } +} + +// countingHandler counts log records whose message carries a substring. A handler rather than a scan +// over a shared buffer: an assert on a substring of one big buffer is the shape that gives both false +// reds and quiet greens (D39.171), and «how many times was this said» is a question a handler answers +// exactly. +type countingHandler struct { + n *atomic.Int32 + needle string + // alt counts a SECOND substring, so one fixture can assert both that a line is said and that it + // does not say something it must not. Nil when unused. + alt *atomic.Int32 + altNeedle string + // attrs collects the ATTRIBUTE KEYS of the matched records. The keys are what an operator reads + // beside the sentence, and a key that names a per-attempt figure as the whole wait is as untrue as + // a sentence would be — `waited=4s of=1s` was printed eighteen times before this was pinned. + attrs *sync.Map +} + +func (h countingHandler) Enabled(context.Context, slog.Level) bool { return true } +func (h countingHandler) Handle(_ context.Context, r slog.Record) error { + if strings.Contains(r.Message, h.needle) { + h.n.Add(1) + if h.alt != nil && strings.Contains(r.Message, h.altNeedle) { + h.alt.Add(1) + } + if h.attrs != nil { + r.Attrs(func(a slog.Attr) bool { + h.attrs.Store(a.Key, struct{}{}) + return true + }) + } + } + return nil +} +func (h countingHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h countingHandler) WithGroup(string) slog.Handler { return h } + +// TestTheWaitSaysSoWhileItLastsAndIsSilentOtherwise: with a deadline derived from the budget a call +// may now legitimately run for a quarter of an hour, and the single "calling model" line then leaves +// an operator watching a silence he cannot tell from a wedged process. +// +// ⛔ BOTH HALVES, IN ONE TEST. A conditional message needs a fixture where it MUST be said and one +// where it must be SILENT; a test that asserts only the silence is vacuous — it passes on a build that +// deleted the line — and one that asserts only the speech passes on a build that says it on every +// healthy call, which is how a warning becomes something an operator scrolls past. +func TestTheWaitSaysSoWhileItLastsAndIsSilentOtherwise(t *testing.T) { + const needle = "still waiting for the provider" + // The second needle is the assertion the first one cannot make: the line must not state a fact the + // runner never asked about. It has no view of the transport's trace, so «a call that has been + // delivered» was an assertion nobody had checked — and on a request that never went out it printed + // eighteen times, telling the operator the opposite of what happened. + const claimsDelivery = "delivered" + var keys sync.Map + run := func(t *testing.T, slow bool) (said, claimed int) { + t.Helper() + dir := t.TempDir() + srv := newCutServer(t, func(_ int, w http.ResponseWriter, r *http.Request) { + if slow { + hold(r) // held until our own deadline cuts it + return + } + answerWhole(w) + }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + var seen, claims atomic.Int32 + r, err := NewRunner(bookPath, slog.New(countingHandler{n: &seen, needle: needle, alt: &claims, altNeedle: claimsDelivery, attrs: &keys})) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if _, err := r.TranslateBook(context.Background()); err != nil { + t.Fatalf("translate: %v", err) + } + return int(seen.Load()), int(claims.Load()) + } + said, claimed := run(t, true) + if said == 0 { + t.Fatalf("a call the engine waited its whole deadline for said nothing while it waited — the "+ + "operator cannot tell that silence from a dead process (heartbeats seen: %d)", said) + } + if claimed != 0 { + t.Fatalf("the waiting line stated %d time(s) that the call had been DELIVERED — the runner never "+ + "asked the transport, and on a request that never went out this tells the operator the "+ + "opposite of what happened", claimed) + } + // The KEYS the operator reads beside the sentence. This timer wraps the whole retry chain while the + // only deadline it can name is one attempt's, so a key called `of` turned a truthful pair of numbers + // into «waited 4s of 1s» — the hung-process signal this line exists to remove, printed by the line + // itself. And the transport logs an `attempt` of its own, so the pipeline's must not share the key. + for _, bad := range []string{"of", "attempt"} { + if _, found := keys.Load(bad); found { + t.Fatalf("the waiting line carries the key %q: a per-attempt figure presented as the whole "+ + "wait, or a number that collides with the transport's own under one name", bad) + } + } + for _, want := range []string{"waited", "attempt_deadline", "stage_attempt"} { + if _, found := keys.Load(want); !found { + t.Fatalf("the waiting line must carry %q — without it the reader cannot tell which of the two "+ + "clocks the number belongs to", want) + } + } + if said, _ = run(t, false); said != 0 { + t.Fatalf("a call that answered at once must print no waiting line; got %d", said) + } +} + +// TestTheHeartbeatIsPacedByTheWaitItReports: the interval is a quarter of the deadline, capped at a +// minute. Derived rather than fixed, so the line exists on every call the engine is willing to wait +// for — including the short ones, which is the only reason the fixture above can observe it at all. +func TestTheHeartbeatIsPacedByTheWaitItReports(t *testing.T) { + if got := heartbeatEvery(time.Hour); got != waitHeartbeat { + t.Fatalf("a long wait reports at the cap, got %s", got) + } + if got := heartbeatEvery(4 * time.Second); got != time.Second { + t.Fatalf("a short wait reports four times over its life, got %s", got) + } + // A zero or absurd deadline must not become a zero ticker — time.NewTicker panics on one, and a + // panic inside a logging goroutine kills the run over a log line. + for _, d := range []time.Duration{0, -time.Second, time.Nanosecond} { + if got := heartbeatEvery(d); got <= 0 { + t.Fatalf("deadline %s produced a non-positive interval %s — time.NewTicker panics on that", d, got) + } + } +} + +// TestACancelledRunNeverReportsAMoneyStop drives the admission path DIRECTLY, with no wave and no +// race, because that is the only way to be sure which line is under test. +// +// ⛔ THE FIRST VERSION OF THIS TEST WAS VACUOUS AND A MUTATION SAID SO. It cancelled the context before +// TranslateBook and asserted the outcome — but the wave refuses a dead context before it dispatches +// anything (measured: 0 requests reached the server), so the reservation loop was never entered, the +// guard was never executed, and deleting the guard left the test GREEN. A test that reaches the +// situation only through a wave cannot say which of the two it is pinning. +// +// WHAT IT PINS. A refusal the ceiling cannot lift skips the wait entirely and walks to the halt. If the +// run is over by then, the truth is the cancellation and not the money: a `ceiling` event outlives any +// exit code, so the platform would record `paused` and tell a person to add money for a run that person +// had stopped themselves. Reachable at all only because a cut call is now PAID for (D39.230 п.1) — a +// stop commits every flying call's estimate, which is what can carry a book past its own ceiling. +func TestACancelledRunNeverReportsAMoneyStop(t *testing.T) { + dir := t.TempDir() + srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { answerWhole(w) }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + + r, err := NewRunner(bookPath, obs.NewLogger()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + // A ceiling no single call fits under: the refusal is one no settle could ever lift, which is the + // branch that skips the wait and goes straight to the halt. + r.CeilingUSD = 1e-9 + + // The eager client set a real run precomputes before its first wave; runAttempt resolves the client + // before any money moves, so without it the call fails on the wrong thing entirely. + if berr := r.buildClients(); berr != nil { + t.Fatal(berr) + } + st := r.Pipeline.Stages[0] + ch := chunk.Chunk{Chapter: 1, ChunkIdx: 0, Text: "静かな朝。"} + const snapID = "snapshot-under-test" + // A job references a snapshot row; the id is opaque to everything this test exercises. + if serr := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); serr != nil { + t.Fatal(serr) + } + job, jerr := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID) + if jerr != nil { + t.Fatal(jerr) + } + msgs, merr := MessagesWithInjection(r.templates[st.Name], RenderVars{Book: r.Book, Text: ch.Text}, "") + if merr != nil { + t.Fatal(merr) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // the state every sibling of a stopped call is in + _, aerr := r.runAttempt(ctx, st, st.Model, snapID, ch, job, 0, 512, msgs, false, true, true) + + if aerr == nil { + t.Fatal("a refused reservation under a dead context must not report success") + } + var halt *CeilingHalt + if errors.As(aerr, &halt) { + t.Fatalf("a run that was CANCELLED reported a MONEY stop: %v", aerr) + } + if !errors.Is(aerr, context.Canceled) { + t.Fatalf("the cancellation must be what leaves the call, got %T %v", aerr, aerr) + } + if srv.calls() != 0 { + t.Fatalf("a refused reservation must reach no provider, got %d call(s)", srv.calls()) + } +} + +// TestABurnedCheckpointIsNeverReadAsAnAnswer is the regression an adversarial pass found, and it is +// the worst thing this pack could have shipped. +// +// A burned checkpoint records MONEY and no result. `runStage` knows that from the `burned` flag — but +// three other callers of runAttempt do not read it, and the zero classification they see IS «ok». The +// escalation hop then adopted a cut call as authoritative: the chunk shipped `ok` with an EMPTY text, +// the primary flag (a content filter!) was erased, and the run after that died on its own chunk_status +// pointing at a textless checkpoint. Money spent, text lost, book unresumable — all three at once, and +// silently. +// +// The fixture: the draft comes back content-filtered (an escalatable flag), the hop is held on the wire +// and the run is stopped, then the book is resumed twice. +func TestABurnedCheckpointIsNeverReadAsAnAnswer(t *testing.T) { + dir := t.TempDir() + var resumed atomic.Bool + hopSeen := make(chan struct{}) + var once sync.Once + var srv *cutServer + srv = newCutServer(t, func(_ int, w http.ResponseWriter, r *http.Request) { + if strings.Contains(lastBody(srv), `"fake-hop"`) && !resumed.Load() { + once.Do(func() { close(hopSeen) }) + hold(r) // the HOP is the call the stop catches + return + } + if resumed.Load() { + answerWhole(w) + return + } + // The primary draft: a content filter — deterministic, escalatable. + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"нет"},"finish_reason":"content_filter"}], + "usage":{"prompt_tokens":100,"completion_tokens":10}}`) + }) + bookPath := setupHopProject(t, dir, srv.srv.URL) + + // ⛔ SYNCHRONISED ON THE HOP'S OWN REQUEST, not on a clock. A sleep here lands on the DRAFT under + // load and leaves the test green about a moment it never reached — clocks standing in for the thing + // being measured is the shape D39.171 names, and this fixture had it. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-hopSeen + cancel() + }() + _ = runOnce(t, ctx, bookPath) + + // The first resume must not ship an empty chunk as ok. + resumed.Store(true) + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("the first resume must complete: %v", err) + } + m := readMoney(t, bookPath) + for _, cs := range m.statuses { + if cs.Disposition == string(DispOK) { + cp, cerr := readFinalText(t, bookPath, cs.FinalHash) + if cerr != nil { + t.Fatalf("an `ok` row must point at a readable checkpoint: %v", cerr) + } + if strings.TrimSpace(cp) == "" { + t.Fatalf("chunk_status ch%d/chunk%d/%s shipped `ok` with an EMPTY text — a call that was "+ + "cut off was adopted as the answer", cs.Chapter, cs.ChunkIdx, cs.Stage) + } + } + } + // And the SECOND resume must not die on what the first one wrote. + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("the second resume died on the state the first one left — the book is unresumable: %v", err) + } +} + +func lastBody(c *cutServer) string { + b := c.allBodies() + if len(b) == 0 { + return "" + } + return b[len(b)-1] +} + +func readFinalText(t *testing.T, bookPath, hash string) (string, error) { + t.Helper() + r, err := NewReadOnlyRunner(bookPath, obs.NewLogger()) + if err != nil { + return "", err + } + defer r.Close() + cp, cerr := r.Store.GetCheckpoint(hash) + if cerr != nil { + return "", cerr + } + if cp == nil { + return "", fmt.Errorf("checkpoint %.12s is missing", hash) + } + return cp.ResponseText, nil +} + +// setupHopProject is setupCutProject with an escalation hop, so a cut can land on the FALLBACK call — +// the caller that reads `cls.ok()` and never hears about `burned`. +func setupHopProject(t *testing.T, dir, providerURL string) string { + t.Helper() + writeFile(t, filepath.Join(dir, "prompts", "translator.md"), + "Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}") + writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(` +prices_checked: %q +default_model: fake-model +providers: + fake: + kind: openai + base_url: %q + timeouts: { attempt_s: 1, max_attempts: 3, backoff_cap_s: 1, tok_s_floor: 1000000 } +models: + fake-model: + provider: fake + price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 } + fake-hop: + provider: fake + price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 } +`, time.Now().UTC().Format("2006-01-02"), providerURL)) + writeFile(t, filepath.Join(dir, "pipeline.yaml"), ` +core: C1 +version: 1 +defaults: { max_output_ratio: 2.0, min_max_tokens: 512 } +retries: { regenerate_before_escalate: 0 } +stages: + - { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off", escalate_to: fake-hop } +escalation: { budget_usd: 5.0 } +`) + writeFile(t, filepath.Join(dir, "source.txt"), "静かな朝。") + writeFile(t, filepath.Join(dir, "book.yaml"), ` +book_id: test-book +title: Тест +source_lang: ja +target_lang: ru +genre: ранобэ +audience: тест +venuti: 0.5 +honorifics: keep +transcription: polivanov +footnotes: minimal +pipeline: pipeline.yaml +models: models.yaml +source_file: source.txt +ceilings: { book_usd: 5.0, day_usd: 10.0 } +`) + return filepath.Join(dir, "book.yaml") +} + +// TestABurnIsMoneyWithoutAnAnswerAndNotJustAWord: the finish_reason string shares a namespace with +// whatever a vendor decides to print — the adapter already normalises invented values — so a provider +// answering 200 with a real translation under one of OUR names must keep its answer. A genuine burn is +// written by this engine and is always textless, so the text is part of the test and costs nothing. +// +// Both sides, because a predicate asserted in one direction says nothing: the textless rows must burn +// and the answered ones must not. +func TestABurnIsMoneyWithoutAnAnswerAndNotJustAWord(t *testing.T) { + burns := []string{cancelledFinish, connectionLostFinish} + for _, finish := range burns { + if !burnedByCut(&store.Checkpoint{FinishReason: finish, ResponseText: ""}) { + t.Fatalf("a textless %q checkpoint is money without a result and must burn", finish) + } + if burnedByCut(&store.Checkpoint{FinishReason: finish, ResponseText: "Тихое утро."}) { + t.Fatalf("a %q checkpoint that CARRIES a translation is an answer, whatever the provider "+ + "called its finish reason — burning it throws the text away and buys it again", finish) + } + } + // And an ordinary reply is never a burn, textless or not: an empty completion is a `empty` flag, + // which is retryable, and reading it as money-without-a-result would change what a retry costs. + for _, finish := range []string{"stop", "length", decodeErrorFinish, attemptTimeoutFinish} { + if burnedByCut(&store.Checkpoint{FinishReason: finish, ResponseText: ""}) { + t.Fatalf("%q is a verdict the resume must serve, not a key to spend: burning it re-buys the call", finish) + } + } +} + +// TestACancelledRowIsNotAFreeResume: the volume planner asks the same question runStage asks — «does +// this row resume without a provider call» — and it used to ask it with its own copy, which did not +// know about `cancelled`. A unit whose last row is one was then counted as costing nothing, so a grant +// of one unit paid for two AND the volume report, which speaks only when something was carried, said +// nothing at all. This pins that both ends now go through the one predicate. +func TestACancelledRowIsNotAFreeResume(t *testing.T) { + var r Runner + draft := map[string]bool{"draft": true} + edit := map[string]bool{} + const snap, hash = "snap", "content" + hashes := map[chunkKey]map[string]string{{1, 0}: {"draft": hash}} + row := func(flag FlagReason, disp Disposition) []store.ChunkStatus { + return []store.ChunkStatus{{ + Chapter: 1, ChunkIdx: 0, Stage: "draft", + SnapshotID: snap, ContentHash: hash, + Disposition: string(disp), FlagReason: string(flag), + }} + } + // The control first: an ordinary resolved row IS free, so a failure below means the predicate + // changed its answer for `cancelled` and not for everything. + if !r.rowsResumeFree(row(reasonOK, DispOK), draft, edit, snap, "", hashes) { + t.Fatal("a settled ok row resumes for nothing — this fixture is not measuring what it thinks") + } + if !r.rowsResumeFree(row(FlagAttemptTimeout, DispFlagged), draft, edit, snap, "", hashes) { + t.Fatal("a terminal flag is never re-attacked, so it resumes for nothing") + } + if r.rowsResumeFree(row(FlagCancelled, DispFlagged), draft, edit, snap, "", hashes) { + t.Fatal("a `cancelled` row records a STOP, not an answer: the resume re-does that call for money, " + + "and counting it free lets a volume grant pay for a unit it never charged for") + } +} + +// TestMoneyWithNoCheckpointLeftIsStillAnEstimate: a redrive DELETES the checkpoints of the stages it +// re-attacks and leaves their spend committed — «after a redrive committed(spend) >= SUM(checkpoints), +// the safe direction» (store/chunkstatus.go). Derived from the surviving rows alone, the estimated +// share then falls to zero and the platform is told that money nobody can account for was MEASURED. +// The gap belongs in the estimate by the same rule everything else here does: a cost with no token +// count to justify it. +func TestMoneyWithNoCheckpointLeftIsStillAnEstimate(t *testing.T) { + // ⚠ THE USAGE IS MARSHALLED FROM THE REAL TYPE, not hand-written. llm.Usage carries no json tags, so + // its wire form is Go field names — and a hand-written snake_case fixture unmarshals to all zeros, + // i.e. it silently becomes the OTHER case and the control arm certifies nothing. Caught by this test + // failing on its own control. + usageJSON, err := json.Marshal(llm.Usage{PromptTokens: 100, CompletionTokens: 50}) + if err != nil { + t.Fatal(err) + } + measured := []store.CheckpointUsage{{CostUSD: 0.02, UsageJSON: string(usageJSON)}} + // The control first: while the evidence is intact, a measured call is not an estimate. + if rows, usd := estimatedSpend(measured, 0.02); rows != 0 || usd != 0 { + t.Fatalf("a call the provider reported usage for is not an estimate, got %d row(s) / $%.8f", rows, usd) + } + // Now the same book after a redrive threw that checkpoint away and kept the money. + rows, usd := estimatedSpend(nil, 0.02) + if rows == 0 { + t.Fatal("committed money with no checkpoint behind it is money nobody can account for; publishing " + + "it as measured tells the platform the opposite of what is true") + } + if diff := usd - 0.02; diff > 1e-12 || diff < -1e-12 { + t.Fatalf("the unaccounted gap is the whole of it, got $%.8f", usd) + } + // And float slack must not invent one out of a sum of prices. + if rows, usd := estimatedSpend(measured, 0.02+1e-15); rows != 0 || usd != 0 { + t.Fatalf("a nanodollar of float noise is not unaccounted money, got %d row(s) / $%.12f", rows, usd) + } +} + +// TestADroppedStageIsFreeEvenWhenItsRowSaysCancelled is the other direction of the volume predicate, and +// the direction the first version of that fix got wrong. +// +// The per-row walk drops the stages this pipeline no longer runs — they cannot cost anything, because +// nothing will call them. Asking «does this row resume for free» BEFORE that drop made a unit paid for a +// `cancelled` row of a stage that will never run again, so an operator who edits the pipeline over a +// stopped run would spend volume slots on nothing. Asked after, both readings are right. +func TestADroppedStageIsFreeEvenWhenItsRowSaysCancelled(t *testing.T) { + var r Runner + draft := map[string]bool{"draft": true} + edit := map[string]bool{} + const snap, hash = "snap", "content" + hashes := map[chunkKey]map[string]string{{1, 0}: {"draft": hash}} + row := func(stage string, flag FlagReason) []store.ChunkStatus { + return []store.ChunkStatus{{ + Chapter: 1, ChunkIdx: 0, Stage: stage, + SnapshotID: snap, ContentHash: hash, + Disposition: string(DispFlagged), FlagReason: string(flag), + }} + } + // The control: on a stage this pipeline STILL runs, `cancelled` costs money — that is the fix. + if r.rowsResumeFree(row("draft", FlagCancelled), draft, edit, snap, "", hashes) { + t.Fatal("a cancelled row of a LIVE stage is re-done for money and must take a slot") + } + // And on one it no longer runs, it costs nothing, because nothing will call it. + if !r.rowsResumeFree(row("retired", FlagCancelled), draft, edit, snap, "", hashes) { + t.Fatal("a stage this pipeline no longer runs cannot cost anything, whatever its last row says — " + + "charging a volume slot for it spends the grant on a call that will never be made") + } +} + +// TestAStopOverAnEscalationHopLeavesAMarkAndIsRedone is the blocker an acceptance verifier measured, +// and both halves of it were invisible to everything this pack had built. +// +// A run stopped over the HOP settled the money and left NO chunk_status row: `committed=0.001176`, +// `checkpoints=2`, `chunk_status_rows=0` — the unmarked hole §4.2 forbids. And the resume made ZERO +// fresh calls, because the hop addresses a fixed attempt 0 and the burned key there is hit forever. +// The mark lived at the attempt loop's error return, and the hop leaves runStage through another one; +// the burn-walk lived in the loop, and the hop is not in it. +// +// ⛔ THE STOP IS SYNCHRONISED ON THE HOP'S OWN REQUEST, not on a clock. A sleep here would land on the +// draft under load and leave the test green about a moment it never reached (D39.171). +func TestAStopOverAnEscalationHopLeavesAMarkAndIsRedone(t *testing.T) { + dir := t.TempDir() + var resumed atomic.Bool + hopSeen := make(chan struct{}) + var once sync.Once + var srv *cutServer + srv = newCutServer(t, func(_ int, w http.ResponseWriter, r *http.Request) { + if resumed.Load() { + answerWhole(w) + return + } + if strings.Contains(lastBody(srv), `"fake-hop"`) { + once.Do(func() { close(hopSeen) }) // the HOP is on the wire — now, and not on a timer + hold(r) + return + } + // The primary draft: content-filtered, which is deterministic and escalatable. + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"нет"},"finish_reason":"content_filter"}], + "usage":{"prompt_tokens":100,"completion_tokens":10}}`) + }) + bookPath := setupHopProject(t, dir, srv.srv.URL) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-hopSeen + cancel() + }() + _ = runOnce(t, ctx, bookPath) + + m := readMoney(t, bookPath) + if m.committed <= 0 { + t.Fatalf("the hop was delivered and cut, so it is owed: committed=%.8f", m.committed) + } + if len(m.statuses) != 1 { + t.Fatalf("a stop over the HOP left %d chunk_status row(s): money is booked and the position "+ + "exports as an unexplained gap, which is exactly the invisible hole this pack forbids", len(m.statuses)) + } + if got := FlagReason(m.statuses[0].FlagReason); got != FlagCancelled { + t.Fatalf("the mark must name the stop, got %q", got) + } + + // And the resume re-does it: the burned hop key is spent, the next one is asked at the same budget. + before := srv.calls() + resumed.Store(true) + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("the resume must complete: %v", err) + } + if fresh := srv.calls() - before; fresh == 0 { + t.Fatal("the resume made NO fresh call after a stop over the hop — the interrupted work is " + + "never re-done and the money for it is spent, which is this construction failing the other way") + } +} + +// TestASelfCutResumesFromITSCHECKPOINTAndNotOnlyFromChunkStatus is the pin §4.3 was supposed to have and +// did not: an acceptance verifier planted both mutations that attack the checkpoint path and BOTH +// SURVIVED on a green package. +// +// The reason is the ordinary resume's own design. runStage resolves from chunk_status BEFORE any render +// (anti-wedge #1), so `TestAfterASelfCutTheResumeCallsNobody` never reaches the checkpoint at all: it +// proves the chunk_status door is shut and says nothing about the second one. Corrupt the checkpoint's +// key, or delete classify's verdict for a cut finish, and that test stays green while the engine is +// broken — in the second case broken expensively, because a cut checkpoint then reads as an EMPTY +// completion, `empty` is RETRYABLE, and the chunk is re-bought on a doubled budget. +// +// So this one enters through the door the checkpoint exists for: the torn store the fast path is +// documented against («kill -9 loses ≤1 call»). The chunk_status row is removed and the resume is asked +// what it does with the checkpoint alone. +func TestASelfCutResumesFromITSCHECKPOINTAndNotOnlyFromChunkStatus(t *testing.T) { + dir := t.TempDir() + var resumed atomic.Bool + srv := newCutServer(t, func(_ int, w http.ResponseWriter, r *http.Request) { + if resumed.Load() { + answerWhole(w) + return + } + hold(r) // delivered, and our own deadline walks away + }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("a self-cut is a disposition, not an infra failure: %v", err) + } + before := srv.calls() + m := readMoney(t, bookPath) + if len(m.checkpoints) != 1 || len(m.statuses) != 1 { + t.Fatalf("the fixture must leave exactly one checkpoint and one status row, got %d/%d", + len(m.checkpoints), len(m.statuses)) + } + + // The torn store: the settle landed, the read-model row did not. Everything the resume can go on is + // the checkpoint. + dropped := dropChunkStatus(t, bookPath) + if dropped != 1 { + t.Fatalf("the fixture removed %d chunk_status row(s), want 1 — it is not measuring the torn store", dropped) + } + + resumed.Store(true) + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("the resume must complete: %v", err) + } + if fresh := srv.calls() - before; fresh != 0 { + t.Fatalf("the resume made %d fresh provider call(s) with the checkpoint of a self-cut in front of "+ + "it: the generation we already paid for is being bought again through the door chunk_status "+ + "was not covering", fresh) + } + after := readMoney(t, bookPath) + if diff := after.committed - m.committed; diff > 1e-12 || diff < -1e-12 { + t.Fatalf("the resume moved the money: %.8f → %.8f", m.committed, after.committed) + } + if FlagReason(after.statuses[0].FlagReason) != FlagAttemptTimeout { + t.Fatalf("the rebuilt row must carry the cut's own cause, got %q — read as an empty completion it "+ + "becomes RETRYABLE and the chunk is re-bought on a doubled budget", after.statuses[0].FlagReason) + } +} + +// dropChunkStatus deletes the book's chunk_status rows and returns how many went, leaving the +// checkpoints alone. It reaches the file directly because that is the only way to reproduce the state +// the checkpoint fast path exists for — a process killed between the settle and the read-model write. +// The redrive primitive cannot stand in: it deletes the checkpoints too, which is the opposite fixture. +func dropChunkStatus(t *testing.T, bookPath string) int64 { + t.Helper() + r, err := NewReadOnlyRunner(bookPath, obs.NewLogger()) + if err != nil { + t.Fatal(err) + } + dbPath := r.Book.ProjectDB + r.Close() + + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + res, err := db.Exec(`DELETE FROM chunk_status`) + if err != nil { + t.Fatal(err) + } + n, err := res.RowsAffected() + if err != nil { + t.Fatal(err) + } + return n +} + +// TestAnAcceptedSocketThatNobodyReadCostsNothing is the fixture an acceptance verifier built to expose +// the over-charge, turned into the pin for its remedy. +// +// A socket that ACCEPTS the connection and never reads a byte is what a load balancer in front of a +// dead backend looks like: our write lands in the peer's TCP window, `WroteRequest` fires, and nothing +// behind it ever sees the request. Measured before the money predicate was narrowed: 3 cancelled +// in-flight calls in 25 settled an estimate for a request no handler had entered. +// +// The class is still named — a delivered request that we cut is a `flagged(attempt_timeout)` position +// either way — and only the number is zero. Charging a reader for a call nobody ran is the one +// direction the canon forbids (D39.196 п.2а); under-counting is the direction it absorbs. +func TestAnAcceptedSocketThatNobodyReadCostsNothing(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var accepted atomic.Int32 + go func() { + var held []net.Conn + for { + c, aerr := ln.Accept() + if aerr != nil { + for _, h := range held { + h.Close() + } + return + } + accepted.Add(1) + held = append(held, c) // accepted, and not one byte is ever read from it + } + }() + t.Cleanup(func() { ln.Close() }) + + dir := t.TempDir() + bookPath := setupCutProject(t, dir, "http://"+ln.Addr().String()) + _ = runOnce(t, context.Background(), bookPath) + + if n := accepted.Load(); n == 0 { + t.Fatal("the socket was never connected to — this fixture measured nothing") + } + m := readMoney(t, bookPath) + if m.committed != 0 { + t.Fatalf("no reply ever came back, so no provider acknowledged this request; charging $%.8f for "+ + "it bills a reader for a call nobody ran", m.committed) + } + if m.reserved != 0 { + t.Fatalf("the reservation must be resolved either way, got %.8f", m.reserved) + } + // The class is still named — the position is not a silent hole just because it cost nothing. + if len(m.statuses) != 1 || FlagReason(m.statuses[0].FlagReason) != FlagAttemptTimeout { + t.Fatalf("a delivered call our own deadline cut is flagged whatever it cost; got %d row(s): %+v", + len(m.statuses), m.statuses) + } + if m.estRows != 0 { + t.Fatalf("nothing was booked, so nothing is an estimate; got %d row(s)", m.estRows) + } +} + +// TestABurnedBankBatchIsNotCountedAsPaid: the bank pass gates its role sub-budget on «does a checkpoint +// exist at this batch's key». Cut calls are now settled at that key, so the answer stopped meaning what +// the gate reads it as — the batch WILL be bought again (runAttempt walks past a burned key), and a +// probe answering «already paid» lets that purchase escape the only budget bounding bank spend. +// +// Before cut calls were settled, «a checkpoint exists» and «this batch is done» were the same statement. +// This pack made them different, so the probe had to learn the difference. +func TestABurnedBankBatchIsNotCountedAsPaid(t *testing.T) { + // The control first: an ordinary answered checkpoint IS a paid batch, or this test would pass on a + // probe that simply always says no. + answered := &store.Checkpoint{FinishReason: "stop", ResponseText: "термин\tterm", CostUSD: 0.01} + if burnedByCut(answered) { + t.Fatal("an answered batch must read as paid — the control of this fixture") + } + for _, finish := range []string{cancelledFinish, connectionLostFinish} { + burned := &store.Checkpoint{FinishReason: finish, ResponseText: "", CostUSD: 0.01} + if !burnedByCut(burned) { + t.Fatalf("a %q checkpoint records money and no result; counting it as a paid batch lets the "+ + "re-purchase escape the role sub-budget", finish) + } + } +} + +// TestTheBankPaidProbeSeesThroughABurnedCheckpoint pins the PROBE, not the predicate under it. The +// first version of this test asserted `burnedByCut` directly — true, and vacuous: removing the probe's +// use of it left the package green and the mutation SURVIVED. +// +// What the probe gates is the role sub-budget: «already paid» means «do not charge this batch again». +// Cut calls are settled at the batch's own key now, so a burned row there is money with no result — the +// batch WILL be bought again (runAttempt walks past it), and a probe that answers «paid» lets that +// purchase escape the only budget bounding bank spend. +func TestTheBankPaidProbeSeesThroughABurnedCheckpoint(t *testing.T) { + dir := t.TempDir() + srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { answerWhole(w) }) + bookPath := setupCutProject(t, dir, srv.srv.URL) + r, err := NewRunner(bookPath, obs.NewLogger()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + st := r.Pipeline.Stages[0] + ch := chunk.Chunk{Chapter: 0, ChunkIdx: 0} + msgs := []llm.Message{{Role: "user", Content: "термины"}} + const snapID = "snapshot-under-test" + if serr := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); serr != nil { + t.Fatal(serr) + } + job, jerr := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID) + if jerr != nil { + t.Fatal(jerr) + } + _, maxTokens := r.bankCallBudget(st.Model, msgs) + hash := RequestHash(r.attemptRequest(st, st.Model, snapID, ch, 0, maxTokens, msgs)) + + // Nothing there yet: the control, so a probe that always answered «no» would not pass this. + if paid, perr := r.bankCheckpointExists(st, snapID, ch, msgs); perr != nil || paid { + t.Fatalf("an empty key is not a paid batch: paid=%t err=%v", paid, perr) + } + write := func(finish, text string) { + t.Helper() + resv, verdict, rerr := r.Store.Reserve(r.Book.BookID, 0.001, store.Ceilings{BookUSD: 100, DayUSD: 100}) + if rerr != nil || verdict != store.ReserveOK { + t.Fatalf("reserve: %v %v", verdict, rerr) + } + if serr := r.Store.SettleWithCheckpoint(resv, 0.001, store.Checkpoint{ + RequestHash: hash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: 0, + Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: st.Model, + ResponseText: text, UsageJSON: "{}", CostUSD: 0.001, FinishReason: finish, + }, nil); serr != nil { + t.Fatalf("settle: %v", serr) + } + } + write(cancelledFinish, "") + if paid, perr := r.bankCheckpointExists(st, snapID, ch, msgs); perr != nil || paid { + t.Fatalf("a burned key is money with no result and the batch will be bought again; answering "+ + "«paid» lets that purchase escape the role sub-budget. paid=%t err=%v", paid, perr) + } + // And the other side: a real answer at the same key IS a paid batch, or the probe now says no to + // everything and the sub-budget gate has simply been turned off. + if derr := r.Store.PutDerivedCheckpoint(store.Checkpoint{ + RequestHash: hash + "-answered", JobID: job.ID, ChunkIdx: ch.ChunkIdx, Stage: st.Name, Role: st.Role, + ResponseText: "термин\tterm", UsageJSON: "{}", FinishReason: "stop", + }); derr != nil { + t.Fatal(derr) + } + if !burnedByCut(&store.Checkpoint{FinishReason: cancelledFinish}) || + burnedByCut(&store.Checkpoint{FinishReason: "stop"}) { + t.Fatal("the predicate under the probe must still tell the two apart") + } +} + +// TestABurnFollowedByARegenerationDoesNotOverBuy is the compound case, and it exists because a +// SURVIVING mutation said the simple one had stopped proving anything. +// +// Moving the burn walk inside runAttempt made the straightforward re-do correct BY CONSTRUCTION: the +// caller's index never advances for a burn, so «budget from the attempt index» and «budget from the +// doubling count» agree and the mutation that swaps them is a no-op. They diverge only AFTER the loop +// copies the walked index back — a burn, then a length flag, then a regeneration. That is where the +// axis split still earns its keep, and where over-buying is real: the regeneration would ask for FOUR +// times the base instead of two, and reserve four times the money with it. +// +// The sequence: a cancelled call at attempt 0 (burned key) → resume walks to attempt 1 and calls at the +// SAME budget → that answer is truncated → one regeneration at exactly TWICE the base. +func TestABurnFollowedByARegenerationDoesNotOverBuy(t *testing.T) { + dir := t.TempDir() + var resumed atomic.Bool + var afterBurn atomic.Int32 + srv := newCutServer(t, func(_ int, w http.ResponseWriter, r *http.Request) { + if !resumed.Load() { + hold(r) // run 1: delivered, and the operator stops the run + return + } + if afterBurn.Add(1) == 1 { + // The re-done call: truncated, which is retryable and buys exactly one doubling. + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"половина главы"},"finish_reason":"length"}], + "usage":{"prompt_tokens":100,"completion_tokens":10}}`) + return + } + answerWhole(w) + }) + bookPath := setupRegenProject(t, dir, srv.srv.URL) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-srv.arrived + cancel() + }() + _ = runOnce(t, ctx, bookPath) + + cutBudget := maxTokensOfBody(t, srv.allBodies()[0]) + // ⛔ THE PREMISE, and without it this fixture measures a different scenario with the same numbers. The + // budgets below agree just as well on a run that was stopped BEFORE delivery: there is no burned key + // then, the resume simply makes a first call and regenerates it, and every assertion still passes. + // What makes this the compound case is that run 1 left a key holding money-or-nothing and NO result, + // which the resume must walk past rather than replay. + if n := len(readMoney(t, bookPath).checkpoints); n != 1 { + t.Fatalf("premise broken: run 1 must leave exactly ONE checkpoint — the cut call's burned key — "+ + "and it left %d. Without that key the resume below is an ordinary first call and this fixture "+ + "proves nothing about the axis it exists for", n) + } + before := srv.calls() + resumed.Store(true) + if err := runOnce(t, context.Background(), bookPath); err != nil { + t.Fatalf("the resume must complete: %v", err) + } + + after := srv.allBodies()[before:] + if len(after) != 2 { + t.Fatalf("the resume must re-do the cut call and then regenerate it once: %d call(s)", len(after)) + } + redone, regenerated := maxTokensOfBody(t, after[0]), maxTokensOfBody(t, after[1]) + if redone != cutBudget { + t.Fatalf("the re-done call must carry the budget the cut one was granted: cut %d, re-do %d", + cutBudget, redone) + } + if regenerated != 2*cutBudget { + t.Fatalf("one regeneration is ONE doubling: want %d, got %d — keyed on the attempt index instead "+ + "of the doubling count it asks for %d and reserves the money for it", + 2*cutBudget, regenerated, 4*cutBudget) + } + // And the walk really happened: the burned key is still there, beside the two the resume bought. + if n := len(readMoney(t, bookPath).checkpoints); n != 3 { + t.Fatalf("premise broken: the burned key plus the re-done call plus its regeneration is three "+ + "checkpoints; got %d, so the resume did not walk past the burn the way this test assumes", n) + } +} + +// setupRegenProject is setupCutProject with one regeneration allowed, so a truncated answer buys a +// doubling instead of a flag. +func setupRegenProject(t *testing.T, dir, providerURL string) string { + t.Helper() + bookPath := setupCutProject(t, dir, providerURL) + writeFile(t, filepath.Join(dir, "pipeline.yaml"), ` +core: C1 +version: 1 +defaults: { max_output_ratio: 2.0, min_max_tokens: 512 } +retries: { regenerate_before_escalate: 1 } +stages: + - { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" } +`) + return bookPath +} diff --git a/backend/internal/pipeline/disposition.go b/backend/internal/pipeline/disposition.go index cb213171..2d840750 100644 --- a/backend/internal/pipeline/disposition.go +++ b/backend/internal/pipeline/disposition.go @@ -11,6 +11,7 @@ import ( "textmachine/backend/internal/lang" "textmachine/backend/internal/langscreen" "textmachine/backend/internal/llm" + "textmachine/backend/internal/store" ) // disposition.go: the per-chunk×stage verdict machinery of the Milestone-2 runner @@ -78,6 +79,27 @@ const ( // Emitted by the runner's billed-decode path. FlagDecodeError FlagReason = "decode_error" // 2xx with an unreadable body — billed, conservatively settled, flagged + // FlagAttemptTimeout and FlagCancelled are the two verdicts of a call THE ENGINE ITSELF cut short + // (backlog row 360). Both name their cause exactly, and they are two constants rather than one + // because the disposition differs on every axis that matters: + // + // - attempt_timeout — OUR deadline fired while the provider was generating. Deterministic on the + // same budget, so it is NOT retryable and NOT escalatable: a same-model retry re-buys the same + // generation, which is the defect row 360 names. The remedy is a bigger deadline (now derived + // from the budget) or a redrive, not another call inside this run. + // - cancelled — a HUMAN stopped the run over a call that was already on the wire. Nothing was + // wrong with it, so the chunk is not «needs a person»; it is «stopped, resume will re-do it». + // It still marks the position: a hole with no mark at all would export as an unexplained gap. + // + // ⚠ The vocabulary is CONVERTED from the transport's causes, not re-typed beside them: one carrier + // for the three names the wire, the checkpoint, the flag and the operator all read. + FlagAttemptTimeout FlagReason = FlagReason(llm.CutBySelfDeadline) + FlagCancelled FlagReason = FlagReason(llm.CutByParent) + // ⚠ A LOST CONNECTION HAS NO FLAG, and that is a statement about the mechanism rather than an + // omission. It is an infra pause, not a chunk that needs a human, and the checkpoint it leaves + // records money with no result — which burnedByCut catches BEFORE anything classifies it, so no + // replay of that row ever reaches a disposition. The guard is that predicate, not a name. + // Reserved for later steps — DEFINED for contract stability, NOT emitted by // Milestone 2. coverage_fail / excision_suspect are verdicts of the configurable // coverage gate (step 6); hard_block / upstream_not_ok are for HTTP-level @@ -144,6 +166,44 @@ var classifierVersion = "classify-v3-refusal+srcscript-echo015+" + langscreen.Ve // path assigned — live and resume must agree (determinism of the resolve). const decodeErrorFinish = "decode_error" +// The finish_reason a cut call's checkpoint carries. They come from the transport's own cause +// vocabulary (llm.CutCause) so that the wire fact, the stored row and the flag cannot be named three +// different things by three files. +// +// ⛔ TWO OF THEM ARE NOT RESULTS, and that is the load-bearing distinction. attemptTimeoutFinish IS a +// verdict — classify resolves it below and a resume serves it without calling anybody, which is the +// point: we already paid for that generation and will not buy it again. The other two record MONEY for +// a call that has no outcome at all, so replaying them would hand the stage an empty answer it never +// received; runAttempt burns them instead (burnedByCut) and the loop re-asks at the SAME budget. +const ( + attemptTimeoutFinish = string(llm.CutBySelfDeadline) + cancelledFinish = string(llm.CutByParent) + connectionLostFinish = string(llm.CutByConnection) +) + +// burnedByCut says a checkpoint records what a call COST without recording what it produced. Such a +// row is money only: it must never be served as a result, and its attempt index is spent — the next +// index re-asks the same budget under a fresh request_hash. +// +// This is the whole reason the money can be booked at all. The store has exactly three money verbs +// and none of them writes spend without a checkpoint, while the checkpoint IS the resume key +// (`ON CONFLICT (request_hash) DO NOTHING`) — so «pay and re-do under the same key» is impossible by +// construction. Separating the ATTEMPT INDEX from the count of budget DOUBLINGS (runStage) is what +// dissolves that: the re-done call is a different key at the same budget, its reserve→settle is +// fresh, and `committed == sum(checkpoints)` never wobbles. +// +// ⚠ THE TEXT IS PART OF THE TEST, and the finish_reason alone was not enough. That string shares a +// namespace with whatever a vendor decides to print — the adapter already normalises invented values +// like «sensitive» — so a provider answering 200 with a real translation and finish_reason +// «cancelled» would have had its answer thrown away and re-bought. A genuine burn is written here and +// is always textless (cutcall.go), so asking for both costs nothing and closes the namespace. +func burnedByCut(cp *store.Checkpoint) bool { + if cp.ResponseText != "" { + return false // a reply with content is an answer, whatever it calls its finish reason + } + return cp.FinishReason == cancelledFinish || cp.FinishReason == connectionLostFinish +} + // retryable reports whether a flag may be re-attacked on the SAME model along the // attempt axis. Only length/empty: both are budget symptoms a bigger max_tokens // can cure. Everything else is deterministic (refusal/filter/echo/loop/decode) — @@ -242,6 +302,23 @@ func classify(in classifyInput) classification { switch finish { case decodeErrorFinish: return classification{FlagDecodeError, "billed 2xx with an unreadable body"} + case attemptTimeoutFinish: + // Live and resume must agree, exactly as for a decode checkpoint: the row was written by the + // live path with this finish and no text, and re-reading it has to resolve to the same verdict + // instead of falling through to «empty completion», which is retryable and would re-buy the + // generation this whole class exists to stop buying twice. + // + // ⚠ It does NOT move classifierVersion, and that is provable rather than hoped: the constant + // guards a change that could RE-VERDICT a stored checkpoint, and no checkpoint written before + // this line can carry this finish_reason — the string did not exist and no provider emits it. + // Bumping it would re-snapshot every book in the world to change the verdict of nothing. + // ⚠ ONLY OVER AN EMPTY OUTPUT. The engine writes this finish_reason with no text at all + // (cutcall.go); the string itself, though, shares a namespace with whatever a vendor prints — + // the adapter already normalises invented values — and a provider answering 200 with a real + // translation under this name would otherwise have its answer thrown away as a lost call. + if out == "" { + return classification{FlagAttemptTimeout, "our own deadline cut a delivered call; the provider generated and billed it, so it is not retried"} + } case llm.FinishContentFilter: return classification{FlagContentFilter, "provider finish_reason=content_filter"} case llm.FinishRefusal: @@ -383,19 +460,30 @@ func degenerateLoop(text string) bool { // --- max_tokens on the attempt axis (D2.3) --- -// maxTokensForAttempt is the PURE output-token budget for a retry attempt: -// attempt 0 = base, each regeneration DOUBLES it (D2.3 remedy for a length cut — -// the previous budget was too small). Purity is load-bearing: the value enters -// request_hash, so resume must reproduce the identical per-attempt budget. Its -// FORMULA is versioned into the snapshot (maxTokensPolicyVersion) so a change is -// a loud --resnapshot, not a silent checkpoint miss on retried chunks (the same -// discipline as estimatorVersion). -func maxTokensForAttempt(base, attempt int) int { - if attempt <= 0 { +// maxTokensForAttempt is the PURE output-token budget for a retry attempt: no doublings = base, each +// regeneration DOUBLES it (D2.3 remedy for a length cut — the previous budget was too small). Purity +// is load-bearing: the value enters request_hash, so resume must reproduce the identical budget. Its +// FORMULA is versioned into the snapshot (maxTokensPolicyVersion) so a change is a loud --resnapshot, +// not a silent checkpoint miss on retried chunks (the same discipline as estimatorVersion). +// +// ⛔ IT COUNTS DOUBLINGS, NOT ATTEMPTS, and the two used to be the same number by accident. Every +// regeneration is a new attempt, so «attempt index» read as «times the budget was doubled» and both +// callers agreed — until a call the ENGINE cut short had to be re-done. That re-do must NOT be paid +// for with a doubled budget: nothing was wrong with the answer, nobody saw it, and doubling would buy +// twice the call for a health nobody lost. Since a checkpoint cannot be written twice under one key, +// the re-do has to be a NEW attempt index — so the two dimensions had to come apart, and this +// parameter is the one that is about money. +// +// ⚠ THE FORMULA AND ITS OUTPUT ARE UNCHANGED FOR EVERY PATH THAT EXISTS TODAY: both loops that +// regenerate increment the doubling count with the attempt index, so on a run with no cut call the +// two are identical and the snapshot does not move. That equality is the reason this could land on +// books that are mid-translation at all — moving maxTokensPolicyVersion would re-pay every one of them. +func maxTokensForAttempt(base, escalations int) int { + if escalations <= 0 { return base } - if attempt > 20 { // defensive: never shift by a runaway amount (overflow guard) - attempt = 20 + if escalations > 20 { // defensive: never shift by a runaway amount (overflow guard) + escalations = 20 } - return base << uint(attempt) + return base << uint(escalations) } diff --git a/backend/internal/pipeline/escalation.go b/backend/internal/pipeline/escalation.go index b7e18739..1c791386 100644 --- a/backend/internal/pipeline/escalation.go +++ b/backend/internal/pipeline/escalation.go @@ -138,20 +138,21 @@ func (r *Runner) maybeEscalate(ctx context.Context, st config.Stage, snapID stri // and flipping the verdict OK→flagged. Only a FRESH hop is budget-gated. The identity is BUILT by the // same helper the hop's own runAttempt uses (attemptRequest), so "mirrors runAttempt" is structural // rather than a promise two field lists keep by vigilance. - fbHash := RequestHash(r.attemptRequest(st, st.ResolvedHop, snapID, ch, 0, hopMaxTokens, msgs)) - fbExists, err := r.Store.GetCheckpoint(fbHash) + mayHop, err := r.paidAfterBurns(st, st.ResolvedHop, snapID, ch, 0, hopMaxTokens, msgs) if err != nil { return out, err } - mayHop := fbExists != nil if !mayHop { // Fresh hop: serialize the budget admission THROUGH the paid settle across the parallel the draft wave draft // workers (R1, escMu). escalationBudgetRemains is a non-atomic read-then-act over EscalationSpentUSD, // so without this N concurrent draft chunks could each read spent= rank(FlagLength) { + t.Fatalf("a contaminated output that was DROPPED must be more severe than a budget symptom: "+ + "defect=%d length=%d", rank(FlagSanitizerDefect), rank(FlagLength)) + } +} diff --git a/backend/internal/pipeline/paidtail.go b/backend/internal/pipeline/paidtail.go index 3c80a062..a5aceaa7 100644 --- a/backend/internal/pipeline/paidtail.go +++ b/backend/internal/pipeline/paidtail.go @@ -1,8 +1,10 @@ package pipeline import ( + "encoding/json" "fmt" + "textmachine/backend/internal/llm" "textmachine/backend/internal/store" ) @@ -199,3 +201,59 @@ func (p pos) name() string { } return fmt.Sprintf("ch%d/chunk%d/%s", p.chapter, p.chunkIdx, p.stage) } + +// estimatedSpend is how much of a book's committed money is an ESTIMATE rather than a figure the +// provider reported, and over how many calls. It sits beside paidTail because it asks the same kind of +// question of the same rows: that one decomposes committed spend by what it BOUGHT, this one by how +// well it is KNOWN. +// +// It is the engine's half of PD-441 and the condition attached to D39.230 п.1. Without it the platform +// bills committed_usd and can say «at least this much», never «at least X, up to Y». +// +// ⛔ THE TEST IS «PAID WITH NO TOKENS TO SHOW FOR IT», not a list of causes. Cost is derived from usage +// everywhere else in the engine, so a settled row with zero tokens and a non-zero cost can only have +// come from a reservation estimate — whatever ended the call: a body that would not decode, a deadline +// of ours, a stopped run, a broken socket, a paid 2xx that reported no usage. A list of finish_reasons +// would answer the same question today and quietly stop answering it the day a sixth way to pay +// without a token count is added. +// +// Reading the checkpoints rather than request_log is what makes the pair a decomposition of +// committed_usd instead of a second, telemetry-shaped opinion about it: the two are the same money, and +// a resumed run writes no new estimate row while its money stands. +// +// ⛔ `committed == SUM(checkpoints)` HOLDS ONLY UNTIL A REDRIVE, and the difference is money this +// figure would otherwise disown. A redrive DELETES the checkpoints of the stages it re-attacks and +// leaves their spend committed — «after a redrive committed(spend) >= SUM(checkpoints), the safe +// direction» (store/chunkstatus.go). Derived from the surviving rows alone, the published share then +// falls to zero and tells the platform that money nobody can account for was measured. So the gap is +// carried INTO the estimate by the same rule the rows are: we hold a cost and have no token count to +// justify it. Measured on a live redrive: committed unchanged at $0.001056, estimated dropped to $0. +func estimatedSpend(usage []store.CheckpointUsage, committedUSD float64) (rows int, usd float64) { + var accounted float64 + for _, u := range usage { + accounted += u.CostUSD + } + for _, u := range usage { + if u.CostUSD <= 0 { + continue // derived $0 checkpoints are not calls and cost nothing + } + var tok llm.Usage + if err := json.Unmarshal([]byte(u.UsageJSON), &tok); err != nil { + // Unreadable usage on a paid row is the same state as absent usage: a cost we cannot + // justify from tokens. Counting it as measured publishes the more comfortable answer. + rows, usd = rows+1, usd+u.CostUSD + continue + } + if tok.PromptTokens == 0 && tok.CompletionTokens == 0 && tok.ReasoningTokens == 0 { + rows, usd = rows+1, usd+u.CostUSD + } + } + // Committed money with no checkpoint behind it at all: a redrive threw the evidence away and kept + // the spend. It is counted as ONE more unaccounted line rather than as a per-call figure, because + // how many calls it stood for is exactly what was deleted. The float slack keeps a sum of prices + // from inventing a nanodollar of «unaccounted». + if gap := committedUSD - accounted; gap > 1e-9 { + rows, usd = rows+1, usd+gap + } + return rows, usd +} diff --git a/backend/internal/pipeline/priceprojection.go b/backend/internal/pipeline/priceprojection.go index b02e5be6..725e8e8f 100644 --- a/backend/internal/pipeline/priceprojection.go +++ b/backend/internal/pipeline/priceprojection.go @@ -349,8 +349,13 @@ func (r *Runner) stepMaxForUnit(p *pricePlan, u editUnit, up unitPrice) float64 max := 0.0 consider := func(sp stagePrice, sizing, prompt int) { base := r.baseMaxTokensFor(sp.st, sizing) - for attempt := 0; attempt <= p.maxRegen; attempt++ { - if usd := ledger.EstimateUSD(sp.price, prompt, maxTokensForAttempt(base, attempt), sp.reasoning); usd > max { + // The walk is over DOUBLINGS, which is what it always was and now says so: the regeneration cap + // bounds how many times a budget may double, and the largest single reservation is the last of + // them. The projection is untouched by splitting the doubling count off the attempt index — + // re-doing a call the engine cut short adds an attempt at an EXISTING budget, so it buys no + // reservation this walk has not already priced. + for escalations := 0; escalations <= p.maxRegen; escalations++ { + if usd := ledger.EstimateUSD(sp.price, prompt, maxTokensForAttempt(base, escalations), sp.reasoning); usd > max { max = usd } } diff --git a/backend/internal/pipeline/repair.go b/backend/internal/pipeline/repair.go index 20021aa3..387d756d 100644 --- a/backend/internal/pipeline/repair.go +++ b/backend/internal/pipeline/repair.go @@ -300,6 +300,12 @@ func (r *Runner) maybeRepair(ctx context.Context, st config.Stage, snapID string } } att, err := r.runRepairAttempt(ctx, st, snapID, ch, job, i, msgs) + // ⛔ THE MONEY IS REPORTED BEFORE THE OUTCOME IS JUDGED. A repair call the engine CUT still cost + // what it cost, and the caller's chunk row is the only place that cost can be recorded — the + // error path used to return before this and left the row below the ledger for the position. + res.CostUSD += att.runCost + res.CumUSD += att.cumCost + res.Fresh = res.Fresh || att.freshCall if err != nil { // A ceiling denial must NOT abort the book: this step is optional and sits BEFORE the // chunk_status write, so propagating would discard the row of an already-paid, successful @@ -312,9 +318,6 @@ func (r *Runner) maybeRepair(ctx context.Context, st config.Stage, snapID string return finalText, res, err } res.Calls++ - res.CostUSD += att.runCost - res.CumUSD += att.cumCost - res.Fresh = res.Fresh || att.freshCall spent += att.runCost // a replay costs 0, so only fresh calls consume the budget reply, verdict := repairReplyVerdict(att, dstSpan, c.Class, cfg.Checkers) switch verdict { @@ -361,11 +364,16 @@ func (r *Runner) maybeRepair(ctx context.Context, st config.Stage, snapID string // mirrors runRepairAttempt's request identity BY CONSTRUCTION — same derived stage, same helper — because // the two must address the same checkpoint; the doccomment used to promise that mirroring while the two // field lists were maintained apart, which held only while the dropped fields were zero. +// +// ⛔ A BURNED KEY IS NOT A PAID REPAIR. A row that records money and no result cannot be replayed, so +// runAttempt walks past it and buys the repair again — and «already paid» is what skips the sub-budget +// comparison two lines up from the call, so that purchase would happen with nothing bounding it. The key +// can hold such a row only because cut calls are settled there now; before that, «a checkpoint exists» +// and «this repair is done» were the same statement. Same reading as the bank probe's. func (r *Runner) repairCheckpointExists(st config.Stage, snapID string, ch chunk.Chunk, ordinal int, msgs []llm.Message) (bool, error) { model, maxTokens := r.repairCallBudget(msgs) rst := r.repairStage(st, model) - cp, err := r.Store.GetCheckpoint(RequestHash(r.attemptRequest(rst, model, snapID, ch, ordinal, maxTokens, msgs))) - return cp != nil, err + return r.paidAfterBurns(rst, model, snapID, ch, ordinal, maxTokens, msgs) } // repairStage derives the stage of a repair call from the stage whose output is being repaired. It takes diff --git a/backend/internal/pipeline/stagerun.go b/backend/internal/pipeline/stagerun.go index f915d389..4b56141f 100644 --- a/backend/internal/pipeline/stagerun.go +++ b/backend/internal/pipeline/stagerun.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "sync" "textmachine/backend/internal/chunk" "textmachine/backend/internal/config" "textmachine/backend/internal/ledger" @@ -35,7 +36,7 @@ import ( // able to tell that a repaired span did not drop a canonical glossary form (with the post-check gate on that // would flip the unit to flagged and ship an EMPTY export). It is threaded rather than re-derived so the // re-gate judges the SAME selection the injection was rendered from. -func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, snapID string, ch chunk.Chunk, prev, injection string, injected []membank.PickedEntry) (*StageResult, error) { +func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, snapID string, ch chunk.Chunk, prev, injection string, injected []membank.PickedEntry) (res *StageResult, err error) { // Enrich ReqInfo FIRST, PRESERVING the admission decisions (LogBodies) — // a from-scratch overwrite would sever the documented debug channel (review // finding). Earlier the enrichment sat AFTER the resume-fast-path — and every @@ -89,6 +90,7 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn if cs, err := r.Store.GetChunkStatus(r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name); err != nil { return nil, fmt.Errorf("pipeline: read chunk_status %s/ch%d/chunk%d/%s: %w", r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, err) } else if cs != nil && cs.ContentHash == contentHash && cs.Disposition != string(DispSkipped) && + resolvedForResume(cs) && (cs.SnapshotID == snapID || r.repinnable(cs.SnapshotID, snapID, waveOfStage(r.Pipeline.Stages, st.Name))) { // POINTWISE RE-EDIT (pack-20 point 5). The exact-snapshot case is the ordinary resume. The second // case is the one that makes signing a term affordable: the snapshot moved ONLY because the BANK @@ -137,45 +139,79 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn var last stageAttempt anyFresh := false attemptsMade := 0 + // ⛔ THE MARK FOR A STOPPED POSITION IS ATTACHED TO EVERY EXIT, not to the attempt loop's. It was + // written at the loop's error return, and the hop and the repair sub-step both leave this function + // through OTHER returns — so a run stopped over an escalation hop settled the + // money and left NO chunk_status row at all: the invisible hole §4.2 forbids, measured as + // `committed=0.001176` with `chunk_status_rows=0`. A deferred rule sees whichever return fires, and + // closes over the counters so it reports what had accumulated by then. recordCancelledStage is a + // no-op for every error that is not a delivered call a person stopped, so this costs nothing on the + // ordinary paths. + defer func() { + r.recordCancelledStage(ctx, cancelledPosition{ + stage: st, chunk: ch, snapshotID: snapID, contentHash: contentHash, + cumCostUSD: cumCost, attempts: attemptsMade, + }, err) + }() // firstFlagReason keeps the FIRST attempt's failure when a later attempt (a regenerate, or the // single-hop escalation below) recovers the chunk. Without it the recovered row is written `ok` // with an empty flag_reason and the primary failure leaves no durable trace at all — which is how // the mini-run of 25.07 reported echo_draft=0.0% on a run where one draft in twenty had echoed. // It is telemetry, never the verdict: `disposition`/`FlagReason` below are untouched by it. firstFlagReason := FlagReason("") + // escalations counts BUDGET DOUBLINGS; attempt counts KEYS. They move together on every + // regeneration and come apart on exactly one path: re-doing a call the engine itself cut short, + // which needs a fresh request_hash (the old one already holds that call's money) at the budget it + // was already granted. See maxTokensForAttempt. + escalations := 0 + judged := 0 for attempt := 0; ; attempt++ { - maxTokens := maxTokensForAttempt(baseMaxTokens, attempt) + maxTokens := maxTokensForAttempt(baseMaxTokens, escalations) att, err := r.runAttempt(ctx, st, st.ResolvedModel, snapID, ch, job, attempt, maxTokens, msgs, false, isFinal, true) - if err != nil { - return nil, err // infra failure - } - attemptsMade = attempt + 1 cumCost += att.cumCost runCost += att.runCost + // ⛔ THE COUNT IS A FACT ABOUT WHAT HAPPENED, not about whether it succeeded — and it is recorded + // before the error check for the same reason the money above is. The deferred mark reports this + // number BESIDE money that was really paid; leaving it behind the check wrote `attempts=0` on a + // position whose ledger said 0.001056. runAttempt may also have walked over burned keys, so the + // loop follows its index and the next regeneration does not re-address a key that is spent. + attempt = att.attempt + attemptsMade = attempt + 1 + if err != nil { + return nil, err // infra failure; the cancelled-position mark is the defer above + } anyFresh = anyFresh || att.freshCall last = att - if attempt == 0 && !att.cls.ok() { + if judged == 0 && !att.cls.ok() { firstFlagReason = att.cls.Reason } + judged++ if att.cls.ok() { break } // Flagged: re-attack only the retryable subset, only while regenerations // remain (a bigger budget on the attempt axis, D2.3). Everything else is // deterministic — a same-model retry would re-refuse and re-bill (D2.2). - if att.cls.Reason.retryable() && attempt < maxRegen { + if att.cls.Reason.retryable() && escalations < maxRegen { r.Log.WarnContext(ctx, "stage flagged, regenerating with a larger budget", "stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx, - "attempt", attempt, "reason", string(att.cls.Reason), "next_max_tokens", maxTokensForAttempt(baseMaxTokens, attempt+1)) + "attempt", attempt, "reason", string(att.cls.Reason), "next_max_tokens", maxTokensForAttempt(baseMaxTokens, escalations+1)) + escalations++ continue } // Echo (cjk_artifact) OPT-IN re-generation before escalation (row 77 / D39.61): on a provider whose // echo is STOCHASTIC per call, a same-model re-gen recovers ~7.6× cheaper than the escalation hop. // Default 0 ⇒ this never fires and echo escalates straight away (the prior behaviour); the echo GATE // is untouched — only the RESPONSE changes. - if att.cls.Reason == FlagCJKArtifact && attempt < echoRegen { + if att.cls.Reason == FlagCJKArtifact && escalations < echoRegen { r.Log.WarnContext(ctx, "echo flagged, regenerating before escalation (echo is stochastic per call, D39.61)", "stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "attempt", attempt) + // ⚠ THE ECHO RE-GEN COUNTS AS A DOUBLING TOO, and it must. It is a fresh roll of a + // stochastic die rather than a bigger-budget remedy, so counting it here looks like a + // detail — but this loop has always given it `base << attempt`, and taking that away would + // move max_tokens, and with it request_hash, and with it every echo-regenerated + // checkpoint on disk. The doubling axis was split to add a case, not to re-price one. + escalations++ continue } break @@ -200,13 +236,17 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn // column. escalated, escModel := false, "" esc, err := r.maybeEscalate(ctx, st, snapID, ch, job, baseMaxTokens, msgs, last, isFinal) + // ⛔ MONEY FIRST, VERDICT SECOND — and the order is the whole point. A hop that was CUT reports its + // cost through `esc.fb` and an error at the same time; adding the cost only on the success path left + // the deferred `cancelled` mark carrying the primary's money alone (measured: ledger 0.001176, row + // 0.000120). `esc.fb` is a zero value when no hop ran, so this adds nothing when nothing happened. + cumCost += esc.fb.cumCost + runCost += esc.fb.runCost + anyFresh = anyFresh || esc.fb.freshCall if err != nil { return nil, err } if esc.attempted { - cumCost += esc.fb.cumCost - runCost += esc.fb.runCost - anyFresh = anyFresh || esc.fb.freshCall escalated = true if esc.fb.cls.ok() || esc.fb.cls.Reason == FlagSanitizerStripped { // The fallback is authoritative when it passed the re-gate OR when it is a cosmetic @@ -246,12 +286,14 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn // intermediate artifact rather than the final one. if isFinal { repaired, rr, rerr := r.maybeRepair(ctx, st, snapID, ch, job, prev, last.text, injected) + // Same order as the hop above: a repair call the engine cut reports its cost together with + // the error, and the row is the only place that cost can land. + cumCost += rr.CumUSD // honest total: a replayed repair still costs what it cost + runCost += rr.CostUSD // this run's spend only if rerr != nil { return nil, rerr } repairRes = rr - cumCost += rr.CumUSD // honest total: a replayed repair still costs what it cost - runCost += rr.CostUSD // this run's spend only anyFresh = anyFresh || rr.Fresh if rr.Applied > 0 { dh, derr := r.commitRepairExport(st, ch, job, last, repaired) @@ -433,8 +475,42 @@ func (r *Runner) callEstimateUSD(st config.Stage, model string, msgs []llm.Messa // dollars would turn «do not start anything new» into «the optional spends what the mandatory needed», // and the hop holds escMu while it waits, so it would also block every other worker's escalation. func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID string, ch chunk.Chunk, job *store.Job, attempt, maxTokens int, msgs []llm.Message, escalation, isFinal, mandatory bool) (stageAttempt, error) { - reqHash := RequestHash(r.attemptRequest(st, model, snapID, ch, attempt, maxTokens, msgs)) - att := stageAttempt{reqHash: reqHash, attempt: attempt, modelActual: model} + // ⛔ BURNED KEYS ARE WALKED OVER HERE, AND NOT BY THE CALLER. A checkpoint that records money and + // no result cannot be replayed as an answer, so its attempt index is spent and the call has to be + // re-asked under the next one at the SAME budget. That rule first lived in runStage's loop, which + // left the three callers OUTSIDE the loop with a key burnt forever: the escalation hop addresses a + // fixed attempt 0, so a run stopped over a hop could never re-do it — measured as `committed` + // booked, `chunk_status_rows = 0` and a resume that made ZERO fresh calls. The bank batches and the + // repair sub-step have the same shape. Walking here gives every caller the behaviour without any of + // them knowing the rule exists. + // + // The walk terminates by construction: it advances only while a checkpoint EXISTS at the key, and + // the first key without one takes the fresh-call path below. The money of every key it steps over + // stays in the chunk's honest total — it was really spent — while `runCost` does not move, because + // a burn can only have been settled by an EARLIER run (both causes end this one). + var burnedCost float64 + var reqHash string + var att stageAttempt + for { + reqHash = RequestHash(r.attemptRequest(st, model, snapID, ch, attempt, maxTokens, msgs)) + att = stageAttempt{reqHash: reqHash, attempt: attempt, modelActual: model} + cp, cerr := r.Store.GetCheckpoint(reqHash) + if cerr != nil { + return att, fmt.Errorf("pipeline: read checkpoint for %s/ch%d/chunk%d/%s attempt %d: %w", r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, attempt, cerr) + } + if cp == nil || !burnedByCut(cp) { + break + } + burnedCost += cp.CostUSD + // «Spent», not «paid for»: the key is used up either way, but what it COST is printed beside it + // and is legitimately zero when the provider never acknowledged the call. A line calling a $0 + // row «paid for» reads as a billing bug to whoever finds it in a log at three in the morning. + r.Log.InfoContext(ctx, "the key of this attempt is SPENT — the call was cut off before its answer arrived and cannot be replayed; re-asking under a fresh key at the SAME budget", + "stage", st.Name, "attempt", attempt, "hash", reqHash[:12], "cause", cp.FinishReason, + "max_tokens", maxTokens, "cost_usd", fmt.Sprintf("%.6f", cp.CostUSD)) + attempt++ + } + att.cumCost = burnedCost // Resume on the attempt axis: a checkpoint means THIS attempt already happened // and was billed — classify its text and never re-bill (kill -9 loses ≤1 call; @@ -451,7 +527,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID att.usage = usage att.finish = cp.FinishReason att.modelActual = cp.ModelActual - att.cumCost = cp.CostUSD + att.cumCost = burnedCost + cp.CostUSD // Banknote: slice the block off the RAW checkpoint text BEFORE classify + before feeding the // editor (WS4 points 1-3,8). The checkpoint stores the RAW draft; resume re-derives the clean // text deterministically (a no-op returning the raw text for a channel-off / no-separator draft). @@ -551,6 +627,24 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID estimate, r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, ctx.Err()) } } + // ⛔ A RUN THAT IS ENDING DID NOT STOP ON MONEY, whatever the ledger says at this instant, and the + // check belongs HERE rather than at the top of the loop — that is where it was first written, and + // it missed. `waitForSettle` answers «nothing is in flight» BEFORE it looks at the context, so a + // worker whose wait ended because the cancelled call had already settled and left falls straight + // through the switch above to the halt below, with its context long dead and the top-of-loop + // check minutes behind it. The decision point is the only place a guard on it is complete. + // + // Why it matters now: the hole was unreachable while a cancelled in-flight call gave its + // reservation back. Paying for such a call (D39.230 п.1) makes it live — a stop commits every + // flying call's estimate, which can carry the book past its own ceiling — and the next worker + // would publish a `ceiling` event for a run a person had stopped themselves. The platform lets + // that event survive any exit code, so it would record `paused` and ask for money that would + // change nothing. + if ctx.Err() != nil { + r.setJobStatus(ctx, job.ID, "failed") + return att, fmt.Errorf("pipeline: reserve $%.6f for %s/ch%d/chunk%d/%s: the run ended before the reservation was granted: %w", + estimate, r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, ctx.Err()) + } if verdict == store.ReserveDeniedBook { // Name WHICH ceiling stopped the run: a caller who passed --ceiling-usd and is told to "raise // ceilings.book_usd" would edit a file that is not in force (row 145). @@ -595,6 +689,12 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID r.Log.InfoContext(ctx, "calling model", "model", model, "attempt", attempt, "max_tokens", maxTokens, "escalation", escalation, "estimate_usd", fmt.Sprintf("%.6f", estimate)) + // A call may now legitimately run for a quarter of an hour: the deadline is derived from the + // budget, and the owner ratified waiting for a provider that has our request. That makes the ONE + // line above ("calling model") insufficient — a fifteen-minute silence after it is + // indistinguishable from a wedged process, which is exactly the pain a visible in-flight marker was + // added for in the first place. So the wait says so while it lasts, and stops the moment the call does. + stopHeartbeat := r.logWaitingForProvider(ctx, st, ch, model, attempt, maxTokens) start := time.Now() resp, err := client.Complete(ctx, llm.LLMRequest{ Model: model, @@ -604,6 +704,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID ReasoningEffort: st.Reasoning, }) att.latency = int(time.Since(start).Milliseconds()) + stopHeartbeat() if err != nil { var bde *llm.BilledDecodeError if errors.As(err, &bde) { @@ -621,7 +722,11 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID } r.events.flush() att.finish = decodeErrorFinish - att.cumCost, att.runCost = estimate, estimate + // `burnedCost +`, not `=`: the walk above may have stepped over keys this position already + // paid for, and that money is this chunk's whatever this attempt turns out to be. Overwriting + // it puts chunk_status.cost_usd below SUM(checkpoints) for the position — the ledger's own + // invariant, read by the projection a person decides on. + att.cumCost, att.runCost = burnedCost+estimate, estimate att.cls = classification{FlagDecodeError, "billed 2xx with an unreadable body: " + err.Error()} rl := r.baseRequestLog(st, ch, model, reqHash) rl.CostUSD, rl.LatencyMS, rl.FinishReason = estimate, att.latency, decodeErrorFinish @@ -631,9 +736,22 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID r.setJobStatus(ctx, job.ID, "done") return att, nil } - // No 2xx ever arrived: nothing was billed. Release and surface as an INFRA + var cut *llm.AttemptCutError + if errors.As(err, &cut) && cut.Delivered { + return r.settleCutCall(ctx, cutCall{ + stage: st, chunk: ch, job: job, model: model, reqHash: reqHash, + resv: resv, estimate: estimate, attempt: attempt, escalation: escalation, + }, cut, err, att) + } + // The request never went out: nothing was billed. Release and surface as an INFRA // failure — a long book pauses/resumes on an outage or terminal 4xx (D4), // rather than flag-storming every remaining chunk on a dead provider. + // + // ⚠ THE CONDITION IS DELIVERY, NOT A STATUS LINE, and it used to be neither: the branch read + // «No 2xx ever arrived ⇒ nothing was billed», which is false on a provider that answers 200 + // while the request is still queued. Every call our own deadline cut fell in here or into the + // decode-error branch beside it, and this one gave the ledger a zero for a generation the + // provider had made and billed. r.releaseReservation(ctx, resv) rl := r.baseRequestLog(st, ch, model, reqHash) rl.LatencyMS, rl.Err, rl.OK = att.latency, err.Error(), false @@ -679,7 +797,8 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID r.setJobStatus(ctx, job.ID, "failed") return att, err } - att.cumCost, att.runCost = cost, cost + // Same as the billed-decode branch above: the burn walk's money is added, never replaced. + att.cumCost, att.runCost = burnedCost+cost, cost // Money: settle + the raw response — one transaction (§3.3). ALWAYS, even for // an empty/truncated/refused response: it was billed by the provider, the @@ -814,3 +933,66 @@ func (r *Runner) baseMaxTokensFor(st config.Stage, sizingTokens int) int { } return base } + +// --- the visible life of a call in flight --- + +// waitHeartbeat bounds how often a call still in flight says so: often enough that an operator learns +// within a minute, rarely enough that a healthy wave prints nothing. +const waitHeartbeat = time.Minute + +// heartbeatEvery is the interval for a call that may run for `deadline` — a quarter of the wait the +// engine has actually granted, capped at a minute. Derived rather than fixed so the line exists on +// every call the engine is willing to wait for, including short ones. +func heartbeatEvery(deadline time.Duration) time.Duration { + q := deadline / 4 + if q <= 0 { + // A wait of zero or less is a caller that could not resolve one, not a call that finishes + // instantly. Falling to a millisecond here would print this line a thousand times a second, + // which is the opposite of what it is for; the cap is the honest answer to «no idea how long». + return waitHeartbeat + } + if q < waitHeartbeat { + return q + } + return waitHeartbeat +} + +// logWaitingForProvider says, while a call is still out, that it is — and returns the function that +// stops it. With a deadline derived from the budget a call may legitimately run for a quarter of an +// hour, and a single «calling model» line then leaves an operator watching a silence he cannot tell +// from a wedged process (the in-flight marker's original reason, met again at a longer timescale). +func (r *Runner) logWaitingForProvider(ctx context.Context, st config.Stage, ch chunk.Chunk, model string, attempt, maxTokens int) (stop func()) { + done := make(chan struct{}) + var once sync.Once + go func() { + t := time.NewTicker(heartbeatEvery(r.Models.AttemptDeadline(model, maxTokens))) + defer t.Stop() + started := time.Now() + for { + select { + case <-done: + return + case <-ctx.Done(): + return + case <-t.C: + // ⚠ IT DOES NOT CLAIM DELIVERY, and it used to. The runner cannot see the transport's + // trace, so «a call that has been delivered» was an assertion nobody had checked — and on + // a request that never went out it printed eighteen times, telling the operator the + // opposite of what happened. A line stating a fact it did not ask about is the same + // defect a flag naming the wrong cause is (D39.93 п.2). + // ⚠ THE FIELDS ARE NAMED FOR WHAT THEY ACTUALLY ARE, and the first version's were not. + // This timer wraps client.Complete — the WHOLE retry chain — while the deadline it can + // name is one ATTEMPT's, so `waited=4s of=1s` printed a contradiction and read as the + // hung process the line exists to rule out. `attempt_deadline` says which of the two it + // is, and a `waited` past it means the transport has retried. `stage_attempt` likewise: + // the transport logs an `attempt` of its own, and two different numbers under one key in + // one stream is a question an operator cannot answer. + r.Log.InfoContext(ctx, "still waiting for the provider on a call in flight", + "stage", st.Name, "chapter", ch.Chapter, "chunk", ch.ChunkIdx, "model", model, + "stage_attempt", attempt, "waited", time.Since(started).Round(time.Millisecond).String(), + "attempt_deadline", r.Models.AttemptDeadline(model, maxTokens).String()) + } + } + }() + return func() { once.Do(func() { close(done) }) } +} diff --git a/backend/internal/pipeline/status.go b/backend/internal/pipeline/status.go index 970873e9..54aa1402 100644 --- a/backend/internal/pipeline/status.go +++ b/backend/internal/pipeline/status.go @@ -306,7 +306,20 @@ type StatusReport struct { // disposition — a nonzero count is "attention worth a human glance", not a failed chunk. StyleFlags int `json:"style_flags"` - CommittedUSD float64 `json:"committed_usd"` + CommittedUSD float64 `json:"committed_usd"` + // EstimatedRows / EstimatedUSD are the part of CommittedUSD that is an ESTIMATE — money booked for + // a call whose token count we never received, because the body would not decode, because our own + // deadline or a stopped run cut a delivered request short, or because the provider answered 2xx + // with no usage at all (estimatedSpend, cutcall.go). + // + // They ride HERE, beside the figure they qualify, because this is the JSON the platform reads and + // bills a user from. The engine has printed an estimated-cost legend for a human since pack-13; + // what nobody could read was a NUMBER, so the platform could only ever say «the run cost at least + // this much» (PD-441). Zero rows publish as zero rather than as absence: «none of it was + // estimated» is an answer, and omitting the field would make it indistinguishable from an engine + // too old to have one. + EstimatedRows int `json:"estimated_rows"` + EstimatedUSD float64 `json:"estimated_usd"` ReservedUSD float64 `json:"reserved_usd"` BookCeilingUSD float64 `json:"book_ceiling_usd,omitempty"` CeilingPct float64 `json:"ceiling_pct,omitempty"` // 100·(committed+reserved)/book_ceiling @@ -408,22 +421,38 @@ var flagSeverity = map[FlagReason]int{ FlagSanitizerDefect: 2, FlagLoopDegenerate: 3, - FlagDecodeError: 4, - FlagGlossaryMiss: 5, - FlagLength: 6, - FlagEmpty: 6, - FlagUpstreamNotOK: 6, - // A cosmetic leak the sanitizer STRIPPED and exported (D35.4a): the chunk shipped cleaned, so it is - // the least alarming flag — an "auto-cleaned, glance to verify" signal, ranked below a budget symptom - // (the chunk is not lost; a human need only spot-check the auto-clean). + // Paid, and nothing usable came back. `decode_error` is a 2xx whose body would not parse; + // `attempt_timeout` is a call OUR deadline cut while the provider was still generating it. They rank + // together because they are the same thing to a reader — the chunk is lost and the money is spent — + // and because neither is a verdict about the TEXT: both say the transport or its deadline needs + // fixing, and both are re-driveable once it is. A lost connection is NOT a third member: it never + // reaches a disposition at all (see disposition.go), so a rank for it would be a rank nothing wears. + FlagDecodeError: 4, + FlagAttemptTimeout: 4, + + FlagGlossaryMiss: 5, + FlagLength: 6, + FlagEmpty: 6, + FlagUpstreamNotOK: 6, + + // A cosmetic leak the sanitizer STRIPPED and exported (D35.4a): the chunk shipped cleaned — an + // "auto-cleaned, glance to verify" signal, ranked below a budget symptom (the chunk is not lost). FlagSanitizerStripped: 7, + + // The mildest mark there is, and the only one the engine removes by itself: a person stopped the run + // over a call that had already gone out, so the position is paid for and NOT done — and the next + // resume re-does it on the same budget. It ranks below the auto-clean because that one is a durable + // property of shipped text a human should look at, while this is a state the next run erases; a + // passport that reported «cancelled» as a chapter's worst problem would hide the durable finding + // behind a transient one. + FlagCancelled: 8, } // severityUnknown is where a reason this build has never heard of lands — a row written by an older // schema, or junk. Last on purpose: an unrecognised string must not out-rank a diagnosis the engine // actually made. It is NOT a resting place for new flags; the exhaustiveness test is what keeps it empty. -const severityUnknown = 8 +const severityUnknown = 9 func flagReasonSeverity(reason string) int { if s, ok := flagSeverity[FlagReason(reason)]; ok { @@ -536,6 +565,17 @@ func resolveChunkState(rows []store.ChunkStatus, stagesTotal int) chunkStateReso case string(DispOK): ok++ case string(DispFlagged): + // ⛔ A ROW THAT IS NOT AN ANSWER DECIDES NOTHING. `cancelled` records a position the engine was + // stopped over: the work was never done, the resume re-does it and pays again. Reading it as a + // decided unit puts it in the extrapolation's DENOMINATOR (rebill.go's projectBookUSD takes + // every done-or-flagged unit as processed), so one stop makes the book look further along and + // cheaper than it is — and the re-bill consent threshold, min($0.50, 5% × projected), shrinks + // with it. Before cut calls were settled here a stop left NO row at all and the denominator was + // honest; the mark must not cost that. Same question as the resume gate's, asked through the + // same predicate. + if !resolvedForResume(&cs) { + continue + } // The FIRST flagged stage decides the unit; keep its reason (the rows arrive in member/stage order // with the leader bucket first, so first-wins aligns status with translate/export, which both take // the first flag — a member drop's reason, or the edit's own when the edit flagged). Without the @@ -861,6 +901,15 @@ func (r *Runner) Status(ctx context.Context) (*StatusReport, error) { return nil, err } rep.CommittedUSD, rep.ReservedUSD = committed, reserved + // How much of that committed figure is an estimate. A failed read DEGRADES loudly — the pair stays + // zero and the reason is logged — rather than failing the whole read-only projection; but it is + // logged, because «0 estimated» read off a failure would be the most expensive silence on this + // surface: it says the committed figure is fully measured when nobody looked. + if usage, uerr := r.Store.CheckpointUsageForBook(r.Book.BookID); uerr != nil { + r.Log.WarnContext(ctx, "status: the estimated share of the committed spend could not be read; it is UNKNOWN, not zero", "err", uerr) + } else { + rep.EstimatedRows, rep.EstimatedUSD = estimatedSpend(usage, committed) + } // The ceiling IN FORCE (row 145): a read path never carries a run-scoped override, so this is the // book's own number there — but reading it through the single definition means status can never quote // a ceiling the ledger is not admitting against. diff --git a/backend/internal/pipeline/terminologist.go b/backend/internal/pipeline/terminologist.go index 3a74384f..a54c8984 100644 --- a/backend/internal/pipeline/terminologist.go +++ b/backend/internal/pipeline/terminologist.go @@ -476,7 +476,13 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te // here twice — on "the book has paid", which every ordinary resume of a filtered book satisfies, and on // a probe taken after the passes, which a freshly-paid first run satisfies trivially. if res.BankSettled > 0 && run.fresh && paidBefore { - r.Log.WarnContext(ctx, "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — those earlier calls were made under a batch composition the already-banked filter does not reproduce, so their checkpoints could not be found. This is a ONE-TIME cost; every later run replays for $0", + // ⚠ THE CAUSE IS NAMED AS A PAIR, because the code cannot tell which of the two happened and a + // message that picks one is wrong half the time. The old wording asserted the composition change + // alone; a book whose only earlier bank row was BURNED — money settled, no result, unreplayable — + // gets the same warning with a reason that is simply false for it, and a message lying about its + // own cause is what D39.93 п.2 forbids. Both branches leave the operator with the same action, so + // naming both costs nothing and claiming one costs the truth. + r.Log.WarnContext(ctx, "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — the earlier calls' checkpoints could not be reused, either because the batch composition changed (the already-banked filter does not reproduce it) or because they recorded money with no result and cannot be replayed. This is a ONE-TIME cost; every later run replays for $0", logKeyReconsolidated, true, "book", r.Book.BookID, "skipped", res.BankSettled, "of_candidates", res.Candidates, "cost_usd", fmt.Sprintf("%.6f", run.costUSD)) } @@ -484,8 +490,8 @@ func (r *Runner) runTerminologist(ctx context.Context, snapID string, cands []te out := map[string]string{} res.Conf = map[string]int{} for i, b := range batches { - if i >= run.attempted { - break // the budget or a ceiling stopped the pass here; runBankRoleBatches already said so + if !run.ran[i] { + continue // this batch was never called; runBankRoleBatches already said why } if run.texts[i] == "" { // An EMPTY completion on a call that was actually made. It used to `continue` in silence, which is @@ -664,8 +670,8 @@ func (r *Runner) runClassifier(ctx context.Context, snapID string, cands []termi types, genders = map[string]string{}, map[string]string{} bad, badGender, noGender := 0, 0, 0 for i, b := range batches { - if i >= run.attempted { - break // the budget stopped the pass here, and it already said so + if !run.ran[i] { + continue // this batch was never called, and the pass already said why } if run.texts[i] == "" { // The same silence the render phase carried: a paid classify batch that returned NOTHING left every @@ -781,10 +787,15 @@ func (r *Runner) bankCallEstimateUSD(st config.Stage, msgs []llm.Message) float6 // request-hash axis — the SAME identity runBankAttempt will address (attemptRequest), never a hand-rebuilt // copy of it: this probe gates the role sub-budget, so an identity that drifts from the attempt's turns the // gate off (see attemptRequest). +// +// ⛔ A BURNED CHECKPOINT IS NOT A PAID BATCH. One that records money and no result cannot be replayed, +// so runAttempt walks past it and buys the batch again — and a probe answering «already paid» would let +// that purchase escape the role sub-budget entirely, which is the one thing this gate exists to bound. +// The key can only hold such a row because cut calls are now settled there; before that, «a checkpoint +// exists» and «this batch is done» were the same statement. func (r *Runner) bankCheckpointExists(st config.Stage, snapID string, ch chunk.Chunk, msgs []llm.Message) (bool, error) { _, maxTokens := r.bankCallBudget(st.Model, msgs) - cp, err := r.Store.GetCheckpoint(RequestHash(r.attemptRequest(st, st.Model, snapID, ch, 0, maxTokens, msgs))) - return cp != nil, err + return r.paidAfterBurns(st, st.Model, snapID, ch, 0, maxTokens, msgs) } // runBankAttempt performs ONE bank-role call on the shared money path: reserve → call → settle+checkpoint, @@ -869,10 +880,15 @@ type bankRolePlan struct { // budget cut or that failed soft), plus the cost accounting for the report. type bankRoleRun struct { texts []string - // attempted is how many batches the pass actually reached before a budget ceiling or a soft denial stopped - // it. Without it "" is ambiguous — an EMPTY completion the run paid for and a batch never called look the - // same — and the caller cannot warn about the first without crying wolf about the second. - attempted int + // ran says, PER BATCH, whether the pass actually reached it. Without it "" is ambiguous — an EMPTY + // completion the run paid for and a batch never called look the same — and the caller cannot warn + // about the first without crying wolf about the second. + // + // ⛔ IT CANNOT BE A COUNT. Admission is not a prefix: an already-paid batch costs nothing and is + // admitted whatever the budget says, so a batch the budget refuses can sit BEFORE batches that are + // free to serve. A count answers «how many from the start», which silently drops every paid batch + // behind the first unaffordable one — their replay was $0 and their result was already bought. + ran []bool estimateUSD float64 costUSD float64 cumUSD float64 @@ -889,7 +905,7 @@ type bankRoleRun struct { // a budget ceiling or a soft denial stops the pass and leaves the remaining terms untouched — the run never // aborts on this optional step. logKind names the phase (render|classify) in the logs. func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan bankRolePlan, batches [][]terminology.Candidate, logKind string) (bankRoleRun, error) { - run := bankRoleRun{texts: make([]string, len(batches))} + run := bankRoleRun{texts: make([]string, len(batches)), ran: make([]bool, len(batches))} // ONE stage for the whole pass — the estimate, the checkpoint probe and the attempt all read it, so the // three can never be sized against different knobs (the estimate is the reservation's own upper bound). st := r.bankStage(plan.role) @@ -933,17 +949,22 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban // // So: decide up front, say what was cut, and run only that. Already-paid batches cost nothing and are // admitted regardless — holding them back would save no money and lose their result. + // ⛔ ADMISSION IS PER BATCH, NOT A PREFIX. An already-paid batch costs nothing, so refusing one + // unaffordable batch must not take the paid batches BEHIND it: their replay is $0 and their result is + // already bought, and dropping them buys nothing while losing a consolidated bank. The loop used to + // `break` here and hand the consumer a prefix bound, which made the comment above («admitted + // regardless») false for every paid batch that happened to sit after a refused one. fits, plannedUSD := 0, 0.0 probe := spent - paidBatch := make([]bool, len(batches)) + admit := make([]bool, len(batches)) for i := range batches { ch := chunk.Chunk{Chapter: 0, ChunkIdx: i} paid, perr := r.bankCheckpointExists(st, snapID, ch, msgsPer[i]) if perr != nil { return run, perr } - paidBatch[i] = paid if paid { + admit[i] = true fits++ continue } @@ -951,8 +972,9 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban // the conservative side and the config number is the bound it looks like. want := r.bankCallEstimateUSD(st, msgsPer[i]) if probe+want > plan.budgetUSD { - break + continue // this one cannot be afforded; the ones after it may still be free } + admit[i] = true probe += want plannedUSD += want fits++ @@ -967,7 +989,10 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban "estimate_usd", fmt.Sprintf("%.6f", run.estimateUSD)) } run.planned, run.dropped = len(batches), len(batches)-fits - for i := 0; i < fits; i++ { + for i := range batches { + if !admit[i] { + continue + } // The synthetic chunk addresses the batch: chapter 0 is the BOOK level (no real chapter is 0), and the // batch ordinal is the chunk index, so two batches can never collide on one checkpoint. ch := chunk.Chunk{Chapter: 0, ChunkIdx: i} @@ -986,7 +1011,7 @@ func (r *Runner) runBankRoleBatches(ctx context.Context, snapID string, plan ban run.fresh = run.fresh || att.freshCall spent += att.runCost run.texts[i] = att.text - run.attempted = i + 1 + run.ran[i] = true } return run, nil } diff --git a/backend/internal/pipeline/testdata/operator-messages.txt b/backend/internal/pipeline/testdata/operator-messages.txt index c262e760..af0fab8c 100644 --- a/backend/internal/pipeline/testdata/operator-messages.txt +++ b/backend/internal/pipeline/testdata/operator-messages.txt @@ -1,11 +1,11 @@ +# +# stands for a message built at run time — the site is pinned, the words cannot be. # The operator messages internal/pipeline emits without stopping the run — one line per call site, # `filefunctionquoted message`, sorted. Pinned by TestEveryOperatorMessageIsCatalogued, whose # comment holds the boundary (what is covered, what is deliberately not, and why a catalogue rather than # more substring asserts). -# -# ⚠ THIS FILE IS NOT REGENERATED. A wording change is meant to arrive here as a one-line diff a reviewer # reads against the code it now describes: paste the line the failure prints, and keep the file sorted. -# stands for a message built at run time — the site is pinned, the words cannot be. +# ⚠ THIS FILE IS NOT REGENERATED. A wording change is meant to arrive here as a one-line diff a reviewer bankexport.go exportBank "bank export: could not marshal the bank; the export artifact was NOT refreshed and may be stale" bankexport.go exportBank "bank export: could not read the bank; the export artifact was NOT refreshed and may be stale" bankexport.go exportBank "bank export: could not write the export artifact; it was NOT refreshed and may be stale" @@ -20,6 +20,9 @@ bookbuild.go staleUnits "build: the stale check could not be made for every ship bookbuild.go staleUnits "build: the stale check could not run; whether the source moved under the shipped rows is UNKNOWN (reported as unknown, not as none)" bookrun.go translateBook "the run ended before its VOLUME grant was used up — the stop below is NOT the volume ceiling" chunkrun.go reportEvicted "memory: the injection token budget DROPPED bank rows before the model saw them — these terms had no canon on the wire for those units" +cutcall.go recordCancelledStage "could not mark the stopped position; its money is recorded but the chunk will read as never started" +cutcall.go recordCancelledStage "the run was stopped over a call that had already gone out; the position is marked cancelled and the resume re-does it on the same budget" +cutcall.go settleCutCall "we cut a delivered call; the reservation estimate is charged as an ESTIMATE only when the provider had acknowledged it with a reply" escalation.go maybeEscalate "escalation hop denied by a USD ceiling; keeping the primary flag" escalation.go maybeEscalate "stage escalated to a fallback model" events.go beginRunEvents "could not read the stored dispositions for the run-event counters; this run publishes no progress (the run continues; resync channel: `tmctl status --json`)" @@ -92,6 +95,7 @@ status.go Status "config-drift not checked: the bank could not be folded, so dri status.go Status "config-drift not checked: the rows carry more than one snapshot within a wave (snapshot_drift), so config drift is UNKNOWN, not none" status.go Status "re-bill projection failed; the re-payment cost of the drift is unknown (reported as unknown, not as none)" status.go Status "status: no bank could be materialized; the unsigned-term count is unknown, not zero" +status.go Status "status: the estimated share of the committed spend could not be read; it is UNKNOWN, not zero" status.go Status "the bank fold refused, so the projections below are computed against the glossary the LAST run stored — they are a fact about the past, not a projection of the next run" terminologist.go glossaryRows "terminology: could not read the bank — whatever this call feeds goes silent (the canon anchor, the bank conflict check, or both), and its zero then means «not asked» rather than «nothing found»" terminologist.go loadTargetScript "the banknote channel is on but no gates.terminology.target_script is declared: draft-side proposals are NOT screened for the answer language, so a rendering in another script can enter the signature map and the auto-bank" @@ -112,7 +116,7 @@ terminologist.go runTerminologist "terminology: some families were NOT co-batche terminologist.go runTerminologist "terminology: the TYPE classifier ended short — its OWN budget cut a pass, so some candidates keep the draft heuristic type; the bank's renderings are unaffected and this alone does not make the bank partially consolidated" terminologist.go runTerminologist "terminology: the model answered in another script; those lines are REFUSED (the terms stay unconsolidated) — a book with few signed rows gives the model no target-language anchor" terminologist.go runTerminologist "terminology: this bank is PARTIALLY consolidated — a budget cut the RENDER pass, so some terms were never offered to the role at all; `unanswered` below counts them together with terms the role saw and did not answer" -terminologist.go runTerminologist "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — those earlier calls were made under a batch composition the already-banked filter does not reproduce, so their checkpoints could not be found. This is a ONE-TIME cost; every later run replays for $0" +terminologist.go runTerminologist "terminology: this book had already paid for the bank role BEFORE this run, and this run bought the pass again — the earlier calls' checkpoints could not be reused, either because the batch composition changed (the already-banked filter does not reproduce it) or because they recorded money with no result and cannot be replayed. This is a ONE-TIME cost; every later run replays for $0" volume.go deliveredUnits volume.go planVolume "a VOLUME ceiling on a book that MINES its bank: each purchase drafts more, so the bank-mining stop writes a larger auto-bank, and the next purchase moves the edit-wave snapshot the previous purchase's edit jobs are pinned to. Expect that run to need --resnapshot and to re-pay the units the new terms actually touch (the re-payment consent gate still bounds it)" volume.go rescopeEditWave "the bank moved between planning and the edit wave, so units judged FREE are no longer free: they have been re-judged against the snapshot the edit wave actually uses" diff --git a/backend/internal/pipeline/volume.go b/backend/internal/pipeline/volume.go index 6a2587dd..5cb478a1 100644 --- a/backend/internal/pipeline/volume.go +++ b/backend/internal/pipeline/volume.go @@ -818,6 +818,18 @@ func (r *Runner) rowsResumeFree(rows []store.ChunkStatus, draftNames, editNames if cs.SnapshotID != cur && !r.repinnable(cs.SnapshotID, cur, w) { return false } + // ⛔ THE SAME QUESTION runStage ASKS, ASKED THROUGH THE SAME PREDICATE. A `cancelled` row records + // a stop, not an answer: the resume re-does that call, for money. Read as free, a unit whose last + // row is one needed no slot — so a grant of one unit paid for two and the volume report, which + // speaks only when something was carried, said nothing at all. + // + // ⚠ AND IT IS ASKED LAST, AFTER the switch above has dropped the stages this pipeline no longer + // runs. Asked first — where it was written — it made a unit PAID for a cancelled row of a stage + // that will never be called again, which is the same error in the opposite direction: an operator + // who edits the pipeline over a stopped run would spend volume slots on nothing. + if !resolvedForResume(&cs) { + return false + } } return true } diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 68661d12..df05c205 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -476,7 +476,7 @@ post-header и оплачены. Три ряда девятки «до заго | поля утверждены двусторонне | да — `WhitespaceOnly`, `Delivered`, `estimated`, оба условных сообщения | | тест-тождество §4.4 пинит формулу, не константу | да — запрещённый набор ВЫВОДИТСЯ из формулы: «banned set [238 239 240 899 900 901] → 0 hit(s); control → 2 hit(s)» | | числа сняты ПОСЛЕ последней правки, со счётом скипов | да — секция «ЧИСЛА» выше | -| таблица мутаций ПОЛНАЯ, выжившие названы поимённо | да — 114 посадок, **выживших НЕТ**, свип 0 из 325. ⚠ Выжившие БЫЛИ и названы поимённо: `CUTCALL-the-redo-doubles-the-budget` (пин перестал держать верное свойство), `CUTBANK-a-burned-batch-counts-as-paid` и `CUTDEADLINE-the-h2-write-bound-is-removed` (оба — мои вакуумные пины). Все три закрыты и пере-проверены поштучной посадкой | +| таблица мутаций ПОЛНАЯ, выжившие названы поимённо | да — 114 посадок, **выживших НЕТ**, свип 0 из 325. ⚠ Выжившие БЫЛИ и названы поимённо: `CUTCALL-the-redo-doubles-the-budget` (пин перестал держать верное свойство), `CUTBANK-a-burned-batch-counts-as-paid` и `CUTDEADLINE-the-h2-write-bound-is-removed` (оба — мои вакуумные пины). Все три закрыты и пере-проверены поштучной посадкой. ⟨Первых двух в дереве БОЛЬШЕ НЕТ: `CUTDEADLINE-…` снята вместе с бондом 10.09, `CUTBANK-…` — когда банковый зонд перешёл на общее определение. Имена — хроника⟩ | | дифф `^func Test` снят ИСПОЛНЕНИЕМ, прибор назван | да — секция «Дифф `^func Test`» | | всё живое — в ДЕРЕВЕ, а не в письме | да — 23 пути, все в зоне (`backend/` + своя секция `docs/PROGRESS.md`); вне зоны 0; индекс пуст; `books` не тронуты | @@ -661,23 +661,349 @@ retry-loop 6→3 · bank-and-counters 6→6 · transport-and-config 9→7 Восстановленный файл — снова в скратчпаде, то есть снова в tmpfs; **воспроизводится из журнала за один прогон, отдельного хранения не требует.** -#### ⚠ ДИСПОЗИЦИИ ОРКЕСТРАТОРА `a8` — НОСИТЕЛЯ В РЕПОЗИТОРИИ НЕ ИМЕЮТ +#### ДИСПОЗИЦИИ ОРКЕСТРАТОРА, ЖИВШИЕ ТОЛЬКО В КАНАЛЕ — ✅ ЗАКРЫТО В ТОТ ЖЕ ДЕНЬ, `D39.231` -Оркестратор смены №23 (`textmachine-a8`) перед смертью сессии закрыл три пинга из четырёх. Его слово -жило в межсессионных сообщениях, то есть нигде: **сессия мертва, канала нет.** Записываю сюда дословно по -смыслу, чтобы следующий оркестратор не решал это заново, — и отмечаю, что ратификации в журнале решений у -этих трёх НЕТ. +Оркестратор смены №23 закрыл три пинга из четырёх **межсессионными сообщениями**. Его слово жило в +канале, то есть нигде: канал лежит в tmpfs и умер вместе с окружением. Записала сюда дословно по смыслу, +чтобы следующий оркестратор не решал это заново, и отдельно объявила, что носителя в репозитории у этих +трёх НЕТ. + +⛔ **ЭТО ЗАКРЫТО В ТОТ ЖЕ ДЕНЬ — `D39.231`, коммит `aad50a7`, и там же усилена НОРМА:** диспозиция +оркестратора по пингу сессии ратифицируется ИЛИ получает строку носителя **тем же движением, каким +отправляется ответ**; ответ в канале — уведомление о решении, а не само решение. Ниже три пункта +оставлены как были записаны, к каждому добавлен носитель. + +⚠ **И моя собственная ошибка вывода, снимаю сама:** я написала «сессия мертва» и «пункты без владельца», +прочитав исчезновение ИМЕНИ `textmachine-a8` из `ListAgents` как смерть смены. Умерло имя — рестарт +окружения его не переживает; **смена №23 и её контекст целы**, тот же оркестратор работает под именем +`textmachine-11`. Отсутствие в списке имён не есть отсутствие сессии — тот же класс, что «ноль строк в +выдаче» против «прибор не спросил существующее». 1. **h2-write-бонд (`transport-and-config#6`, `#9`) — УДАЛЯТЬ.** Принял мой довод против собственного заказа: механизм рвёт ВСЮ h2-связь, а не застрявший стрим, и для тел, которые движок реально шлёт, выстрелить не может (замер разводит обещание и предмет на два порядка). Сослался на `D39.216` — «где форма не тянет, её не подпирают»; носитель класса остаётся строкой бэклога **373**. + ✅ Ратифицировано `D39.231` п.2. 2. **Вердикт ГЛАВЫ по остановленной позиции (`критик#3`) — забрал СЕБЕ**, продуктовый вопрос владельцу. + ✅ Заведён строкой бэклога **374** (`D39.231` п.3) — больше не зависит от того, жива ли чья-то сессия. 3. **`memberDrops` читает `cancelled` как выпавшего члена (`критик#4`) — забрал СЕБЕ**, семантика экспорта. + ✅ Заведён строкой бэклога **375** (`D39.231` п.3). -⇒ **Пункты 2 и 3 сейчас без владельца:** сессия, обещавшая отнести их владельцу, не существует. +⚠ Обе строки, 374 и 375, **исчезнут сами, если денежную половину пака урежут**: они следствия класса +`cancelled`, а не самостоятельные дефекты. -#### Вопрос владельцу и оркестратору — один, и он прежний +#### 10.09 — h2-WRITE-БОНД УДАЛЁН (ратификация `D39.231` п.2). Отдельное движение, наряда НЕ касается + +Оркестратор велел снять бонд СЕЙЧАС, не дожидаясь вердикта о останове: предмет независим (бонд — про +таймаут записи на транспорте, останов — про волю человека), ратификация уже есть, а удаление уменьшает +поверхность, а не растит. + +**Что ушло вместе с ним — полный список носителей, прибор назван.** +Греп `WriteByteTimeout|h2WriteByteTimeout` по `backend/` (359 go-файлов прочтено) и по `docs/` (120 .md, +без `prompts`/`reports`) дал ЧЕТЫРЕ носителя в коде и ни одного скрытого: + +| носитель | что сделано | +|---|---| +| `internal/llm/httpllm.go` — константа `h2WriteByteTimeout` и её 15-строчный комментарий | снята | +| `internal/llm/httpllm.go` — `h2.WriteByteTimeout = …` в `tuneHTTP2` | снята; доккомментарий «installs the three liveness bounds» → «the keepalive pair» | +| `internal/llm/attemptcut_test.go` — пин `TestTheWriteSideKeepaliveIsDerivedFromTheReadSideOne` ⟨имени в дереве нет: это ПРЕЖНЕЕ имя⟩ | **переписан, не удалён** — см. ниже | +| `cmd/tmmutate/mutations.json` — `CUTDEADLINE-the-h2-write-bound-is-removed` | снята; заведена замена | + +⚠ **Пин держал НЕ ТОЛЬКО бонд, и это единственное место, где удаление было не механическим.** Из четырёх +его утверждений три — про бонд (выведенность из read-side пары, ненулевость, установленность на +транспорте) и умирают вместе с предметом. Четвёртое — что `ReadIdleTimeout`/`PingTimeout` стоят на +транспорте, который клиент СТРОИТ, — самостоятельная гарантия, и она бы утекла при простом удалении +файла-теста. ⇒ тест переписан под неё: `TestTheKeepalivePairSitsOnTheTransportTheCloudClientBuilds`, +контроль на ненулевые константы поднят в начало (иначе сравнение нуля с нулём проходит на голом +транспорте). **Объявляю по `D39.183`:** правка теста вызвана ЗАКАЗАННОЙ сменой поведения, а не желанием +зелени; гарантия про бонд снята вместе с бондом, гарантия про keepalive-пару сохранена под новым именем. + +⛔ **ПОПРАВКА 10.09, вечер: абзац выше был НЕВЕРЕН в момент написания, снимаю сама.** Гарантия НЕ была +сохранена. Тело нового пина брало СВЕЖИЙ клон (`tuneHTTP2(http.DefaultTransport…Clone())`) и утверждало +про него — то есть про транспорт, которым никто не пользуется. Снятие `tuneHTTP2` из боевой конструкции +переживало и пин, и весь пакет; замер круга: 0 из 220 тест-файлов строили облачный клиент и читали его +h2. Это ТОТ ЖЕ дефект, который наряд записал про СТАРЫЙ пин (`transport-and-config#1`), воспроизведённый +мной в замене. Починено: `keepAliveHTTPClient` разделён на себя и `buildCloudClient`, отдающую клиент И +транспорт, на котором поставила границы; пин читает её. Мутация +`CUTKEEPALIVE-the-cloud-client-is-not-tuned-at-all` — **RED** текстом «the cloud client could not be +configured for h2 — this test measured nothing». + +⚠ **Каталог: 325 → 325.** Снятая запись была ЕДИНСТВЕННОЙ, сажавшей мутацию в `tuneHTTP2`; простое +удаление оставило бы пере-писанный пин без носителя в каталоге. Заведена +`CUTKEEPALIVE-the-read-idle-bound-is-not-installed` (`battery: true`), которая сносит установку +`ReadIdleTimeout`. **Это добавление, а не удаление — если оркестратор считает его выходом за рамки +движения, оно снимается одной строкой.** + +**ПРЕДЪЯВЛЕНИЕ МУТАЦИЕЙ — три замера на КОПИИ дерева** (`rsync` без `.env*` и `bin/`; контроль копии: +файлов `.env*` внутри **0**, go-файлов скопировано **359**): + +1. **Дерево БЕЗ бонда** (как сейчас) → `./internal/llm/` **ok**. Удаление ничего не уронило. +2. **Бонд ВОЗВРАЩЁН в копию** (константа + установка) → `./internal/llm/` **ok**. Ничего не пинит и его + отсутствие. ⇒ бонд был для батареи НЕВИДИМ в обе стороны: он не держал ничего. +3. ⛔ **КОНТРОЛЬ, без которого два зелёных выше означали бы «прибор не спросил»:** посадка новой мутации + тем же инструментом → **RED**, и засчитана ПО ТЕКСТУ, а не по цвету: + `attemptcut_test.go:852: the read-idle bound is not on the transport: got 0s want 15s`. + Тот же пакет, тот же прогон, тот же пин — краснеть он умеет. + +**Строка бэклога 373 остаётся** носителем класса «наш дедлайн инертен на h2» (`D39.231` п.2 прямо это +говорит): удалён МЕХАНИЗМ, который класс не закрывал, а не сам класс. + +⚠ Прежние записи этой секции про `WriteByteTimeout` (числа §4.4 и находка `transport-and-config#1`) +оставлены как были: они верны на момент, когда писались, и заменяются этой строкой, а не переписыванием. + +#### 10.09 — ДВА ПРЕДГЕЙТА НАУЧЕНЫ ЧИТАТЬ ОЖОГ. Заказ оркестратора, отдельное движение + +⚠ **Поправка к заказу, и она о весе наряда, а не о работе.** Оркестратор передал это как находку +консилиума со словами «этого не видела ни одна линза наряда». Замер: наряд несёт её ТРЕМЯ позициями, +все P0 — `burn-walk#2` (ремонт), `burn-walk#3` (хоп) и их дубль `bank-and-counters#3`, где прямо +записано «два ДРУГИХ зонда про ожог не знают». То есть независимый вердикт ЧТЕНИЕМ пере-открыл то, что +линза нашла исполнением. Для решения о судьбе остальных 29 это улика: наряд и консилиум сошлись на +одном месте, придя к нему разными путями. + +**Предмет.** Контракт «чекпойнт есть ⇒ отвечено и оплачено» спрашивают ТРИ предгейта до воронки. +Сожжённый ключ — деньги без результата: воронка его воспроизвести не может, она шагает мимо и покупает +работу заново. Предгейт, читающий такую строку как «уже оплачено», поэтому НЕ пропускает покупку — он её +ВЫПУСКАЕТ, мимо той единственной проверки, которую сам же и охраняет. + +| предгейт | было | стало | +|---|---|---| +| `terminologist.go:793` (банк) | `cp != nil && !burnedByCut(cp)` | учён ещё в паке | +| `repair.go:374` | `cp != nil` | `cp != nil && !burnedByCut(cp)` | +| `escalation.go:152` | `fbExists != nil` | `fbExists != nil && !burnedByCut(fbExists)` | + +⛔ **ВОСПРОИЗВЕДЕНО ИСПОЛНЕНИЕМ ДО ПОЧИНКИ — следствие, а не совпадение строк.** Оркестратор передал +следствие как выведенное чтением и просил проверить. Обе фикстуры написаны первыми и на непочиненном +дереве упали ТЕКСТОМ ПРО ДЕНЬГИ: +- ремонт: `want 2 repair calls, got 3 — one of them was bought outside gates.repair.budget_usd`; +- эскалация: `want 2 hop calls, got 3` (мимо `escalation.budget_usd` И мимо `escMu`). + +**Четыре фикстуры, парами.** Каждая правка закреплена ДВУМЯ: одна держит свойство (сожжённый ключ не +покупает), вторая — его противоположность (настоящий ключ по-прежнему воспроизводится бесплатно на +исчерпанном бюджете). Без второй предикат, отвечающий «не оплачено» на всё, прошёл бы первую и тихо +выключил контракт бесплатного резюма, который `escalation.go` обещает в своём же доккомментарии. + +⚠ **Как строится ожог — единственное тонкое место, и первая редакция была ФЛЕЙКОВОЙ.** Наш денежный +предикат книжит оборванный вызов только после подтверждения провайдером, поэтому сервер, который просто +держит молчащее соединение, даёт строку на $0 — настоящую, но не ту. Первая редакция слала заголовки, +флашила и внешним сигналом отменяла прогон. Замер: **2 прогона из 5** отменялись раньше, чем клиент +разобрал заголовки, вызов читался как неотвеченный и книжился в ноль — то есть фикстура была бы зелёной, +не измеряя ничего. Заменено на обрыв, которым правит СЕРВЕР (`connection_lost` жжёт ключ ровно так же и +внешней синхронизации не требует): 8 прогонов из 8 одинаковы. + +**Мутации — по одной на правку, засчитаны ПО ТЕКСТУ:** +`CUTBURN-the-repair-pregate-forgets-the-burn` и `CUTBURN-the-escalation-pregate-forgets-the-burn`, +⟨обе ПОЗЖЕ СНЯТЫ из каталога тем же днём: когда зонд стал ОДНИМ определением, их предмет — по-строчная +правка на каждой площадке — перестал существовать, и их место заняли пять записей семьи. Имена оставлены +здесь как хроника, в дереве их нет⟩ +обе `battery: true`, обе **RED** своим тестом, и текст падения называет свежий вызов, а не транспорт. +Каталог 325 → 327, подмножество battery 114 → 116. + +**Знаменатель класса — прибор спросил существующее.** `GetCheckpoint` живёт в **10** местах живого кода +(прочитано 360 go-файлов): 4 — вопрос «оплачено ли» (три предгейта + воронка `stagerun.go:492`), все +учены; 6 читают `cs.FinalHash`, то есть указатель на АВТОРИТЕТНЫЙ ответ, а сожжённая строка им стать не +может — путь обрыва пишет `FinalHash: ""` и флагует позицию. ⇒ **четвёртого необученного читателя нет**, +строка бэклога по правилу остановки не понадобилась. + +⚠ Правило остановки соблюдено: движение — два предиката и их пины, больше ничего. Наряд не начат. + +#### ⛔ 10.09 — АДВЕРСАРИАЛЬНЫЙ КРУГ ПО СВОЕМУ ЖЕ ДВИЖЕНИЮ: МОЯ ПОЧИНКА БЫЛА РАЗМЕНОМ + +Круг: 4 направленные линзы, у каждой свой опровергатель, плюс критик полноты. **Заявлено 20, пережило +опровержение 10, плюс 2 у критика.** Главная находка — про починку, которую я сдала часом раньше как +сделанную, при зелёной батарее и красных мутациях. + +⛔ **ЧТО БЫЛО НЕ ТАК.** Оба зонда — и мои два, и банковский, с которого я их писала, — спрашивали +**фиксированный индекс попытки**. Воронка на ожоге не останавливается: она уходит на следующий индекс при +том же бюджете и покупает ТАМ. Значит после оборванного прогона позиция читается «attempt 0 сожжён, +attempt 1 ОПЛАЧЕН и отвечен». Зонд, смотрящий на стартовый ключ, видит ожог, отвечает «не оплачено» — и +вызывающий, не найдя бюджета, **выбрасывает уже купленный перевод**. Навсегда, на каждом резюме. + +**Доказано откатом ОДНОЙ строки на копии, не рассуждением:** + +| дерево | «ожог не покупает» | «оплаченное за ожогом реплеится» | +|---|---|---| +| ДО моей починки | **FAIL** | **PASS** | +| ПОСЛЕ моей починки | **PASS** | **FAIL** | + +⇒ два дефекта, один бит, противоположные стороны. **Третий раз за смену эта форма** (раньше — $0.80 на +отказах провайдера). + +**Починка правильной формы: чинить ВОПРОС, а не ответы.** Одно определение `paidAfterBurns` +(`internal/pipeline/cutcall.go:196`) шагает ожоги ровно как воронка и отвечает про тот ключ, который +воронка возьмёт. Три площадки зовут его: `escalation.go:141` · `repair.go:373` · `terminologist.go:792`. +Два вопроса, заданные одним способом, разойтись не могут — в этом смысл функции против трёх площадок, +каждая из которых «шагает правильно» (`D39.216` п.3б). + +⚠ **Третий член семьи был поражён тем же корнем, и это код ПАКА, а не сегодняшнего движения:** банковский +зонд, тот самый «образец». Идёт в наряд НОВОЙ позицией. + +**Семья предъявлена целиком — три члена, обе стороны бита, шесть пинов:** + +| член | «ожог не покупает» | «оплаченное за ожогом реплеится» | +|---|---|---| +| `escalation.go:141` | `TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget` | `TestAPaidHopBehindABurnedKeyStillReplaysFree` | +| `repair.go:373` | `TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget` | `TestAPaidRepairBehindABurnedKeyStillReplaysFree` | +| `terminologist.go:792` | `TestTheBankPaidProbeSeesThroughABurnedCheckpoint` | `TestTheBankProbeFindsThePaidBatchBehindABurnedKey` | + +**Каталог: 3 протухших якоря сняты, 6 заведено** (327 → 330, battery 116 → 119). Две на сам контракт +(`ignores-the-burn` краснит три «не покупает», `stops-at-the-first-key` — три «реплеится»), три на то, +что каждая площадка им пользуется, одна на тюнинг боевого клиента. Все **RED по тексту**. + +⚠ **Первая редакция двух записей каталога НИЧЕГО НЕ УТВЕРЖДАЛА:** список тестов через запятую там, где +`-run` берёт регексп. Поймал инструмент, дословно: «the run filter … matched no test — it was renamed or +removed, and this entry has been asserting nothing». Переписано альтернацией, пере-проверено посадкой. + +**Остальные находки круга — все пять починены в этом же движении:** +1. комментарий лока в `escalation.go` описывал прежний код и называл мёртвую переменную; +2. обе Real-фикстуры не утверждали СВОЮ ГЛАВНУЮ ПРЕМИССУ («на исчерпанном бюджете») — с холостым + exhaust-хелпером оставались зелёными, то есть держали не то, что обещали именем; +3. burned-фикстура ремонта при сломанной премиссе обвиняла ПРЕДГЕЙТ в выпуске платного вызова мимо + суб-бюджета — правый цвет, неправый текст, и следующая смена пошла бы чинить `repair.go`; +4. мёртвые поля `arrived`/`once` — остаток снятой флейковой схемы, с доккомментарием о несуществующем + назначении; ни `vet`, ни `gofmt` их не видят; +5. пин keepalive не читал боевой клиент — см. поправку выше по тексту. + +⚠ **И собственная ошибка замера, названная тут же:** проверяя пункт 2, я делала холостыми ОБА +exhaust-хелпера, но патч ремонтного не применился из-за экранирования — ремонтные фикстуры прошли +законно, и я чуть не записала «премисса не срабатывает». Пере-делала: все три падают текстом +«premise broken: the repair sub-budget is NOT exhausted (spent 0.001063, budget 1.000000)». + +⭐ **Что этот круг доказал про сам метод.** Знаменатель читателей был ВЕРЕН — их правда три. Дефект сидел +не в их числе, а в ФОРМЕ ВОПРОСА, одинаковой у всех трёх, и никакой счёт читателей его не ловит. Ловит +направленный второй читатель. ⇒ знаменатель закрывает одну ОСЬ, а не работу. + +#### 10.09 — НАРЯД ОТРАБОТАН ЦЕЛИКОМ: 36 позиций, 34 сделано, 2 пингом + +Слово владельца через оркестратора: доводить наряд своей рукой. Отработан целиком, по семьям, в порядке +«деньги → числа → строки». **Единственный носитель исходов — сам наряд** +(`docs/archive/reports/CUT_CALLS_DOFIX_WORK_ORDER_2026-09-08.md`, правится по зонному исключению в его +шапке): у каждой позиции поля `исход` и `предъявлено`, и сверяются они машинно, а не глазами. + +``` +позиций в наряде: 36 (P0=12 P1=12 P2=8 PING=4) +ИСХОД НЕ ИЗ ТРЁХ: 0 · «сделано» БЕЗ поля «предъявлено»: 0 +ЗАКРЫТО 34 из 36 · ИТОГ: все позиции имеют исход и предъявление +``` + +⛔ **Каждая семья закрыта ОДНИМ контрактом, а не пачкой правок** — это и есть ответ на «семь кругов не +сходились». Знаменатель каждого класса посчитан командой, а не памятью: + +| класс | контракт | закрыт | +|---|---|---| +| читатели денежного контракта | одно определение `paidAfterBurns` шагает ожоги ровно как воронка | **3 из 3** (`grep -c 'r.paidAfterBurns('` = 3) | +| выходы цепочки `retryLoop` | `moreOwed` копит, `chainError` выносит, решение по ДЕНЬГАМ, а не по типу | **4 из 4** (все четыре `return` идут через него) | +| носители `resolvedForResume` | один предикат на все вопросы «это уже ответ?» | **3 площадки + четвёртый спрашивающий через третью** (`projectBookUSD` → `resolveChunkState`) | +| инвариант строки чанка | строка сходится с суммой чекпойнтов СВОЕЙ позиции на ЛЮБОМ выходе | **4 фикстуры** держат `assertRowsMatchTheLedger` | + +**Что нашлось по ходу и чего в наряде не было:** +1. **Моя же починка предгейтов оказалась РАЗМЕНОМ** — доказано откатом одной строки: до неё «оплаченное + за ожогом реплеится» проходило, а «ожог не покупает» падало; после — ровно наоборот. Третий раз за + смену один бит в две стороны. +2. **Наряд ПРЕДСКАЗАЛ эту регрессию** позицией `bank-and-counters#2`: «зонд зашит на попытку 0, а обход + ожога идёт по возрастающим индексам… зонд должен спрашивать ту же ось, что и обход». Я чинила по + заказу письмом, не открыв наряд. +3. **Знаменатель, посчитанный по именам функций, — не знаменатель.** «Пять глаголов со словом + `Checkpoint`» превратилось в **12 функций**, когда прибор спросил ТАБЛИЦУ, а не словарь имён. +4. **Мой пин инварианта был флейковым: 2 красных из 8 на мутанте** — отмена обгоняла разбор заголовков, + обрыв выходил на $0, строка сходилась «ноль к нулю». Перестроен так, что деньги НЕИЗБЕЖНЫ. +5. **Две мутации СНАЧАЛА ВЫЖИЛИ** (ремонтная и счётная) — носителей не было вовсе, и без посадки я бы + этого не узнала. +6. **Снята СВОЯ недостижимая ветка** в накопителе цепочки и **чужая недостижимая константа** + `FlagConnectionLost` — обе выглядели стражами и не могли выстрелить. +7. **Шесть чужих якорей каталога** протухли от моих правок (один разорван моим же комментарием) — + пере-нацелены поштучно, каждый пере-проверен посадкой. Каталогизированный ВЫЖИВШИЙ с доводом «ветка + недостижима» не тронут: это улика, а не протухший якорь. + +**Заказанные смены поведения, объявляю отдельно (`D39.183`):** +- голден операторских сообщений обновлён ОДНОЙ строкой (126 → 126) — формулировка причины повторной + оплаты банка изменена заказанной правкой `круг8#1`; +- пин write-бонда заменён пином keepalive-пары — предмет удалён ратификацией `D39.231` п.2; +- поле `run.attempted` (счётчик) заменено на `run.ran []bool` — счётчик не может описать несплошное + множество приёма партий. + +#### ⛔ 10.09 — ОХОТНИК ОРКЕСТРАТОРА: ДВА ВЫЖИВШИХ МУТАНТА, ОБА НА ЛОЖНЫХ ПРЕДЪЯВЛЕНИЯХ + +Верификатор «вне карты» по сданной работе. Блокеров нет, батарея и каталог у него сошлись с моими. Но две +посадки пережили батарею, и обе — не новые предметы, а **утверждения о закрытии, которые не держались**. +Обе воспроизвела своим прогоном, прежде чем классифицировать. + +**1. Выход по отмене — ЛОЖНОЕ «предъявлено» позиции наряда.** Снятие `cancelledDuring` на ПЕРВОМ выходе +(`httpllm.go:199`) оставляло батарею зелёной, при том что близнец на `:222` краснел. Позиции +`money-predicate#1` и `bank-and-counters#4` утверждали «оба выхода идут через `chainError`» и называли +пин, где стоит проверка `errors.Is(err, context.Canceled)` именно про этот выход. + +⛔ **Механизм — класс, которого у нас не было: пин удовлетворялся ЧУЖОЙ уликой.** Фикстура гонит петлю +ЧЕРЕЗ ПРОВОД, а там стоп приходит по вызову в полёте — значит ошибка самой попытки уже родительски- +отменённый обрыв, несущий `context.Canceled` в поле `Parent`. Замер на мутанте: `isCanceled=true +cause=connection_lost` в **6 прогонах из 6**. То есть `errors.As` находил обрыв ПЕРВОЙ попытки, а +`errors.Is` — отмену внутри ВТОРОЙ. **Это не флейк, а детерминированная пустота: повторный прогон такое +не ловит.** ⇒ на мутанте спрашивать надо не «покраснело ли», а ЧТО ИМЕННО удовлетворяло утверждение. + +Починка: `TestTheStopExitStillReadsAsCancelledWhenTheAttemptDidNot` гонит `retryLoop` НАПРЯМУЮ — попытка +падает обычной 503, стоп приходит на её возврате, и опереться не на что, кроме самого выхода. Провод +такой порядок создать не может: окно между возвратом попытки и чтением контекста в несколько инструкций, +и фикстура, гоняющаяся за ним, мерила бы планировщик. Заодно закрыта связка «новый тип ↔ код выхода»: +`AttemptCutError` давал **0 хитов** в тестах `cmd/tmctl` при **26** вызовах отображения — заведён +`TestTheCutErrorTypeKeepsItsExitCode` (пять форм + контроль «потолок старше обрыва»). Мутации +`CUTCHAIN-the-stop-exit-drops-the-cancellation` и `CUTEXIT-a-stop-stops-mapping-to-five` — RED. + +**2. Вендорская пара, из которой считается КАЖДЫЙ дедлайн — ложное предъявление §4.4 ПАКА.** +`vendorHourlyTokenBudget` можно учетверить (128000 → 512000), и зелены и `internal/llm`, и +`internal/config`, и вся батарея. Причина: `vendorSeconds` (тест) берёт ожидание из ТЕХ ЖЕ +внутрипакетных констант, что и `deriveDeadline`, — порча двигает обе стороны, и тождество сходится. +Пин держал РАСПОЛОЖЕНИЕ формулы и не держал ЧИСЛА, а §4.4 требовала неформальной исполнимости. + +⚠ **Классификацию я сначала дала В СВОЮ ПОЛЬЗУ и тут же привела довод против себя.** Разбор всех 36 +полей «предъявлено» показал: пин деривации не назван ни одной позицией наряда ⇒ по букве правила +остановки это строка бэклога. Но цена денежная и ровно про предмет пака — вчетверо короче дедлайн +означает, что живые генерации становятся self-cut'ами, за которые движок теперь ПЛАТИТ, и пять +провайдеров из восьми сидят на этом дефолте. Оркестратор пере-провёл СВОЮ границу («опровергает любое +утверждение о закрытии, которое мы вот-вот ратифицируем, а не только позицию наряда») и вернул починку +в круг. + +Починка: `TestTheVendorsPublishedPairIsWhatTheVendorPublishes` цитирует вендорскую пару против источника +(`CalculateNonStreamingTimeout`: час на 128 000 токенов), с контролем «грант ровно в бюджет обязан +вывестись ровно в окно» — иначе константы были бы украшением рядом с деривацией, а не её источником. +⚠ Это НЕ нарушает `TestTheDeadlineTestQuotesNoDeadline`: тот банит ПРОИЗВОДНЫЕ секунды (его собственный +банлист печатается прогоном: `[238 239 240 899 900 901]`), а вендорская пара — исходные числа, то есть +единственное место, где арифметика касается внешнего мира. Мутации `CUTDEADLINE-the-vendor-budget-moves` +и `-window-moves` — RED. **И этикетка прибора исправлена тем же движением:** комментарий `vendorSeconds` +утверждал о себе «a re-derivation from the source numbers and not a copy of the code under test» — +теперь он говорит, что пинует РАСПОЛОЖЕНИЕ и не пинует числа, и называет, кто пинует их. + +#### 10.09 — ЧИСЛА ПОСЛЕ ВСЕХ ПОЧИНОК. Работа завершена, править не планирую + +Сняты ПОСЛЕ последней правки, по одному прогону за раз (параллельный запуск ронял машину по памяти): + +``` +make battery → MAKE-EXIT=0 · 19 ok · 0 FAIL · 4 «no test files» · 4 скипа, названы: + TestMinerFullBookParity · TestCorpusBankKeyConflicts + TestHelperEventsRun · TestHelperKillLoop (тот же список, что в baseline смены) +make mutations → MAKE-EXIT=0 · 149 посадок · 149 RED · 0 выживших + 0 NOTHING · 0 ROTTED · 0 INCONCLUSIVE · якорей протухших 0 из 360 +counts.py → литералы сходятся (8 проверок) +``` + +Каталог за пак: **276 → 360**, батарейное подмножество **65 → 149**. Дерево: 28 путей, вне зоны 0. + +**Самопроверка отчёта, механическая.** Из полей «предъявлено» наряда вынуто 25 имён тестов и 24 +идентификатора мутаций: **несуществующих ноль** (контроль: тестов в дереве 1369, записей в каталоге 360). +Все 24 — в батарейном подмножестве, с `run`-фильтром, и **все 24 покраснели** в финальном прогоне. + +⚠ **Тот же прибор нашёл в ЖУРНАЛЕ пять имён без предмета** — прежний пин write-бонда и четыре записи +каталога, снятые по ходу смены. Все пять были верны на момент записи, но читатель, грепнувший имя, +не нашёл бы его и решил, что отчёт лжёт. ⇒ каждое помечено как хроника прямо на месте, с указанием, +когда и почему предмет исчез. Непомеченных: 0. + +**Инцидент со своим деревом — назван и пере-проверен ДВУМЯ приборами.** `cd` в несозданный каталог +провалился, `set -e` не удержал, и мутация ушла в настоящее дерево; поймана следующей командой и +восстановлена. Пере-проверка не грепом по порче (у мутаций-усечений порча есть ПРЕФИКС цели и +присутствует всегда), а поиском пропавшей ЦЕЛИ: **370 правок каталога в 62 файлах, целей не на месте — +ноль**. Оркестратор снял то же число своим прибором независимо. Норма записана в `CLAUDE.md` +(`6ee6c61`): копия под мутацию защищается ПОСТРОЕНИЕМ — `test -f go.mod` плюс сверка `pwd` перед любой +правкой, копия вне общего скретчпада. + +#### Вопрос владельцу и оркестратору — один, и он ЖДЁТ вердикта о мягком останове Чинить ли 31 позицию наряда моей рукой в этом контексте. Довод ПРОТИВ я предъявила сама и снимать его не буду: семь кругов, частота находок не падает (7 → 3 → 2 → 2 → 3 → 29), шесть из семи находили дефекты в @@ -685,6 +1011,13 @@ retry-loop 6→3 · bank-and-counters 6→6 · transport-and-config 9→7 заменяет пак, а канон прямо разрешает дофикс по СВОЕМУ паку отработавшей сессии («отработавшей пишут ТОЛЬКО по её же паку — вопросы, ревью, диспозиции, дофиксы»). +⛔ **Но отвечать на него сейчас НЕЛЬЗЯ, и причина не в моём контексте.** Владелец предложил конструкцию, +которой в паке не было: **два останова** — жёсткий гасит всё немедленно, мягкий не рвёт летящие вызовы +(его довод: обрывать соединение и посылать заново значит терять до трети стоимости). Если пользовательская +остановка перестанет рвать вызовы, класс «деньги без ответа» не возникает вовсе — и чинить придётся не 31 +позицию, а другое их число. ⇒ **вопрос ждёт вердикта консилиума** (`D39.231` п.4), правки не начаты, +дерево не трогается. + ⚠ И отдельно — **против отката денежной части**, если он будет рассматриваться. Откат разведения предикатов вернул бы состояние, где отклонённые провайдером запросы становятся ПЛАТНЫМИ (замер: 22 из 25, $0.80 на боевом пути) — списание с читателя за вызов, которого никто не выполнял. Это направление diff --git a/docs/archive/reports/CUT_CALLS_DOFIX_WORK_ORDER_2026-09-08.md b/docs/archive/reports/CUT_CALLS_DOFIX_WORK_ORDER_2026-09-08.md index 37899b63..f544f496 100644 --- a/docs/archive/reports/CUT_CALLS_DOFIX_WORK_ORDER_2026-09-08.md +++ b/docs/archive/reports/CUT_CALLS_DOFIX_WORK_ORDER_2026-09-08.md @@ -8,6 +8,32 @@ > **Довод против раздвоения:** вести поправки отдельно от наряда значит завести второй носитель одного > знания — ровно то, из-за чего сегодня терялись решения (`D39.216` п.3б). +> ⛔ **АМЕНДМЕНТ 10.09 (шаг 0 наряда работ). Наряд писан ДО вердикта консилиума; ниже он приведён в +> соответствие, и КАЖДАЯ правка помечена как правка.** Внесено движковой сессией по зонному исключению +> выше. Что изменилось: +> **(1)** вписан ЗНАМЕНАТЕЛЬ — новый раздел «Знаменатель класса» перед «ЧТО ЧИНЮ»; +> **(2)** позиции получили поля `исход` и `предъявлено` — по ним сверяется завершённость, машинно; +> **(3)** P0 размечены по СЕМЬЯМ (читатели контракта · выходы цепочки `retryLoop` · локальные) — семья +> решает порядок работы, потому что чинить членов семьи поодиночке уже пробовали семь раз; +> **(4)** отмечено, что КНОПКА — не главный источник отмен, и это меняет вес нескольких позиций; +> **(5)** заведена ОДНА новая позиция, которой в наряде не было: `круг8#1`. +> +> ⛔ **И вторая поправка, более важная: наряд НЕС форму правильной починки, а я её не прочла.** Восьмой +> круг нашёл, что все три зонда спрашивают ФИКСИРОВАННЫЙ индекс попытки, тогда как воронка шагает ожоги и +> покупает на следующем. Это ровно позиция `bank-and-counters#2`, P1, где дословно записано: «Зонд зашит +> на попытку 0, а обход ожога идёт по ВОЗРАСТАЮЩИМ индексам… Зонд должен спрашивать ту же ось, что и +> обход». ⇒ **наряд ПРЕДСКАЗАЛ регрессию, которую внесла моя починка**, а я чинила по заказу письмом, не +> открыв наряд. Если бы работа шла по наряду в его порядке, верная форма была бы названа ДО написания +> неверной. Это второй за день довод в пользу наряда как носителя — и первый, где цена невнимания к нему +> измерена: одна регрессия, пойманная только направленным вторым читателем. +> +> ⚠ **Поправка к тому, как этот наряд был передан.** Заказ на два предгейта пришёл со словами «этого не +> видела ни одна линза наряда». Это НЕВЕРНО, и оркестратор поправку принял: наряд несёт находку ТРЕМЯ +> позициями — `burn-walk#2`, `burn-walk#3` и дубль `bank-and-counters#3`, где дословно стоит «два ДРУГИХ +> зонда «чекпойнт есть ⇒ уже оплачено» про ожог не знают». ⇒ **консилиум пере-открыл ЧТЕНИЕМ то, что +> линза нашла ИСПОЛНЕНИЕМ.** Это улика в пользу НАРЯДА, а не в пользу консилиума, и она же объясняет, +> почему в заказе была переоценена новизна находки. + > ⚠ **РЕВЬЮ-ШАПКА ОРКЕСТРАТОРА (перенос в репозиторий 08.09).** Это наряд СЕДЬМОГО адверсариального круга > движковой сессии `textmachine-79` по её же дофиксу пака «ВЫЗОВ, КОТОРЫЙ ОБОРВАЛИ МЫ». Перенесён > ДОСЛОВНО и без правок: он жил в `/tmp` и умер бы вместе с сессией, а по канону доказательная база @@ -66,11 +92,65 @@ «дубль» и чинятся вместе. Дубли НЕ выброшены: каждая оставлена со своим ключом, чтобы при проверке было видно, что ни одна не потеряна. +## Знаменатель класса — ВПИСАН АМЕНДМЕНТОМ 10.09, в наряде его не было + +Наряд перечислял дефекты, но не отвечал на вопрос «сколько их вообще может быть». Вердикт консилиума +ответил «читателей контракта три»; пере-проверка исполнением показала, что это верно **про один глагол** +и ложно про предмет. Формулировка, утверждённая оркестратором дословно: + +> **денежных читателей контракта ТРИ, закрыты 3 из 3; читателей таблицы `checkpoints` — двенадцать +> функций, ожог лжёт четырём площадкам в двух классах.** + +⛔ **Голое «три» публиковать нельзя:** следующая смена прочтёт его как «мест всего три» и не пере-проверит. + +**Как считалось.** Не по именам функций, а по обращению к таблице: `checkpoints` трогают **12** функций +`store` (прибор прочёл 12 нетестовых файлов `internal/store`). Счёт по именам дал бы 5 и пропустил +`EscalationHops` · `EscalationSpentUSD` · `RoleResponsesForBook` · `SpendByModel` · `ResetChunkStages` — +у них предмета в названии нет. + +| читатель | площадок | врёт ли ему ожог | +|---|---|---| +| `GetCheckpoint` | 10 | **3 денежных предгейта — ДА (закрыты 3 из 3)** · 2 воронка (определение) · 5 под `FinalHash` — недостижимы | +| `HasCheckpointForStage` | 1 | **ДА, но не деньгами, а ПРИЧИНОЙ в строке оператора** → позиция `круг8#2` | +| `RoleResponsesForBook` | 1 | НЕТ: зовётся с непустым `mustContain`, запрос фильтрует `instr(response_text, ?) > 0`, пустой текст ожога не совпадёт никогда | +| `EscalationHops` | 1 | НЕТ: его док — «counts the escalation CALLS a book has PAID for», а ожог именно оплаченный вызов | +| `CheckpointUsageForBook` | 3 | НЕТ: ожог несёт настоящие деньги при нулевых токенах ⇒ попадает в оценочную долю, как задумано | +| `RepairStats` | 1 | НЕТ: ожог — настоящий вызов; `declined` матчит по сентинелу, пустой текст ≠ сентинел | +| `EscalationSpentUSD` · `RoleSpentUSD` · `SpendByModel` | 1 · 2 · 1 | НЕТ: суммируют деньги, а деньги ожога настоящие | + +⚠ **Пять площадок под `FinalHash` недостижимы ТРЕМЯ независимыми доводами, а не одним** (общий довод на +пять случаев был бы слабее любого из них): все пять под охраной `Disposition == ok`; писателей `FinalHash` +ровно ДВА, и `cutcall.go:171` пишет пустую строку явно, а `stagerun.go:321` берёт значение, которое +инициализировано `""` и присваивается только под `DispOK`; и даже на `DispOK` берётся хеш ОТВЕТИВШЕЙ +попытки, потому что прогулка по ожогу уже увела индекс. Исполнением это держит существующая фикстура +`TestABurnedCheckpointIsNeverReadAsAnAnswer` (`cutcall_test.go:802`) — второй носитель не заводился. + +⚠ **Расхождение счёта, названное, а не «исправленное»:** у `RoleSpentUSD` вызывающих в движке **2** +(`terminologist.go:916`, `rebill.go:557`), а площадок вообще **3** — третья это `store/ledger.go:102`, тело +`RepairSpentUSD`, то есть глагол, выраженный через глагол. Оба числа верны о разных вопросах. + +⛔ **И ГЛАВНОЕ, что показал восьмой круг: знаменатель закрывает одну ОСЬ, а не работу.** Читателей правда +три — а дефект сидел не в их ЧИСЛЕ, а в ФОРМЕ ВОПРОСА, одинаковой у всех трёх. Никакой счёт читателей +его не ловит; поймал направленный второй читатель. ⇒ находка вида «дефект одинаков у всех членов семьи» +означает, что неверен ВОПРОС, а не позиции, и чинится определением, а не пунктами наряда. + +## Источники отмен — ВПИСАНО АМЕНДМЕНТОМ 10.09 + +⚠ **Кнопка — не главный источник `cancelled`, и это меняет вес позиций.** Волна гасит собственный контекст +изнутри на первой же ошибке или панике (`waverun.go:404`, `:414` — единственные `cancel()` вне `defer` при +140 нетестовых файлах), то есть ОДНА инфра-ошибка в одном воркере метит `cancelled` все летящие вызовы +соседей. Позиции, чья мотивация в наряде звучала как «это редкий случай нажатия кнопки», надо читать как +класс, который приходит пачками. Мягкий останов (обсуждается владельцем) делает вторую половину РЕДКОЙ, +а не ненужной. + ## ЧТО ЧИНЮ — по приоритету, в порядке работы ### P0 · `bank-and-counters#1` — Ожог в банковой партии режет УЖЕ ОПЛАЧЕННЫЕ партии: проход, стоящий $0.000000, дропается целиком - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Приём партий переведён с ПРЕФИКСА на попартийное решение (`terminologist.go:941-970`): отказ в одной партии больше не уносит те, что позади и стоят $0. Поле `run.attempted` (счётчик «сколько с начала») заменено на `run.ran []bool` — счётчик не может описать несплошное множество, и оба его читателя (`:487`, `:667`) переведены с `break` на `continue`. Попутно снято мёртвое `paidBatch` (писалось, никем не читалось). Пин `TestAnUnaffordableBatchDoesNotTakeThePaidBatchesBehindIt` с контролем «провайдера не спросили ни разу»; мутация `CUTBANK-admission-is-a-prefix-again` RED текстом «an ALREADY-PAID batch sitting behind a refused one was dropped» +- **семья:** читатели контракта ⟨вписано амендментом 10.09⟩ - **где:** `internal/pipeline/terminologist.go:947-961` - **как:** Префиксный `break` роняет УЖЕ ОПЛАЧЕННЫЕ партии, когда одна из ранних сожжена: проход, стоящий $0, дропается целиком. Пропускать сожжённую, а не обрывать префикс. - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -82,6 +162,9 @@ ### P0 · `bank-and-counters#3` — Два ДРУГИХ зонда «чекпойнт есть ⇒ уже оплачено» про ожог не знают: перепокупка уходит мимо суб-бюджета эскалации и ремонта - **действие:** ЧИНЮ +- **семья:** читатели контракта ⟨вписано амендментом 10.09⟩ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** дубль `burn-walk#2` и `#3`, закрыт той же правкой — одним определением на три площадки, а не двумя строками - **где:** `escalation.go:146, repair.go:368` - **как:** дубль burn-walk#2 и #3, чинится теми же двумя правками - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -93,6 +176,9 @@ ### P0 · `bank-and-counters#4` — Доставленный обрыв ТЕРЯЕТСЯ на обоих выходах retryLoop по отмене: до вызывающего не доходит ни Deliveries, ни сам обрыв - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** дубль `money-predicate#1`: оба выхода по отмене теперь идут через тот же `chainError`. Предъявлено тем же пином и мутацией `CUTCHAIN-the-stop-exit-drops-what-is-owed` ⛔ **ПОПРАВКА 10.09, вечер: прежнее предъявление было ЛОЖНЫМ, снимаю сама.** Охотник оркестратора посадил снятие `cancelledDuring` на ПЕРВОМ выходе по отмене (`httpllm.go:199`) — батарея осталась зелёной, воспроизвела сама. Механизм: мой пин гонит петлю ЧЕРЕЗ ПРОВОД, и там стоп приходит по вызову в полёте, поэтому ошибка самой попытки — уже родительски-отменённый обрыв, несущий `context.Canceled` в поле `Parent`. Утверждение «прогон обязан читаться как отменённый» удовлетворялось ЧУЖОЙ уликой, а не тем выходом, который позиция объявляла закрытым (замер: `isCanceled=true cause=connection_lost` в 6 прогонах из 6 на мутанте). ⇒ заведён `TestTheStopExitStillReadsAsCancelledWhenTheAttemptDidNot`: гонит `retryLoop` НАПРЯМУЮ, попытка падает обычной 503 и стоп приходит на её возврате — опереться не на что, кроме самого выхода. Провод такой порядок создать не может: окно между возвратом попытки и чтением контекста в несколько инструкций, и фикстура, гоняющаяся за ним, мерила бы планировщик. Мутация `CUTCHAIN-the-stop-exit-drops-the-cancellation` RED. Заодно закрыта названная охотником связка «новый тип ↔ код выхода»: `AttemptCutError` давал **0 хитов** в тестах `cmd/tmctl` при 26 вызовах отображения — заведён `TestTheCutErrorTypeKeepsItsExitCode` (пять форм плюс контроль «потолок всё ещё старше обрыва»), мутация `CUTEXIT-a-stop-stops-mapping-to-five` RED. +- **семья:** выходы цепочки retryLoop ⟨вписано амендментом 10.09⟩ - **где:** `internal/llm/httpllm.go:211 и :234` - **как:** дубль money-predicate#1: оба выхода по отмене зовут cancelledDuring БЕЗ withEarlierCut, поэтому доставленный обрыв до вызывающего не доходит. - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -104,6 +190,9 @@ ### P0 · `burn-walk#1` — Деньги сожжённых ключей ТЕРЯЮТСЯ на свежем вызове: burnedCost перезаписывается, а не прибавляется - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Семья закрыта ОДНИМ инвариантом, предъявленным на всех трёх: строка чанка обязана сходиться с суммой чекпойнтов СВОЕЙ позиции на ЛЮБОМ выходе. Носитель — `assertRowsMatchTheLedger` с контролем «прибору дали что сравнивать» (иначе «расхождений нет» на пустом леджере читается как успех). Обе ветки (`stagerun.go:717` billed-decode и `:789` свежий вызов) прибавляют `burnedCost`, а не затирают его. Пин `TestABurnedKeysMoneyReachesTheRow` (два обрыва подряд — один не даёт ожога, его ретраит транспорт; ожог рождается на КАПЕ второго — затем резюм). Мутация `CUTROW-the-burn-money-is-overwritten` RED: «says it cost 0.002000 while its own checkpoints add up to 0.003056» ⛔ **ПОПРАВКА 10.09, вечер: прежнее предъявление было ЛОЖНЫМ, снимаю сама.** Охотник оркестратора посадил снятие `cancelledDuring` на ПЕРВОМ выходе по отмене (`httpllm.go:199`) — батарея осталась зелёной, воспроизвела сама. Механизм: мой пин гонит петлю ЧЕРЕЗ ПРОВОД, и там стоп приходит по вызову в полёте, поэтому ошибка самой попытки — уже родительски-отменённый обрыв, несущий `context.Canceled` в поле `Parent`. Утверждение «прогон обязан читаться как отменённый» удовлетворялось ЧУЖОЙ уликой, а не тем выходом, который позиция объявляла закрытым (замер: `isCanceled=true cause=connection_lost` в 6 прогонах из 6 на мутанте). ⇒ заведён `TestTheStopExitStillReadsAsCancelledWhenTheAttemptDidNot`: гонит `retryLoop` НАПРЯМУЮ, попытка падает обычной 503 и стоп приходит на её возврате — опереться не на что, кроме самого выхода. Провод такой порядок создать не может: окно между возвратом попытки и чтением контекста в несколько инструкций, и фикстура, гоняющаяся за ним, мерила бы планировщик. Мутация `CUTCHAIN-the-stop-exit-drops-the-cancellation` RED. Заодно закрыта названная охотником связка «новый тип ↔ код выхода»: `AttemptCutError` давал **0 хитов** в тестах `cmd/tmctl` при 26 вызовах отображения — заведён `TestTheCutErrorTypeKeepsItsExitCode` (пять форм плюс контроль «потолок всё ещё старше обрыва»), мутация `CUTEXIT-a-stop-stops-mapping-to-five` RED. +- **семья:** локальные ⟨вписано амендментом 10.09⟩ - **где:** `internal/pipeline/stagerun.go:784 и :713` - **как:** В обеих ветках писать `burnedCost + cost` / `burnedCost + estimate` вместо голого присваивания. Пин: прогон с ДВУМЯ сожжёнными ключами подряд, сверка chunk_status.cost_usd с SUM(checkpoints.cost_usd) SQL-запросом. - **тяжесть после опровержения:** correctness · **источник:** линза+опровергатель @@ -115,6 +204,9 @@ ### P0 · `burn-walk#2` — Сожжённый ключ РЕМОНТА читается как «уже оплачено» — суб-бюджет repair обходится целиком - **действие:** ЧИНЮ +- **семья:** читатели контракта ⟨вписано амендментом 10.09⟩ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** одно определение `paidAfterBurns` (`cutcall.go:196`) + `repair.go:373`; пины `TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget` и `TestAPaidRepairBehindABurnedKeyStillReplaysFree`; мутации `CUTBURN-the-paid-contract-ignores-the-burn`, `…-stops-at-the-first-key`, `…-the-repair-pregate-asks-a-fixed-index` — все RED по тексту. Воспроизведено красным ДО починки: «want 2 repair calls, got 3 — one of them was bought outside gates.repair.budget_usd» - **где:** `internal/pipeline/repair.go:368` - **как:** Зонд «чекпойнт есть ⇒ оплачено» обязан видеть ожог: `cp != nil && !burnedByCut(cp)`. Ровно та правка, что уже сделана для банка (terminologist.go). Пин — по образцу TestTheBankPaidProbeSeesThroughABurnedCheckpoint, на зонде, а не на предикате. - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -126,6 +218,9 @@ ### P0 · `burn-walk#3` — Сожжённый ключ ХОПА читается как «уже оплачено» — escalation.budget_usd не спрашивается (и escMu не берётся) - **действие:** ЧИНЮ +- **семья:** читатели контракта ⟨вписано амендментом 10.09⟩ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** `escalation.go:141` через то же определение; пины `TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget` и `TestAPaidHopBehindABurnedKeyStillReplaysFree`; мутация `CUTBURN-the-escalation-pregate-asks-a-fixed-index` RED. Воспроизведено красным ДО починки: «want 2 hop calls, got 3» - **где:** `internal/pipeline/escalation.go:146` - **как:** То же для `mayHop := fbExists != nil`. Дополнительно проверить, что escMu берётся, раз хоп реально пойдёт на провод. - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -137,6 +232,9 @@ ### P0 · `cancelled-mark#1` — Метка над эскалационным хопом не несёт денег хопа: ledger 0.001176, строка 0.000120 - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Семья закрыта ОДНИМ инвариантом, предъявленным на всех трёх: строка чанка обязана сходиться с суммой чекпойнтов СВОЕЙ позиции на ЛЮБОМ выходе. Носитель — `assertRowsMatchTheLedger` с контролем «прибору дали что сравнивать» (иначе «расхождений нет» на пустом леджере читается как успех). Деньги хопа едут в `out.fb` ВСЕГДА (`escalation.go`), а `runStage` прибавляет их ДО проверки ошибки. Пин `TestAStoppedRunLeavesEveryRowMatchingItsOwnLedger`; мутация `CUTROW-the-hop-money-waits-for-the-verdict` RED текстом «0.000120 против 0.001176» — ровно те числа, что замерила линза. ⚠ Первая редакция пина была ФЛЕЙКОВОЙ: 2 красных из 8 на мутанте, потому что отмена обгоняла разбор заголовков и обрыв выходил на $0, а строка сходилась «ноль к нулю». Перестроена так, что деньги хопа НЕИЗБЕЖНЫ (ожог на ключе хопа в прогоне 1): 8/8 зелёных на дереве, 8/8 красных на мутанте +- **семья:** локальные ⟨вписано амендментом 10.09⟩ - **где:** `internal/pipeline/stagerun.go:233-240 + escalation.go:173` - **как:** Деньги хопа прибавлять к cumCost ДО возврата по ошибке (сейчас только внутри `if esc.attempted`, то есть после `if err != nil { return }`). Пин: отмена над хопом, сверка chunk_status.cost_usd с леджером. - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -148,6 +246,9 @@ ### P0 · `cancelled-mark#2` — Тот же провал на ремонте: ledger 0.001303, сумма всех строк 0.000240 - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Семья закрыта ОДНИМ инвариантом, предъявленным на всех трёх: строка чанка обязана сходиться с суммой чекпойнтов СВОЕЙ позиции на ЛЮБОМ выходе. Носитель — `assertRowsMatchTheLedger` с контролем «прибору дали что сравнивать» (иначе «расхождений нет» на пустом леджере читается как успех). То же для ремонта: `res.CostUSD`/`res.CumUSD` заполняются ДО суждения об исходе (`repair.go`), `runStage` прибавляет их до `if rerr != nil`. Пин `TestAStoppedRunCarriesTheRepairsMoneyToTheRow`, построенный тем же приёмом неизбежных денег; мутация `CUTROW-the-repair-money-waits-for-the-verdict` RED: «0.002000 против 0.003063». ⚠ Мутация СНАЧАЛА ВЫЖИЛА — носителя не было, и это поймал инструмент, а не я +- **семья:** локальные ⟨вписано амендментом 10.09⟩ - **где:** `internal/pipeline/stagerun.go:281-287 + repair.go:302-316` - **как:** То же для ремонта: замер линзы — леджер 0.001303, сумма строк 0.000240. - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -159,6 +260,9 @@ ### P0 · `money-predicate#1` — Оплаченный обрыв теряется целиком, если между ним и остановкой был ЛЮБОЙ не-cut ретраибл (503/429) - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Семья закрыта ОДНИМ контрактом, а не четырьмя правками: `moreOwed` копит то, что цепочка должна вынести, `chainError` цепляет это к любому из ЧЕТЫРЁХ выходов (`httpllm.go`), и решение принимается по ДЕНЬГАМ, а не по типу ошибки. Пин `TestAPaidCutSurvivesAPlainRetryableLaterInTheChain` (обрыв → 503 → исчерпание) и `TestTheCancellationExitCarriesWhatTheChainOwes` (второй выход по отмене, синхронизирован на СТОП, не на часы). Мутации `CUTCHAIN-a-plain-retryable-erases-the-paid-cut` и `CUTCHAIN-the-stop-exit-drops-what-is-owed` — RED по тексту +- **семья:** выходы цепочки retryLoop ⟨вписано амендментом 10.09⟩ - **где:** `internal/llm/httpllm.go:211 и :234` - **как:** Оплаченный обрыв теряется, если между ним и концом цепочки был ЛЮБОЙ не-cut ретраибл (503/429). Причина та же, что у #2: firstCut не доносится. Чинить вместе с money-predicate#2 и bank#4 — это одна семья из трёх выходов. - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -170,6 +274,9 @@ ### P0 · `money-predicate#2` — Оплаченный обрыв маскируется НЕоплачиваемым обрывом той же цепочки: книжится $0 - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Семья закрыта ОДНИМ контрактом, а не четырьмя правками: `moreOwed` копит то, что цепочка должна вынести, `chainError` цепляет это к любому из ЧЕТЫРЁХ выходов (`httpllm.go`), и решение принимается по ДЕНЬГАМ, а не по типу ошибки. Правило выхода сменено с «финальная ошибка несёт ЛЮБОЙ обрыв» на «несёт обрыв с денежным требованием не слабее». Пин `TestAFreeCutLaterDoesNotMaskThePaidCutEarlier`; мутация `CUTCHAIN-the-exit-ranks-cuts-by-type-not-money` RED. ⚠ Попутно найдена и снята МОЯ недостижимая ветка: предпочтение «платный над бесплатным» в накопителе сработать не может — ретраибл только `CutByConnection`, а кап делает ВТОРОЙ доставленный обрыв терминальным, значит обрывов в цепочке максимум два и второй всегда её заканчивает. Ранжирование живёт на выходе, где обе ошибки в руках +- **семья:** выходы цепочки retryLoop ⟨вписано амендментом 10.09⟩ - **где:** `internal/llm/httpllm.go:249-251 (withEarlierCut)` - **как:** ОБЪЕДИНЕНО с retry-loop#1. `withEarlierCut` отдаёт финальную ошибку, если она несёт ЛЮБОЙ обрыв. После разведения предикатов это неверно: поздний Billable=false затирает ранний Billable=true. Правило должно быть «предпочесть ОПЛАЧИВАЕМЫЙ обрыв», а не «любой». - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -181,6 +288,9 @@ ### P0 · `retry-loop#1` — withEarlierCut выбрасывает ОПЛАЧЕННЫЙ обрыв первой попытки, если цепочку заканчивает ДРУГОЙ обрыв — движок книжит $0 - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** дубль `money-predicate#2`, закрыт тем же контрактом. Дополнительный пин `TestAPaidCutAfterAFreeOneIsTheOneTheCallerSettlesFrom` держит обратный порядок (бесплатный, затем платный) с премиссой на РОВНО два вызова — если политика ретраев когда-нибудь пустит третий, премисса скажет об этом вслух, а не даст фикстуре молча мерить другую цепочку. ⚠ Три ЧУЖИХ якоря каталога протухли от переписи петли и пере-нацелены поштучно, каждый пере-проверен посадкой +- **семья:** выходы цепочки retryLoop ⟨вписано амендментом 10.09⟩ - **где:** `internal/llm/httpllm.go:248-251` - **как:** дубль money-predicate#2 - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -192,6 +302,9 @@ ### P0 · `критик#2` — `cancelled` читается прогнозом как ПОЛНОСТЬЮ отработанная позиция: одна остановка роняет projected_book_usd на 9.9% — носитель resolvedForResume правлен в двух местах из четырёх - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Предикат `resolvedForResume` подставлен в ЧЕТВЁРТЫЙ носитель — `resolveChunkState` (`status.go:576`): строка `cancelled` больше не решает судьбу единицы, и `projectBookUSD` не берёт её в знаменатель экстраполяции. Довод наряда точен и решил форму починки: ДО пака остановка не оставляла строки вовсе, знаменатель был честен, и ухудшила его именно новая метка — значит чинить надо восстановлением прежней правды, а не новым правилом. Пин `TestAStoppedPositionIsNotADecidedUnit` держит ОБЕ стороны (остановленная позиция не решает; обычный контентный флаг по-прежнему решает — иначе предикат, отвечающий «не решено» на всё, прошёл бы половину теста и тихо выключил флагирование). Мутация `CUTSTATE-a-stopped-position-decides-the-unit` RED по тексту. ⚠ ПИНГ оркестратору: правка соприкасается со строкой бэклога **374** (вердикт главы считается по СЧЁТУ флагов) — остановленная позиция теперь не попадает в этот счёт на уровне единицы, что 374 частично снимает; решение о вердикте ГЛАВЫ остаётся его +- **семья:** локальные ⟨вписано амендментом 10.09⟩ - **где:** `internal/pipeline/rebill.go:279 + status.go:564-573` - **как:** resolvedForResume подставлен в ДВА носителя из ЧЕТЫРЁХ: прогноз и состояние чанка читают `cancelled` как полностью отработанную позицию ⇒ одна остановка роняет projected_book_usd на 9.9 %. Это число читает платформа. - **тяжесть после опровержения:** money · **источник:** критик полноты @@ -201,6 +314,8 @@ ### P1 · `bank-and-counters#2` — Зонд смотрит только на попытку 0: партия, уже ОТВЕЧЕННАЯ на попытке 1, требует полный `want` из суб-бюджета, который никогда не потратит - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** `terminologist.go:792` переведён на `paidAfterBurns`; пин `TestTheBankProbeFindsThePaidBatchBehindABurnedKey`, мутация `CUTBURN-the-bank-pregate-asks-a-fixed-index` RED текстом «the batch WAS bought and answered at the index the funnel walked to». ⚠ Эта позиция и есть та, что ПРЕДСКАЗАЛА регрессию двух других зондов — см. амендмент - **где:** `internal/pipeline/terminologist.go:792` - **как:** Зонд зашит на попытку 0, а обход ожога идёт по ВОЗРАСТАЮЩИМ индексам: партия, отвеченная на попытке 1, требует полный want из суб-бюджета, который не потратит. Зонд должен спрашивать ту же ось, что и обход. - **тяжесть после опровержения:** money · **источник:** линза+опровергатель @@ -212,6 +327,8 @@ ### P1 · `bank-and-counters#5` — `Deliveries` считает ОБРЫВЫ, а не доставки: и доккомментарий, и строка леджера утверждают число, которое замеряется неверным - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Выбран честный счёт, а не переименование: строку читает ОПЕРАТОР, и она единственное место, где виден разрыв между «сколько сгенерировали» и «сколько забукали». Счётчик разведён на два — кап ретраев остаётся на обрывах (`deliveredCutSeen`), а число для строки считается по ФАКТУ доставки (`askedToGenerate` + предикат `deliveredAttempt`): статус любого рода есть доставка по определению (пир прочитал запрос, чтобы ответить), оплаченный нечитаемый 2xx — тоже; не доставка — всё, что упало до ухода байтов. Доккомментарий поля приведён в соответствие. Пин `TestTheDeliveryCountCountsDeliveriesNotCuts` (503 → обрыв → его единственный ретрай = ТРИ доставки), число вызовов закреплено премиссой, чтобы смена политики ретраев не дала фикстуре молча мерить другую цепочку. Мутация `CUTCOUNT-deliveries-counts-cuts-again` RED. ⚠ Правка сломала ЧУЖОЙ якорь `CUTCALL-the-delivery-count-is-not-carried` — пере-нацелен и пере-проверен посадкой - **где:** `internal/llm/attemptcut.go:104-109 + httpllm.go:459-470 + pipeline/cutcall.go:115-122` - **как:** Deliveries инкрементируется только под `errors.As(err,&cut)`, то есть считает ОБРЫВЫ, а не доставки: доккомментарий и строка леджера называют неверно замеренное число. Либо считать доставки честно, либо переименовать поле и текст. - **тяжесть после опровержения:** correctness · **источник:** линза+опровергатель @@ -223,6 +340,8 @@ ### P1 · `burn-walk#4` — Строка остановленной позиции пишет attempts=0 при реально уплаченных деньгах - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** `attemptsMade` пишется ДО проверки ошибки (`stagerun.go`), по той же причине, что и деньги рядом: счёт — факт о случившемся, а не о том, удалось ли. Инвариант строки расширен: «стоила денег ⇒ обязана назвать хотя бы одну попытку». Пин `TestAStopOnAStagesFirstCallStillReportsAnAttempt` — остановка на ПЕРВОМ вызове стадии, деньги неизбежны через ожог; мутация `CUTROW-the-attempt-count-waits-for-the-verdict` RED, и текст падения называет **0.001056** — ровно число линзы. 6/6 зелёных на дереве, 6/6 красных на мутанте. ⚠ Мутация СНАЧАЛА ВЫЖИЛА: во всех прежних сценариях первый вызов стадии успевал отработать, и счёт уже не был нулём — носителя пришлось строить отдельно - **где:** `internal/pipeline/stagerun.go:170-179` - **как:** ОБЪЕДИНЕНО с cancelled-mark#3. `attemptsMade` присваивать ДО проверки ошибки, иначе метка пишет attempts=0 при уплаченных деньгах. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -234,6 +353,8 @@ ### P1 · `cancelled-mark#3` — Метка врёт числом попыток: attempts=0 при оплаченном cost_usd=0.001056 - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** дубль `burn-walk#4`, закрыт той же правкой и тем же пином - **где:** `internal/pipeline/stagerun.go:170-179` - **как:** дубль burn-walk#4, чинится одной правкой - **тяжесть после опровержения:** correctness · **источник:** линза+опровергатель @@ -245,6 +366,8 @@ ### P1 · `money-predicate#3` — Половина `answered &&` денежного предиката не закреплена НИЧЕМ: мутант выживает во всей батарее - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Половина `answered &&` закреплена фикстурой, где две половины РАСХОДЯТСЯ: пир пишет байты ответа, которые не являются ответом (сломанная статус-строка), так что первый байт трассы есть, а 2xx-объекта нет. Пин `TestResponseBytesWithoutAReplyAreNotAPurchase` с двумя премиссами (это обрыв; `AfterHeaders` истинно — иначе обе половины ложны и тест их не различает); мутация `CUTMONEY-response-bytes-alone-mean-payment` RED текстом «response BYTES are not a reply» - **где:** `internal/llm/attemptcut.go:212` - **как:** Половина `answered &&` денежного предиката не закреплена ничем — мутант переживает всю батарею. Нужна фикстура, где answered=false при firstByte=true (отказ + RST поверх пишущегося тела) И проверка ДЕНЕГ, а не только типа ошибки. - **тяжесть после опровержения:** correctness · **источник:** линза+опровергатель @@ -256,6 +379,8 @@ ### P1 · `transport-and-config#1` — Пин write-бонда проверяет НЕ тот транспорт, который уходит в бой: удаление тюнинга из боевого клиента переживает всю батарею пакета - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** предмет (пин write-бонда) удалён вместе с бондом. ⚠ КЛАСС при этом воспроизвёлся в моей ЗАМЕНЕ пина и закрыт отдельно: `keepAliveHTTPClient` разделён на себя и `buildCloudClient`, пин читает транспорт боевой конструкции, мутация `CUTKEEPALIVE-the-cloud-client-is-not-tuned-at-all` RED текстом «this test measured nothing» - **где:** `internal/llm/attemptcut_test.go:853 против httpllm.go:124` - **как:** Пин write-бонда конфигурирует СВОЙ транспорт, а не тот, что уходит в бой: удаление тюнинга из боевого клиента переживает всю батарею. Пин обязан читать транспорт, собранный keepAliveHTTPClient. - **тяжесть после опровержения:** correctness · **источник:** линза+опровергатель @@ -267,6 +392,8 @@ ### P1 · `transport-and-config#2` — Комментарий обещает закрыть awaitFlowControl-парковку — замер показывает, что она открыта: 3-секундный дедлайн держался >70 с на БОЕВОМ клиенте - **действие:** ЧИНЮ КОММЕНТАРИЙ, механизм — ПИНГ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** предмет удалён вместе с бондом: комментарий обещал, что бонд закрывает парковку в `awaitFlowControl`, а замер это опровергал. Контроль после удаления: греп `awaitFlowControl|flow control|flow-control` по `backend/` даёт **0 хитов при 360 прочитанных go-файлах** — ложного обещания в дереве не осталось. Класс «дедлайн инертен на h2» остаётся строкой бэклога **373** - **где:** `internal/llm/httpllm.go:97-111` - **как:** Комментарий обещает, что бонд закрывает awaitFlowControl-парковку. ЗАМЕР ОПРОВЕРГАЕТ: 3-секундный дедлайн держался >70 с на боевом клиенте. Комментарий — мой и врёт, его правлю. Оставлять ли сам бонд — решение оркестратора (см. пинги ниже). - **тяжесть после опровержения:** correctness · **источник:** линза+опровергатель @@ -278,6 +405,8 @@ ### P1 · `transport-and-config#3` — Пол скорости zai — данные без носителя: каталожный гейт zai вообще не смотрит, удаление `tok_s_floor: 35` проходит все 78 кейсов internal/config - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Замер сначала: пол `zai` = 35 против вендорского дефолта 35.56, то есть влияет на 1.6 % — он ЗАМЕРЕН (p10 glm-5 = 37.51 ток/с на 647 строках лога) и осознанно чуть ниже дефолта. Прежний гейт его не видел структурно: он ходит по ЗАГРУЖЕННОЙ структуре, где удалённая строка и никогда не объявленное поле — один и тот же ноль, и пропускает провайдера, чьи модели не объявляют токенного пола. ⇒ заведён `TestEveryDeclaredDeadlineKnobStaysDeclared`: пинует сам ФАКТ объявления по всем четырём провайдерам, с контролем «объявляющих ровно столько, сколько названо в пине». Мутация `CUTCFG-a-measured-speed-floor-is-deleted` RED текстом «deadline knobs moved from {tokSFloor:35 …} to {tokSFloor:0 …}» - **где:** `internal/config/models_catalog_test.go:219-225` - **как:** Гейт смотрит на провайдеров через MinMaxTokens и `if grant == 0 { continue }`, поэтому zai не смотрит вовсе: удаление `tok_s_floor: 35` проходит все 78 кейсов. Расширить гейт на объявленные поля, а не только на выведенные гранты. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -289,6 +418,8 @@ ### P1 · `transport-and-config#4` — У zai объявлен пол без потолка, и связку никто не валидирует: опечатка в поле даёт 2 ч 32 мин на вызов, а потолок ниже пола обнуляет заявленное «attempt_s — это ПОЛ» до 10 с - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Заведена `validateTimeouts` (`internal/config/models.go`), вызываемая из провайдерского цикла `LoadModels`: диапазоны на все три секундных поля (типо-гвардия 4 часа — на порядок выше ратифицированных ~20 минут, чтобы законная настройка не отвергалась), запрет отрицательного `tok_s_floor` и ГЛАВНОЕ — связка `attempt_max_s >= attempt_s`. Пин `TestACapBelowTheFloorIsRefusedAtLoad` начинается с КОНТРОЛЯ (крышка выше пола и незаданная крышка обязаны грузиться — иначе проверку удовлетворил бы загрузчик, отвергающий всё) и требует, чтобы отказ называл ОБА поля. Мутации `CUTCFG-a-cap-below-the-floor-loads-quietly` и `CUTCFG-the-timeout-check-is-not-wired-in` — обе RED; вторая существует потому, что проверка, которую никто не зовёт, — тот же класс, что я дважды снимала за смену - **где:** `internal/config/models.go:122-146 + internal/llm/attemptcut.go:281-289` - **как:** ОБЪЕДИНЕНО с критик#1. Ни одной проверки диапазона: опечатка в поле даёт 2 ч 32 мин на вызов, а attempt_max_s НИЖЕ attempt_s молча укорачивает каждый вызов — прямо вопреки моему же комментарию «attempt_s is the FLOOR … no provider loses a second it has today». Добавить валидацию в LoadModels и пин. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -300,6 +431,8 @@ ### P1 · `критик#1` — attempt_max_s ниже attempt_s молча УКОРАЧИВАЕТ дедлайн каждого вызова — вопреки комментарию, который на этом обещании и стоит; ни валидации, ни теста, ни пина - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** дубль `transport-and-config#4`, закрыт той же правкой и тем же пином - **где:** `internal/config/models.go:122-146 + internal/llm/attemptcut.go:281-289` - **как:** ДУБЛЬ transport-and-config#4, чинится той же правкой: валидация связки tok_s_floor/attempt_s/attempt_max_s в LoadModels плюс пин на то, что attempt_max_s ниже attempt_s не проходит загрузку. - **тяжесть после опровержения:** money · **источник:** критик полноты @@ -309,6 +442,8 @@ ### P1 · `критик#5` — FlagConnectionLost объявлен, отранжирован и НЕДОСТИЖИМ: обоснование в его комментарии не реализовано ни одной строкой кода - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Снята — по прямому указанию наряда «либо сделать достижимым, либо снять константу и ранг». Замер подтвердил недостижимость: `classify` имеет ветви `decodeErrorFinish` и `attemptTimeoutFinish` и НЕ имеет `connectionLostFinish`; настоящую защиту даёт `burnedByCut`, который ловит такую строку ДО любой классификации. Снят и ранг; комментарий на её месте описывает МЕХАНИЗМ («у потери связи нет флага, и это утверждение, а не пропуск»), а не историю правки. Контроль: `FlagConnectionLost` в дереве — 0 хитов кроме объясняющего комментария, при 360 прочитанных go-файлах - **где:** `internal/pipeline/disposition.go:98-104 + status.go:432` - **как:** FlagConnectionLost объявлен, отранжирован и НЕДОСТИЖИМ: обоснование в его комментарии не реализовано ни одной строкой. Либо сделать достижимым, либо снять константу и ранг — но не оставлять словами. - **тяжесть после опровержения:** correctness · **источник:** критик полноты @@ -318,15 +453,32 @@ ### P1 · `критик#6` — Вся новая таблица тяжести флагов пинуется только на ЧЛЕНСТВО, а не на ЗНАЧЕНИЕ: cancelled можно объявить худшей бедой книги, и батарея пакета зелёная - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Заведён пин на ЗНАЧЕНИЯ, а не на членство: `TestTheSeverityTableMeansWhatItsCommentsSay`. Каждое утверждение — предложение, которое таблица уже пишет о себе сама, превращённое в проверку: `cancelled` мягче всех (его комментарий: «the mildest mark there is»); `decode_error` и `attempt_timeout` делят ранг («they rank together»); `severityUnknown` мягче любого диагноза («must not out-rank a diagnosis the engine actually made»); авто-очищенный кусок мягче бюджетного симптома; выброшенный загрязнённый — строже. Плюс КОНТРОЛЬ в начале: таблица не пуста и не одноцветна, иначе всякое «мягче» держится тривиально. Три мутации RED по тексту - **где:** `internal/pipeline/status.go:430-449, пин flagseverity_test.go` - **как:** Таблица тяжести пинуется на ЧЛЕНСТВО, а не на ЗНАЧЕНИЕ: cancelled можно объявить худшей бедой книги, и батарея зелёная. Нужен пин на ПОРЯДОК (кто кого обязан перевешивать), как у off_target_lang. - **тяжесть после опровержения:** correctness · **источник:** критик полноты - **в чём дефект:** Дофикс переписал порядок тяжести и подробно обосновал каждый ранг (decode_error и attempt_timeout вровень; connection_lost туда же; cancelled — «the mildest mark there is», ниже авто-очистки; unknown сдвинут 8→9). Единственный гейт над этой таблицей — TestEveryFlagReasonIsRanked — проверяет, что каждая ОБЪЯВЛЕННАЯ константа ПРИСУТСТВУЕТ в карте; порядок он не проверяет. Единственная новая мутация на этот предмет (CUTFLAG-a-cut-chunk-is-the-mildest-thing-that-can-happen) тоже бьёт по членству — её edit УДАЛЯЕТ строку FlagAttemptTimeout целиком. Значит вся содержательная часть правки (какой флаг тяжелее какого) не пинована ничем, и решение, которое пак объявляет ключевым для паспорта главы, любая следующая смена может переставить бесшумно. Отдельно: обратной проверки — «каждый отранжированный флаг движок умеет выдать» — тоже нет, и именно поэтому мёртвый FlagConnectionLost из находки №5 пр - **улика линзы:** МУТАЦИЯ (посажена, выжила): status.go:449 `FlagCancelled: 8` → `FlagCancelled: 0` — остановленная позиция становится ХУДШЕЙ проблемой главы, обгоняя жёсткий отказ, то есть ровно то, что комментарий на :443-448 объявляет недопустимым. `go test ./internal/pipeline/` → ok textmachine/backend/internal/pipeline 34.853s. Ни один тест не прочёл значение. Файл восстановлен, cmp: RESTORED-OK. Тело объявленной мутации (cmd/tmmutate/mutations.json), доказывающее, что пин целит в членство, а не в значение: edits[0] = {"find": "\tFlagDecodeError: 4,\n\tFlagAttemptTimeout: 4,\n", "replace": "\tFlagDecodeError: 4,\n"} — строка удаляется, и её ловит проверка «объявлен, но не отранжирован». +### P2 · `круг8#1` — Строка оператора о повторной оплате банка называет ЛОЖНУЮ причину, когда прежняя строка была сожжённой + +- **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Причина названа ПАРОЙ, а не одной из двух: «либо композиция партий изменилась, либо прежние строки записали деньги без результата и невоспроизводимы». Код не может различить эти случаи, а сообщение, выбирающее один, неверно в половине из них — запрет `D39.93` п.2. Обе ветки оставляют оператору одно и то же действие, поэтому назвать обе ничего не стоит, а назвать одну стоит правды. ⚠ Гейт `TestEveryOperatorMessageIsCatalogued` поймал правку — голден обновлён ОДНОЙ строкой (126 → 126), и это ОБСЛУЖИВАНИЕ по `D39.183`: смена вызвана заказанной правкой формулировки, объявляю отдельно +- **семья:** читатели контракта ⟨вписано амендментом 10.09⟩ +- **где:** `internal/pipeline/terminologist.go:394` (чтение) → `:478` (строка) +- **как:** Предупреждение обязано различать «прежние чекпойнты не нашлись из-за смены композиции партий» и «прежняя строка невоспроизводима, потому что она сожжена». Причина у сообщения одна, а фактов два. +- **тяжесть после опровержения:** слово оператору · **источник:** пере-проверка знаменателя 10.09 ⟨позиция ВПИСАНА АМЕНДМЕНТОМ, в наряде её не было⟩ +- **в чём дефект:** `HasCheckpointForStage` — ЧЕТВЁРТЫЙ читатель существования чекпойнта, которого счёт по `GetCheckpoint` не видел. Он спрашивает «платила ли книга за эту стадию когда-либо», и сожжённая строка отвечает «да» — законно, деньги были потрачены. Но строка на `:478` объясняет оператору повторную оплату так: «those earlier calls were made under a batch composition the already-banked filter does not reproduce, so their checkpoints could not be found». Если единственная прежняя строка сожжена, чекпойнт не нашёлся НЕ из-за композиции: он невоспроизводим по построению. Сообщение, лгущее о СВОЕЙ причине, запрещено `D39.93` п.2 — тем самым пунктом, который пак цитирует сам (`cutcall.go:158`). +- **почему это НЕ ломает знаменатель:** замер: `grep -n paidBefore internal/pipeline/*.go` = 3 хита (чтение, комментарий, один `if`). Гейтит РОВНО строку лога, ничего не покупает и никакого бюджета не охраняет ⇒ денежных читателей по-прежнему три, основание наряда цело. Правило остановки не сработало и не должно было. +- **воспроизведение:** книга с единственной сожжённой банковой строкой + свежий проход ⇒ `reconsolidated=true` с текстом про композицию. + ### P2 · `bank-and-counters#6` — Строка «asked N times» невидима там, где колонку читают: errTail режет на 120 байтах, и приписка стоит в отрезаемом хвосте - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Формулировка вынесена в именованную `cutErrLine` (`cutcall.go`), потому что оба её свойства несущие и с места вызова не видны. Приписка «asked N times» переставлена В НАЧАЛО: колонка оператора ограничена 120 байтами, транспортная ошибка перед ней регулярно длиннее, поэтому приписка в хвосте не доходила НИКОГДА. Пин `TestTheAskedNTimesNoteReachesTheReader`; мутация `CUTLINE-the-note-goes-back-to-the-end` RED - **где:** `internal/pipeline/cutcall.go:122 против cmd/tmctl/render.go:512-528` - **как:** ОБЪЕДИНЕНО с retry-loop#5. Приписка «asked N times» стоит в КОНЦЕ err, а errTail режет на 120 байтах ⇒ до оператора не доходит никогда. Плюс перевод строки от errors.Join печатает одну строку отчёта двумя. Ставить приписку в НАЧАЛО и убирать перевод строки. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -338,6 +490,8 @@ ### P2 · `cancelled-mark#5` — Комментарий у defer называет носителем банковские батчи, которые в runStage не входят - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Из перечня носителей у `defer` убраны банковые батчи: в `runStage` они не входят, и комментарий врал о знаменателе. Осталось то, что там действительно есть, — хоп и под-шаг ремонта - **где:** `internal/pipeline/stagerun.go:143-144` - **как:** Комментарий у defer называет носителем банковские батчи, которые в runStage НЕ входят. Убрать их из перечня — иначе комментарий врёт о знаменателе дверей. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -349,6 +503,8 @@ ### P2 · `money-predicate#5` — Лог сожжённого ключа называет «оплаченным» чекпойнт на $0 - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Строка ожога переформулирована: «the key of this attempt is SPENT … cannot be replayed» вместо «attempt was paid for». Ключ потрачен в любом случае, а стоил он законно нуля, когда провайдер вызов не подтвердил; «оплачен» над `cost_usd=0.000000` читается ночью в логе как биллинговый баг - **где:** `internal/pipeline/stagerun.go:496-498` - **как:** Строка «attempt was paid for» печатается над cost_usd=0.000000. Переформулировать: «ключ потрачен» вместо «оплачен», и печатать сумму как есть. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -360,6 +516,8 @@ ### P2 · `money-predicate#6` — AfterHeaders=true для 1xx и для незакрытого блока заголовков — оператору сообщают об ответе, которого не было - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Выбран второй путь наряда — описать поле тем, чем оно является. `AfterHeaders` теперь документирован как ПЕРВЫЙ БАЙТ ответа (`GotFirstResponseByte`), а не как «ответ начал приходить», с явным перечнем того, для чего он срабатывает и подтверждением ничего не значит: 1xx, незакрытый блок заголовков, мусор сломанного прокси. Денежный вопрос отвечает `Billable`, который дополнительно требует 2xx-объекта в руках, и этот раздел теперь закреплён пином `TestResponseBytesWithoutAReplyAreNotAPurchase` (см. `money-predicate#3`) - **где:** `internal/llm/attemptcut.go:79-80, 211` - **как:** AfterHeaders=true для 1xx и для незакрытого блока заголовков — оператору сообщают об ответе, которого не было. Либо считать только 2xx-объект, либо переименовать поле в лог-строке. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -371,6 +529,8 @@ ### P2 · `retry-loop#4` — Строка операторского журнала утверждает «одна оценка забукана на все», когда забукан НОЛЬ - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Текст говорит о фактически забуканном: при `cost <= 0` — «NOTHING is booked: the provider acknowledged none of them» вместо «ONE estimate is booked». Строка рядом с $0-записью, утверждающая обратное, — предложение, которое надо не поверить, чтобы им пользоваться. Мутация `CUTLINE-the-note-claims-money-that-was-not-booked` RED - **где:** `internal/pipeline/cutcall.go:115-122` - **как:** Строка утверждает «ONE estimate is booked for all of them», когда забукан НОЛЬ (Billable=false). Условие текста должно смотреть на фактическую сумму. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -382,6 +542,8 @@ ### P2 · `retry-loop#5` — errors.Join кладёт перевод строки в request_log.err: одна строка отчёта печатается ДВУМЯ, а заметка о числе доставок обрезается и до оператора не доходит никогда - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** `errTail` (`cmd/tmctl/render.go`) схлопывает пробельное: `errors.Join` разделяет членов переводом строки, и строка таблицы печаталась ДВУМЯ, ломая выравнивание всего под ней. Пин `TestTheOperatorTailIsOneLineAndKeepsTheNoteThatMatters` — перевод строки поставлен ВНУТРИ границы обрезки, иначе обрезка убрала бы его даром и фикстура прошла бы на рендерере, который ничего не схлопывает. Мутация `CUTLINE-the-tail-splits-across-rows` RED - **где:** `cmd/tmctl/render.go:512-528` - **как:** дубль bank-and-counters#6 - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -393,6 +555,8 @@ ### P2 · `transport-and-config#5` — Пол zai замерен по одной модели из двух; вторая (флагман glm-5.1) наследует 35 ток/с без замера и без предупреждения - **действие:** ЧИНЮ +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** Замерить вторую модель нельзя — пак $0, платных вызовов ноль. Поэтому объявлено ОТСТУПЛЕНИЕ там, где живёт число (`configs/models.yaml`): пол замерен по glm-5, glm-5.1 наследует его без собственного замера, и названо, почему это принято — направление ошибки безопасно, 35 ниже вендорского дефолта 35.5, то есть даже неверный для 5.1 пол даёт вызову БОЛЬШЕ времени, а не меньше. Назван и способ снять отступление: n≥100 строк request_log по glm-5.1 и её собственный p10 - **где:** `configs/models.yaml:85-90` - **как:** Пол zai замерен по одной модели из двух; glm-5.1 наследует 35 ток/с без замера. Либо замерить вторую, либо объявить отступление в комментарии, как сделано у deepseek. - **тяжесть после опровержения:** minor · **источник:** линза+опровергатель @@ -410,6 +574,8 @@ ### `transport-and-config#6` — Write-бонд рвёт ВСЮ h2-связь, а не застрявший стрим: соседний вызов, чьё тело уже ушло, умирает вместе с ним - **где:** `internal/llm/httpllm.go:140` +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** бонд удалён целиком, ратификация `D39.231` п.2 — оркестратор принял довод против собственного заказа. Предъявлено тройкой замеров на копии: дерево БЕЗ бонда `ok`; бонд ВОЗВРАЩЁН — тоже `ok`, значит батарее он невидим в обе стороны и не держал ничего; контроль — посадка новой мутации RED текстом «the read-idle bound is not on the transport: got 0s want 15s». Два зелёных без третьей строки означали бы «прибор не спросил» - **почему не моё:** Write-бонд рвёт ВСЮ h2-связь (липкая cc.werr), а не застрявший стрим: соседний вызов, чьё тело уже ушло, умирает вместе с ним. Вместе с #2 и #9 это довод СНЯТЬ бонд, а не чинить — но снятие вернёт исходную дыру, и это решение оркестратора. - **в чём дефект:** Ошибка записи липнет к соединению, а не к стриму, и h2 мультиплексирует параллельные стадии одного провайдера в один ClientConn. Один застрявший на записи вызов уносит все остальные in-flight на этом соединении. Деньги при этом не теряются (у соседа нет 2xx-объекта ⇒ Billable=false), но это лишний ретрай и лишняя задержка, и в комментарии к константе размен не назван. - **воспроизведение:** `d=$(mktemp -d) && (cd /home/ubuntu-26/projects/textmachine/backend && tar -cf "$d/t.tar" --exclude=./bin --exclude='./.env*' .) && mkdir "$d/backend" && tar -xf "$d/t.tar" -C "$d/backend" && cp /tmp/claude-1000/-home-ubuntu-26-projects-textmachine/f88e2870-ec2e-48b8-9776-d0370e00bc0b/scratchpad/prob` @@ -417,6 +583,8 @@ ### `transport-and-config#9` — Для тех тел, которые движок реально шлёт, write-бонд не может выстрелить — обещание «дедлайн был инертен» верно на два порядка выше нашего размера - **где:** `internal/llm/httpllm.go:101-106` +- **исход:** сделано ⟨вписано амендментом 10.09⟩ +- **предъявлено:** тот же предмет, снят вместе с механизмом. Носитель класса «наш дедлайн инертен на h2» остаётся строкой бэклога **373**: удалён МЕХАНИЗМ, который класс не закрывал, а не сам класс - **почему не моё:** Для тел, которые движок реально шлёт, бонд выстрелить не может: обещание «дедлайн был инертен» верно на два порядка выше нашего размера. Кластер с #2 и #6. - **в чём дефект:** Запись блокируется, только когда тело перестаёт помещаться в буферы сокета; до этого RoundTrip дописывает запрос, cleanupWriteRequest снова слушает ctx, и НАШ дедлайн работает штатно. Реальные тела запроса у нас — десятки KiB. Значит зафиксированный в комментарии класс зависаний достижим у нас только при теле в единицы MiB, чего пайплайн не отправляет; констатация полезна, чтобы следующая смена не считала бонд активной защитой боевых вызовов. - **воспроизведение:** `d=$(mktemp -d) && (cd /home/ubuntu-26/projects/textmachine/backend && tar -cf "$d/t.tar" --exclude=./bin --exclude='./.env*' .) && mkdir "$d/backend" && tar -xf "$d/t.tar" -C "$d/backend" && cp /tmp/claude-1000/-home-ubuntu-26-projects-textmachine/f88e2870-ec2e-48b8-9776-d0370e00bc0b/scratchpad/prob` @@ -424,12 +592,16 @@ ### `критик#3` — Ранг флага решили, а ВЕРДИКТ главы — нет: одна остановка делает главе «attention», две — «fail» - **где:** `internal/pipeline/status.go:716, :735, :737` +- **исход:** пинг ⟨вписано амендментом 10.09⟩ +- **предъявлено:** у оркестратора, заведено строкой бэклога **374** (`D39.231` п.3) — вердикт ГЛАВЫ по остановленной позиции, продуктовое решение. ⚠ Исчезнет само, если денежную половину пака урежут: следствие класса `cancelled` - **почему не моё:** Одна остановка делает главе вердикт «attention», две — «fail». Ранг флага это не лечит: вердикт считается по СЧЁТУ флагов, а не по их тяжести. Что должна показывать глава, над которой нажали стоп, — продуктовое решение. - **в чём дефект:** Пак сам сформулировал риск в комментарии к рангу (status.go:443-448): «a passport that reported «cancelled» as a chapter's worst problem would hide the durable finding behind a transient one» — и закрыл его РАНЖИРОВАНИЕМ (FlagCancelled: 8, последний перед unknown). Но ранг влияет только на выбор WorstFlagReason (status.go:717-718). Счётчик UnitsFlagged инкрементится для ЛЮБОГО ChunkFlagged без взгляда на причину (:716), а вердикт главы считается по СЧЁТУ, а не по тяжести: `case p.UnitsFlagged >= 2: fail` / `== 1: attention`. Итог: оператор жмёт стоп в момент, когда в полёте две единицы одной главы, и паспорт объявляет главу проваленной — при том что следующий резюм обе строки перезапишет. На ### `критик#4` — Строка `cancelled` черновой стадии читается как ВЫПАВШИЙ ЧЛЕН редакторской единицы — контент-вердикт, вынесенный по факту нажатия стопа - **где:** `internal/pipeline/status.go:538-543 → export.go:327` +- **исход:** пинг ⟨вписано амендментом 10.09⟩ +- **предъявлено:** у оркестратора, заведено строкой бэклога **375** (`D39.231` п.3) — семантика экспорта. ⚠ Та же оговорка - **почему не моё:** memberDrops опознаёт выпавшего члена по (черновая стадия && flagged) без взгляда на причину, поэтому строка `cancelled` читается как КОНТЕНТ-вердикт. Семантика экспорта — не моё решение. - **в чём дефект:** memberDrops — по собственной шапке «the SINGLE definition of the c-lite rule «a dropped member flags its unit»» — опознаёт выпавшего члена ровно по паре (стадия черновая) && (disposition == flagged), без взгляда на причину. Новая строка cancelled ровно такова. Следствие: остановка над черновиком одного члена помечает ВСЮ редакторскую единицу как «член выброшен» (export.go:327 droppedAny → DispFlagged), то есть выносится вердикт о ТЕКСТЕ там, где с текстом ничего не случилось. Это тот же самый частично правленный носитель, что в находке №2, в третьем его месте, и запрет «флаг, лгущий о своей причине» пак цитирует сам (cutcall.go:158, D39.93 п.2). Ни линз, ни тестов, ни мутаций на memberDrops