textmachine/backend/internal/pipeline/burnedpregates_test.go

1193 lines
54 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package pipeline
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"textmachine/backend/internal/chunk"
"textmachine/backend/internal/config"
"textmachine/backend/internal/llm"
"textmachine/backend/internal/obs"
"textmachine/backend/internal/store"
"textmachine/backend/internal/terminology"
)
// burnedpregates_test.go: THE PAID-CALL CONTRACT read before the funnel — who asks «is this call already
// paid for», what a BURNED key does to their answer, and what the answer is allowed to decide.
//
// A burned checkpoint records money and NO result (a delivered call the engine cut short), so runAttempt
// cannot replay it — it walks past and buys the work again. A pre-gate that reads such a row as «already
// paid» therefore does not skip a purchase, it lets one through: the fresh call happens with the gate's
// own budget check jumped over. The bank's probe learned this when cut calls started settling at the
// batch key; these two gates ask the same question about repair and about the escalation hop.
//
// Each pair is deliberate. The first fixture holds the property (a burned key must not buy), the second
// holds its opposite (a real key must still replay for free) — because a predicate that simply answered
// «not paid» to everything would pass the first alone while turning the free-replay contract off.
// gateServer is this file's own provider stub. It exists rather than reusing cutServer because these
// fixtures need to tell the REPAIR call apart from every other one — the whole assertion is «how many
// repair calls were bought» — and because a handler that must both read the body and then hold the
// connection cannot share a helper that drains the body before it is handed the request.
type gateServer struct {
mu sync.Mutex
// counted is how many calls of the kind THIS fixture is about have landed — repair calls for
// newGateServer, hop calls for newHopServer. One field, because a fixture watches one gate.
counted int
srv *httptest.Server
}
// newGateServer answers every non-repair call with a draft carrying the defect the repair class detects,
// and hands the repair call to `onRepair` — with its 1-based ordinal, so a fixture can make the first
// call behave differently from the ones that follow it.
func newGateServer(t *testing.T, onRepair func(i int, w http.ResponseWriter, req *http.Request)) *gateServer {
t.Helper()
gs := &gateServer{}
gs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
body, _ := io.ReadAll(req.Body)
if !isRepairBody(string(body)) {
answerJSON(w, "Он прождал полчаса и вошёл внутрь.")
return
}
gs.mu.Lock()
gs.counted++
i := gs.counted
gs.mu.Unlock()
onRepair(i, w, req)
}))
t.Cleanup(gs.srv.Close)
return gs
}
func (g *gateServer) repairCalls() int { g.mu.Lock(); defer g.mu.Unlock(); return g.counted }
// newHopServer is the escalation twin of newGateServer: every primary-model call is answered with a
// draft, and only the call addressed to the FALLBACK model — the hop — is counted and handed to the
// fixture. Counting by model rather than by ordinal matters: the primary's own attempts share the
// connection and would otherwise be indistinguishable from the hop in the tally.
func newHopServer(t *testing.T, onHop func(i int, w http.ResponseWriter, req *http.Request)) *gateServer {
t.Helper()
gs := &gateServer{}
gs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
body, _ := io.ReadAll(req.Body)
if !isHopBody(string(body)) {
answerJSON(w, "ЧЕРНОВИК ПЕРЕВОДА.")
return
}
gs.mu.Lock()
gs.counted++
i := gs.counted
gs.mu.Unlock()
onHop(i, w, req)
}))
t.Cleanup(gs.srv.Close)
return gs
}
func (g *gateServer) hopCalls() int { return g.repairCalls() }
// answerJSON writes one ordinary finished reply.
func answerJSON(w http.ResponseWriter, text string) {
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%q},"finish_reason":"stop"}],
"usage":{"prompt_tokens":1000,"completion_tokens":500}}`, text)
}
// repairBudgetLine is the pipeline.yaml line the fixtures rewrite to exhaust the repair sub-budget
// between the two runs. It is matched exactly so a change to repairGateYAML breaks this loudly instead
// of silently leaving the budget wide open and the assertions vacuous.
const repairBudgetLine = " budget_usd: 1.0\n"
// exhaustRepairBudget rewrites the fixture's repair budget to a value no call can fit under, so the ONLY
// thing that can still produce a repair call is a pre-gate answering «already paid».
func exhaustRepairBudget(t *testing.T, bookPath string) {
t.Helper()
p := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
raw, err := os.ReadFile(p)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), repairBudgetLine) {
t.Fatalf("premise broken: %s no longer carries %q, so this fixture would run with an unbounded "+
"repair budget and prove nothing", p, strings.TrimSpace(repairBudgetLine))
}
writeFile(t, p, strings.Replace(string(raw), repairBudgetLine, " budget_usd: 0.0000001\n", 1))
}
// assertRepairBudgetExhausted fails LOUDLY when the fixture's own premise is not in place. Without it a
// broken exhaust-helper leaves the burned fixtures accusing the pre-gate of buying past the sub-budget
// when the budget was simply never exhausted — the right colour under the wrong text, which the next
// shift reads as a money defect and goes to fix in repair.go.
func assertRepairBudgetExhausted(t *testing.T, r *Runner, spent float64) {
t.Helper()
if b := r.Pipeline.Gates.Repair.BudgetUSD; spent <= b {
t.Fatalf("premise broken: the repair sub-budget is NOT exhausted (spent %.6f, budget %.6f), so "+
"nothing below is measuring what it claims", spent, b)
}
}
// repairJob sets up the snapshot and job a direct maybeRepair call needs, and returns the edit stage it
// runs over. maybeRepair is called directly rather than through TranslateBook because the property is
// about ONE gate's decision, and a whole-book run would reach it through a resume path that has its own
// short-circuits.
func repairJob(t *testing.T, r *Runner) (config.Stage, chunk.Chunk, string, *store.Job) {
t.Helper()
st := r.Pipeline.Stages[len(r.Pipeline.Stages)-1]
if st.Name != "edit" {
t.Fatalf("premise broken: the repair gate runs on the SHIPPING stage; the fixture's last stage is %q", st.Name)
}
ch := chunk.Chunk{Chapter: 1, ChunkIdx: 0, Text: "他等了半个时辰。"}
const snapID = "snapshot-under-test"
if err := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); err != nil {
t.Fatal(err)
}
job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID)
if err != nil {
t.Fatal(err)
}
// The waves build the provider clients eagerly before any worker runs; a direct call into one stage
// has to do the same or the first request finds no client and never reaches the wire — which would
// leave every «no call was made» assertion below true for the wrong reason.
if err := r.buildClients(); err != nil {
t.Fatal(err)
}
return st, ch, snapID, job
}
// TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget is the money property of repair.go's pre-gate.
//
// The sequence is the production one, not a hand-written row: the first run's repair call IS delivered
// and IS cut (the operator stops the run while it is in flight), so the engine settles it at the repair
// request's own key — money, no text. The second run then finds that key with the sub-budget already
// exhausted. `repairCheckpointExists` answering «paid» skips the budget comparison entirely and walks
// into runRepairAttempt, which cannot replay a burned row and buys the repair again — a fresh paid call
// the only budget bounding repair spend never saw.
func TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget(t *testing.T) {
// ⛔ THE BURN MUST BE A BILLABLE ONE, and how it is produced is the fixture's only delicate part. Our
// money predicate charges a cut call only once the provider has ACKNOWLEDGED it with a reply, so a
// server that merely holds a silent connection leaves a $0 row — real, but not the row this gate is
// about. The handler therefore answers 200, flushes the first bytes of a body, and kills the socket:
// money owed for a generation, no result to show for it.
//
// The cause is the CONNECTION rather than a stop button on purpose. An externally-timed cancel has to
// be fired after the client has parsed the response headers, and the handler cannot know when that
// happened — flushing only pushes bytes at the socket. Measured on the first draft of this fixture:
// 2 runs in 5 cancelled early enough that the call read as unanswered and settled for $0, which would
// have made the assertions below pass while measuring nothing. A server-driven drop needs no
// synchronisation at all, and `connection_lost` burns the key exactly the same way.
//
// Only the FIRST run's calls are dropped. A later one is answered, so that if the gate does buy a
// repair it cannot afford, the fixture fails on the MONEY assertion — «a call was bought outside the
// sub-budget» — rather than on whatever the transport happened to say about the second drop.
dropUntil := 0
srv := newGateServer(t, func(i int, w http.ResponseWriter, _ *http.Request) {
if dropUntil == 0 || i <= dropUntil {
dropAfterHeaders(t, w)
return
}
answerJSON(w, "Он прождал час и вошёл внутрь.")
})
bookPath := setupRepairProject(t, srv.srv.URL)
// Run 1: the repair call is cut in flight, so its key holds money and no result.
r1 := newRunner(t, bookPath)
st, ch, snapID, job := repairJob(t, r1)
const drafted = "Он прождал полчаса и вошёл внутрь."
_, _, mrErr := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil)
if mrErr == nil {
t.Fatal("premise broken: the repair call was supposed to be cut in flight, and it returned cleanly")
}
burned, err := r1.Store.RepairSpentUSD(r1.Book.BookID)
if err != nil {
t.Fatal(err)
}
if burned <= 0 {
t.Fatalf("premise broken: a cut repair call must be SETTLED at the repair key — that row is the "+
"whole subject of this fixture. repair spend %.6f", burned)
}
// The count is recorded, not asserted to be one: a lost connection is retried once inside the
// transport, so run 1 legitimately asks twice under ONE request key and settles ONE row. What the
// second run must add is zero, and that is the assertion below.
repairsAfterRun1 := srv.repairCalls()
if repairsAfterRun1 == 0 {
t.Fatal("premise broken: run 1 never reached the provider, so there is no burned key to test")
}
dropUntil = repairsAfterRun1 // from here on the provider answers; only the money may still object
r1.Close()
// Run 2: the sub-budget can no longer fit a call. Nothing may buy one.
exhaustRepairBudget(t, bookPath)
r2 := newRunner(t, bookPath)
defer r2.Close()
st2, ch2, snapID2, job2 := repairJob(t, r2)
assertRepairBudgetExhausted(t, r2, burned)
if _, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil); err != nil {
t.Fatalf("a repair the budget cannot afford is SKIPPED, not an error: %v", err)
}
if got := srv.repairCalls(); got != repairsAfterRun1 {
t.Fatalf("a burned key is money with NO result: runAttempt cannot replay it and buys the repair "+
"again, so a pre-gate answering «already paid» lets that purchase jump the sub-budget check. "+
"want %d repair calls, got %d — one of them was bought outside gates.repair.budget_usd",
repairsAfterRun1, got)
}
spent, err := r2.Store.RepairSpentUSD(r2.Book.BookID)
if err != nil {
t.Fatal(err)
}
if spent != burned {
t.Fatalf("the exhausted sub-budget still moved: %.6f → %.6f", burned, spent)
}
}
// TestARealRepairKeyStillReplaysFreeOnAnExhaustedBudget is the other half, and it is what stops the fix
// above from being «answer no to everything». An ANSWERED repair key must replay for $0 even when the
// sub-budget is gone: the money was already spent, the bytes already exist, and re-deciding them would
// make a resumed run ship different text than the run that paid for it.
func TestARealRepairKeyStillReplaysFreeOnAnExhaustedBudget(t *testing.T) {
srv := newGateServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
answerJSON(w, "Он прождал час и вошёл внутрь.")
})
bookPath := setupRepairProject(t, srv.srv.URL)
r1 := newRunner(t, bookPath)
st, ch, snapID, job := repairJob(t, r1)
const drafted = "Он прождал полчаса и вошёл внутрь."
fixed, _, err := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(fixed, "час") || strings.Contains(fixed, "полчаса") {
t.Fatalf("premise broken: run 1 must have APPLIED a repair, got %q", fixed)
}
paidSpend, err := r1.Store.RepairSpentUSD(r1.Book.BookID)
if err != nil || paidSpend <= 0 {
t.Fatalf("premise broken: the answered repair must be booked, got %.6f %v", paidSpend, err)
}
repairsAfterRun1 := srv.repairCalls()
r1.Close()
exhaustRepairBudget(t, bookPath)
r2 := newRunner(t, bookPath)
defer r2.Close()
st2, ch2, snapID2, job2 := repairJob(t, r2)
assertRepairBudgetExhausted(t, r2, paidSpend)
again, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil)
if err != nil {
t.Fatal(err)
}
if again != fixed {
t.Fatalf("an already-paid repair must re-serve the SAME bytes:\n first: %q\nsecond: %q", fixed, again)
}
if got := srv.repairCalls(); got != repairsAfterRun1 {
t.Fatalf("an already-paid repair replays for $0 and asks nobody: want %d repair calls, got %d",
repairsAfterRun1, got)
}
if spent, serr := r2.Store.RepairSpentUSD(r2.Book.BookID); serr != nil || spent != paidSpend {
t.Fatalf("a free replay must not move the sub-budget spend: %.6f → %.6f (%v)", paidSpend, spent, serr)
}
}
// exhaustEscalationBudget rewrites the fixture's escalation budget below what run 1 already spent, so
// escalationBudgetRemains() answers «no» and the ONLY thing that can still buy a hop is the idempotency
// probe answering «already paid».
func exhaustEscalationBudget(t *testing.T, bookPath string) {
t.Helper()
p := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
raw, err := os.ReadFile(p)
if err != nil {
t.Fatal(err)
}
i := strings.Index(string(raw), "escalation: { budget_usd:")
if i < 0 {
t.Fatalf("premise broken: %s carries no escalation budget line, so this fixture would run with the "+
"gate wide open and prove nothing", p)
}
j := strings.Index(string(raw)[i:], "\n")
writeFile(t, p, string(raw)[:i]+"escalation: { budget_usd: 0.0000001 }"+string(raw)[i+j:])
}
// isHopBody reports whether a mock request is the ESCALATION hop: it is the only call addressed to the
// fallback model, which the wire body names.
func isHopBody(body string) bool { return strings.Contains(body, `"model":"fake-fallback"`) }
// escalationJob prepares a direct maybeEscalate call over the draft stage, which is the one carrying
// `escalate_to` in the fixture.
func escalationJob(t *testing.T, r *Runner) (config.Stage, chunk.Chunk, string, *store.Job) {
t.Helper()
st := r.Pipeline.Stages[0]
if st.ResolvedHop == "" {
t.Fatalf("premise broken: stage %q has no resolved hop, so maybeEscalate returns before it decides "+
"anything and every assertion below would be vacuous", st.Name)
}
ch := chunk.Chunk{Chapter: 1, ChunkIdx: 0, Text: "静かな図書館の朝。"}
const snapID = "snapshot-under-test"
if err := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); err != nil {
t.Fatal(err)
}
job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID)
if err != nil {
t.Fatal(err)
}
if err := r.buildClients(); err != nil {
t.Fatal(err)
}
return st, ch, snapID, job
}
// hopMessages is the prompt both runs hand maybeEscalate. It is a constant of the fixture because the
// hop's request hash is derived from it: two runs that differed here would address two different keys
// and the second would find nothing, passing for the wrong reason.
var hopMessages = []llm.Message{{Role: "user", Content: "静かな図書館の朝。"}}
const hopBaseMaxTokens = 512
// TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget is the money property of escalation.go's
// idempotency probe, and it has a second half the repair gate does not: the fresh-hop branch is also
// where escMu is taken. A burned key read as «already paid» skips BOTH — the budget check and the
// serialisation — so the hop it then buys is unbounded AND unserialised against the other draft workers.
func TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget(t *testing.T) {
dropUntil := 0
srv := newHopServer(t, func(i int, w http.ResponseWriter, _ *http.Request) {
if dropUntil == 0 || i <= dropUntil {
dropAfterHeaders(t, w) // delivered, acknowledged, then the socket dies: money, no result
return
}
answerJSON(w, "Тихое утро в библиотеке.")
})
bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0)
r1 := newRunner(t, bookPath)
st, ch, snapID, job := escalationJob(t, r1)
primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}}
if _, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false); err == nil {
t.Fatal("premise broken: the hop was supposed to be cut by the connection, and it returned cleanly")
}
burned, err := r1.Store.EscalationSpentUSD(r1.Book.BookID)
if err != nil {
t.Fatal(err)
}
if burned <= 0 {
t.Fatalf("premise broken: a cut hop must be SETTLED at the hop key — that row is the subject of "+
"this fixture. escalation spend %.6f", burned)
}
hopsAfterRun1 := srv.hopCalls()
if hopsAfterRun1 == 0 {
t.Fatal("premise broken: run 1 never reached the provider, so there is no burned key to test")
}
dropUntil = hopsAfterRun1
r1.Close()
exhaustEscalationBudget(t, bookPath)
r2 := newRunner(t, bookPath)
defer r2.Close()
st2, ch2, snapID2, job2 := escalationJob(t, r2)
if remains, rerr := r2.escalationBudgetRemains(); rerr != nil || remains {
t.Fatalf("premise broken: the escalation budget must be exhausted for run 2, remains=%t err=%v", remains, rerr)
}
if _, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false); err != nil {
t.Fatalf("a hop the budget cannot afford is SKIPPED, not an error: %v", err)
}
if got := srv.hopCalls(); got != hopsAfterRun1 {
t.Fatalf("a burned key is money with NO result: runAttempt cannot replay it and buys the hop again, "+
"so a probe answering «already paid» sends that purchase past escalation.budget_usd AND past "+
"escMu. want %d hop calls, got %d", hopsAfterRun1, got)
}
if spent, serr := r2.Store.EscalationSpentUSD(r2.Book.BookID); serr != nil || spent != burned {
t.Fatalf("the exhausted escalation budget still moved: %.6f → %.6f (%v)", burned, spent, serr)
}
}
// TestARealHopKeyStillReplaysFreeOnAnExhaustedBudget is the other half. escalation.go's doccomment
// promises a paid hop replays «regardless of the budget» — a crash between the hop's settle and the
// chunk_status write must not discard a paid, successful translation and flip the verdict OK→flagged.
// Teaching the probe about burned keys must not cost that promise.
func TestARealHopKeyStillReplaysFreeOnAnExhaustedBudget(t *testing.T) {
srv := newHopServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
answerJSON(w, "Тихое утро в библиотеке.")
})
bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0)
r1 := newRunner(t, bookPath)
st, ch, snapID, job := escalationJob(t, r1)
primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}}
out1, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false)
if err != nil {
t.Fatal(err)
}
if !out1.attempted {
t.Fatal("premise broken: run 1 must have made the hop")
}
paid, err := r1.Store.EscalationSpentUSD(r1.Book.BookID)
if err != nil || paid <= 0 {
t.Fatalf("premise broken: the answered hop must be booked, got %.6f %v", paid, err)
}
hopsAfterRun1 := srv.hopCalls()
r1.Close()
exhaustEscalationBudget(t, bookPath)
r2 := newRunner(t, bookPath)
defer r2.Close()
st2, ch2, snapID2, job2 := escalationJob(t, r2)
if remains, rerr := r2.escalationBudgetRemains(); rerr != nil || remains {
t.Fatalf("premise broken: the escalation budget must be exhausted for run 2, remains=%t err=%v", remains, rerr)
}
out2, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false)
if err != nil {
t.Fatal(err)
}
if !out2.attempted {
t.Fatal("an already-paid hop must still be REPLAYED on an exhausted budget: discarding it flips a " +
"paid, successful translation to flagged on every resume")
}
if out2.fb.text != out1.fb.text {
t.Fatalf("the replayed hop must re-serve the SAME bytes:\n first: %q\nsecond: %q", out1.fb.text, out2.fb.text)
}
if got := srv.hopCalls(); got != hopsAfterRun1 {
t.Fatalf("an already-paid hop replays for $0 and asks nobody: want %d hop calls, got %d", hopsAfterRun1, got)
}
if spent, serr := r2.Store.EscalationSpentUSD(r2.Book.BookID); serr != nil || spent != paid {
t.Fatalf("a free replay must not move the escalation spend: %.6f → %.6f (%v)", paid, spent, serr)
}
}
// TestAPaidHopBehindABurnedKeyStillReplaysFree is the case both halves of the pair above miss, and it is
// the one that makes the probe's SHAPE matter rather than its answer.
//
// The funnel does not stop at a burned key — it walks to the next attempt index at the same budget and
// buys there. So after a stopped run the position looks like this: attempt 0 burned, attempt 1 PAID and
// answered. A probe that asks about a FIXED index sees only the burn, answers «not paid», and hands the
// decision to the budget — which, being exhausted, discards a translation that was already bought. The
// probe must ask what the FUNNEL will ask: walk the burns, then look.
//
// Three runs, because the state needs two of them to exist: run 1 burns attempt 0, run 2 buys the answer
// at attempt 1, run 3 arrives with no budget left and must still serve it for $0.
func TestAPaidHopBehindABurnedKeyStillReplaysFree(t *testing.T) {
dropFirst := true
srv := newHopServer(t, func(i int, w http.ResponseWriter, _ *http.Request) {
if dropFirst {
dropAfterHeaders(t, w) // run 1: money, no result — the key at attempt 0 is burned
return
}
answerJSON(w, "Тихое утро в библиотеке.")
})
bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0)
primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}}
r1 := newRunner(t, bookPath)
st, ch, snapID, job := escalationJob(t, r1)
if _, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false); err == nil {
t.Fatal("premise broken: run 1's hop was supposed to be cut by the connection")
}
r1.Close()
// Run 2: the budget is still there, so the funnel walks past the burn and buys the answer one index up.
dropFirst = false
r2 := newRunner(t, bookPath)
st2, ch2, snapID2, job2 := escalationJob(t, r2)
out2, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false)
if err != nil {
t.Fatal(err)
}
if !out2.attempted || out2.fb.text == "" {
t.Fatalf("premise broken: run 2 must have BOUGHT the hop past the burn, got attempted=%t text=%q", out2.attempted, out2.fb.text)
}
paid, err := r2.Store.EscalationSpentUSD(r2.Book.BookID)
if err != nil || paid <= 0 {
t.Fatalf("premise broken: run 2's hop must be booked, got %.6f %v", paid, err)
}
hopsAfterRun2 := srv.hopCalls()
r2.Close()
// Run 3: no budget left. The hop was PAID and ANSWERED — discarding it now throws away bought work
// and flips the unit's verdict on every later resume.
exhaustEscalationBudget(t, bookPath)
r3 := newRunner(t, bookPath)
defer r3.Close()
st3, ch3, snapID3, job3 := escalationJob(t, r3)
if remains, rerr := r3.escalationBudgetRemains(); rerr != nil || remains {
t.Fatalf("premise broken: the escalation budget must be exhausted for run 3, remains=%t err=%v", remains, rerr)
}
out3, err := r3.maybeEscalate(context.Background(), st3, snapID3, ch3, job3, hopBaseMaxTokens, hopMessages, primary, false)
if err != nil {
t.Fatal(err)
}
if !out3.attempted || out3.fb.text != out2.fb.text {
t.Fatalf("A PAID, SUCCESSFUL HOP WAS DISCARDED because a BURNED key sits in front of it. The probe "+
"asks a fixed attempt index; the funnel walks burns and answers one index up, so «is this paid» "+
"must be asked the funnel's way. run 2 bought %q, run 3 served attempted=%t %q",
out2.fb.text, out3.attempted, out3.fb.text)
}
// The control: serving it must have cost nothing — otherwise this test would pass by re-buying.
if got := srv.hopCalls(); got != hopsAfterRun2 {
t.Fatalf("the replay must ask nobody: want %d hop calls, got %d", hopsAfterRun2, got)
}
if spent, serr := r3.Store.EscalationSpentUSD(r3.Book.BookID); serr != nil || spent != paid {
t.Fatalf("a free replay must not move the escalation spend: %.6f → %.6f (%v)", paid, spent, serr)
}
}
// TestAPaidRepairBehindABurnedKeyStillReplaysFree is the repair twin of the hop case above. Its own
// contract is stated at repair.go's budget block: an already-paid repair replays regardless of the
// remaining budget, because a crash between the paid call and the chunk_status write would otherwise
// make the resumed run ship DIFFERENT bytes than the run that paid for them.
func TestAPaidRepairBehindABurnedKeyStillReplaysFree(t *testing.T) {
dropFirst := true
srv := newGateServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
if dropFirst {
dropAfterHeaders(t, w) // run 1: the repair key at the candidate's own index is burned
return
}
answerJSON(w, "Он прождал час и вошёл внутрь.")
})
bookPath := setupRepairProject(t, srv.srv.URL)
const drafted = "Он прождал полчаса и вошёл внутрь."
r1 := newRunner(t, bookPath)
st, ch, snapID, job := repairJob(t, r1)
if _, _, err := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil); err == nil {
t.Fatal("premise broken: run 1's repair was supposed to be cut by the connection")
}
r1.Close()
dropFirst = false
r2 := newRunner(t, bookPath)
st2, ch2, snapID2, job2 := repairJob(t, r2)
fixed, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(fixed, "час") || strings.Contains(fixed, "полчаса") {
t.Fatalf("premise broken: run 2 must have BOUGHT the repair past the burn, got %q", fixed)
}
paid, err := r2.Store.RepairSpentUSD(r2.Book.BookID)
if err != nil || paid <= 0 {
t.Fatalf("premise broken: run 2's repair must be booked, got %.6f %v", paid, err)
}
repairsAfterRun2 := srv.repairCalls()
r2.Close()
exhaustRepairBudget(t, bookPath)
r3 := newRunner(t, bookPath)
defer r3.Close()
st3, ch3, snapID3, job3 := repairJob(t, r3)
assertRepairBudgetExhausted(t, r3, paid)
again, _, err := r3.maybeRepair(context.Background(), st3, snapID3, ch3, job3, drafted, drafted, nil)
if err != nil {
t.Fatal(err)
}
if again != fixed {
t.Fatalf("A PAID REPAIR WAS DISCARDED because a BURNED key sits in front of it, so the resumed run "+
"ships DIFFERENT bytes than the run that paid:\n paid for: %q\n now ships: %q", fixed, again)
}
if got := srv.repairCalls(); got != repairsAfterRun2 {
t.Fatalf("the replay must ask nobody: want %d repair calls, got %d", repairsAfterRun2, got)
}
if spent, serr := r3.Store.RepairSpentUSD(r3.Book.BookID); serr != nil || spent != paid {
t.Fatalf("a free replay must not move the sub-budget spend: %.6f → %.6f (%v)", paid, spent, serr)
}
}
// TestTheBankProbeFindsThePaidBatchBehindABurnedKey is the third member of the family, pinned directly
// rather than end-to-end: the bank probe is a pure function of the store, and the two runs the other two
// fixtures need exist here only to create rows. What it holds is the same contract — a burn in front of
// an answer must not hide the answer.
func TestTheBankProbeFindsThePaidBatchBehindABurnedKey(t *testing.T) {
dir := t.TempDir()
srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { answerWhole(w) })
bookPath := setupCutProject(t, dir, srv.srv.URL)
r, err := NewRunner(bookPath, obs.NewLogger())
if err != nil {
t.Fatal(err)
}
defer r.Close()
st := r.Pipeline.Stages[0]
ch := chunk.Chunk{Chapter: 0, ChunkIdx: 0}
msgs := []llm.Message{{Role: "user", Content: "термины"}}
const snapID = "snapshot-under-test"
if serr := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); serr != nil {
t.Fatal(serr)
}
job, jerr := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID)
if jerr != nil {
t.Fatal(jerr)
}
_, maxTokens := r.bankCallBudget(st.Model, msgs)
hashAt := func(attempt int) string {
return RequestHash(r.attemptRequest(st, st.Model, snapID, ch, attempt, maxTokens, msgs))
}
write := func(hash, finish, text string) {
t.Helper()
resv, verdict, rerr := r.Store.Reserve(r.Book.BookID, 0.001, store.Ceilings{BookUSD: 100, DayUSD: 100})
if rerr != nil || verdict != store.ReserveOK {
t.Fatalf("reserve: %v %v", verdict, rerr)
}
if serr := r.Store.SettleWithCheckpoint(resv, 0.001, store.Checkpoint{
RequestHash: hash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: 0,
Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: st.Model,
ResponseText: text, UsageJSON: "{}", CostUSD: 0.001, FinishReason: finish,
}, nil); serr != nil {
t.Fatalf("settle: %v", serr)
}
}
// The control first: with only the burn there, the batch is NOT paid — the same assertion the probe's
// own fixture makes, repeated here so a probe that answered «paid» to everything would fail before
// reaching the case this test is about.
write(hashAt(0), cancelledFinish, "")
if paid, perr := r.bankCheckpointExists(st, snapID, ch, msgs); perr != nil || paid {
t.Fatalf("a lone burned key is not a paid batch: paid=%t err=%v", paid, perr)
}
// And now the answer the funnel bought one index up, exactly where a stopped run leaves it.
write(hashAt(1), "stop", "термин\tterm")
if paid, perr := r.bankCheckpointExists(st, snapID, ch, msgs); perr != nil || !paid {
t.Fatalf("the batch WAS bought and answered at the index the funnel walked to; a probe that only "+
"looks at the starting key hides it, and the role sub-budget then refuses to serve work it "+
"already paid for. paid=%t err=%v", paid, perr)
}
}
// TestAnUnaffordableBatchDoesNotTakeThePaidBatchesBehindIt pins the admission SHAPE of the bank pass.
//
// Admission is not a prefix. An already-paid batch costs nothing and is admitted whatever the budget
// says — the loop's own comment promises exactly that — so a batch the budget refuses can sit in FRONT of
// batches that are free to serve. While the decision was a prefix bound, one refused batch took every
// paid batch behind it: their replay was $0, their result was already bought, and dropping them bought
// nothing while losing a consolidated bank.
func TestAnUnaffordableBatchDoesNotTakeThePaidBatchesBehindIt(t *testing.T) {
dir := t.TempDir()
srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { answerWhole(w) })
bookPath := setupCutProject(t, dir, srv.srv.URL)
r, err := NewRunner(bookPath, obs.NewLogger())
if err != nil {
t.Fatal(err)
}
defer r.Close()
const snapID = "snapshot-under-test"
if serr := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); serr != nil {
t.Fatal(serr)
}
st := r.bankStage(roleTerminologist)
job, jerr := r.Store.EnsureJob(r.Book.BookID, 0, st.Name, snapID)
if jerr != nil {
t.Fatal(jerr)
}
// Two batches. The messages function is the fixture's own, so the request identity below is built the
// same way the pass builds it — re-deriving it any other way would address a different key and this
// test would pass by measuring nothing.
batches := [][]terminology.Candidate{{{Key: "甲", Src: "甲"}}, {{Key: "乙", Src: "乙"}}}
msgsOf := func(b []terminology.Candidate) ([]llm.Message, error) {
return []llm.Message{{Role: "user", Content: "batch " + b[0].Key}}, nil
}
plan := bankRolePlan{role: roleTerminologist, budgetUSD: 0.0000001, messages: msgsOf}
// Batch 1 is ALREADY PAID: a real answered checkpoint at its own key. Batch 0 is not, and the budget
// cannot fit it — so batch 0 is refused and batch 1 must still be served for $0.
msgs1, _ := msgsOf(batches[1])
_, maxTokens := r.bankCallBudget(st.Model, msgs1)
hash1 := RequestHash(r.attemptRequest(st, st.Model, snapID, chunk.Chunk{Chapter: 0, ChunkIdx: 1}, 0, maxTokens, msgs1))
resv, verdict, rerr := r.Store.Reserve(r.Book.BookID, 0.001, store.Ceilings{BookUSD: 100, DayUSD: 100})
if rerr != nil || verdict != store.ReserveOK {
t.Fatalf("reserve: %v %v", verdict, rerr)
}
if serr := r.Store.SettleWithCheckpoint(resv, 0.001, store.Checkpoint{
RequestHash: hash1, JobID: job.ID, ChunkIdx: 1, Attempt: 0,
Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: st.Model,
ResponseText: "乙\tvtoroy", UsageJSON: "{}", CostUSD: 0.001, FinishReason: "stop",
}, nil); serr != nil {
t.Fatalf("settle: %v", serr)
}
if err := r.buildClients(); err != nil {
t.Fatal(err)
}
callsBefore := srv.calls()
run, err := r.runBankRoleBatches(context.Background(), snapID, plan, batches, "render")
if err != nil {
t.Fatal(err)
}
// The premise: batch 0 really was refused, or the assertion below would hold for the wrong reason.
if run.ran[0] {
t.Fatalf("premise broken: batch 0 was supposed to be unaffordable under a budget of %g", plan.budgetUSD)
}
if !run.ran[1] || run.texts[1] == "" {
t.Fatalf("an ALREADY-PAID batch sitting behind a refused one was dropped. Its replay costs $0 and "+
"its result was already bought, so refusing it buys nothing and loses a consolidated bank. "+
"ran=%v texts=%q dropped=%d", run.ran, run.texts, run.dropped)
}
// And the control: serving it asked the provider nothing.
if got := srv.calls(); got != callsBefore {
t.Fatalf("a paid batch must replay for $0: the provider was asked %d times", got-callsBefore)
}
if run.dropped != 1 {
t.Fatalf("exactly one batch was refused, so the report must say one: dropped=%d", run.dropped)
}
}
// assertRowsMatchTheLedger holds the invariant this family is about: a position's chunk_status row
// records what that position COST, and the checkpoints of that position are what it cost. The two are
// written by different code on different exits — the funnel, the escalation hop, the repair sub-step,
// the cut settle — and every one of them can walk out early. A row below its own ledger is money the
// book spent and the projection a person decides on does not show.
//
// Bank roles are excluded because they write no chunk_status at all (synthetic stage, chapter 0): they
// are a different accounting axis, and counting them here would compare a row against a sum that
// includes rows it never claimed.
func assertRowsMatchTheLedger(t *testing.T, bookPath string) {
t.Helper()
m := readMoney(t, bookPath)
type key struct {
ch, chunk int
stage string
}
ledger := map[key]float64{}
bankRoles := map[string]bool{roleTerminologist: true, roleClassifier: true}
counted := 0
for _, u := range m.checkpoints {
if bankRoles[u.Role] {
continue
}
ledger[key{u.Chapter, u.ChunkIdx, u.Stage}] += u.CostUSD
counted++
}
// The control: the instrument must have been given something to compare. «No mismatch» over an empty
// ledger and «no mismatch» over a real one read identically in a pass.
if counted == 0 || len(m.statuses) == 0 {
t.Fatalf("premise broken: nothing to compare — %d checkpoints outside the bank roles, %d status rows",
counted, len(m.statuses))
}
const cent = 0.0000005 // half a micro-dollar: the columns are float64 sums of the same values
for _, cs := range m.statuses {
// A row that cost money must say how many attempts bought it. `attempts=0` beside a non-zero
// cost is a row that reports work nobody did — measured at 0 attempts against $0.001056 — and it
// is the number an operator reads to tell «one call was cut» from «the position never started».
if cs.CostUSD > cent && cs.Attempts == 0 {
t.Fatalf("chunk_status ch%d/chunk%d/%s cost %.6f and reports attempts=0: the count is written "+
"after the error check, so a position that PAID reads as one that never started",
cs.Chapter, cs.ChunkIdx, cs.Stage, cs.CostUSD)
}
want := ledger[key{cs.Chapter, cs.ChunkIdx, cs.Stage}]
if diff := cs.CostUSD - want; diff > cent || diff < -cent {
t.Fatalf("chunk_status ch%d/chunk%d/%s says it cost %.6f while its own checkpoints add up to "+
"%.6f. The row is what the projection reads; money that reaches the ledger and not the row "+
"is money the book spent and nobody is shown.",
cs.Chapter, cs.ChunkIdx, cs.Stage, cs.CostUSD, want)
}
}
t.Logf("ledger invariant held over %d rows against %d checkpoints", len(m.statuses), counted)
}
// TestAStoppedRunLeavesEveryRowMatchingItsOwnLedger is the family's contract, pinned end to end: the
// position's money comes from settles written by different code on different exits, and the row that
// records it is written by a third piece on an error exit.
//
// ⚠ THE HOP'S MONEY IS MADE INEVITABLE RATHER THAN RACED FOR, and the first draft of this fixture got
// that wrong. It stopped the run over a hop held on the wire and relied on the stop landing AFTER the
// client had parsed the hop's headers — only an acknowledged call is billable. Measured on the mutated
// tree, where it must fail every time: **2 reds in 8 runs**. The other six the stop won the race, the cut
// settled for $0, and the row matched the ledger at nothing to nothing — a pin that passed by measuring
// an empty scenario.
//
// So the money is put there BEFORE the race can matter: run 1 burns the hop's key (two drops — one is
// retried inside the transport), and run 2's funnel walks that burn and carries its cost whatever
// happens to the fresh call. Whether the stop beats the headers or not, `esc.fb.cumCost` is non-zero,
// and the only question left is the one being asked — does that money reach the row.
func TestAStoppedRunLeavesEveryRowMatchingItsOwnLedger(t *testing.T) {
dir := t.TempDir()
var hopCalls atomic.Int32
hopHeld := make(chan struct{})
var once sync.Once
var srv *cutServer
srv = newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) {
if strings.Contains(lastBody(srv), `"fake-hop"`) {
if hopCalls.Add(1) <= 2 {
dropAfterHeaders(t, w) // run 1: two drops burn the hop's key — money, no result
return
}
once.Do(func() { close(hopHeld) }) // run 2: held, and the run is stopped over it
hold(req)
return
}
// The primary draft: a content filter — deterministic and escalatable, so a hop follows.
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"нет"},"finish_reason":"content_filter"}],
"usage":{"prompt_tokens":100,"completion_tokens":10}}`)
})
bookPath := setupHopProject(t, dir, srv.srv.URL)
if err := runOnce(t, context.Background(), bookPath); err == nil {
t.Fatal("premise broken: run 1's hop was supposed to end on a delivered cut and burn its key")
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
<-hopHeld
cancel()
}()
_ = runOnce(t, ctx, bookPath)
// The premise, and it is what the first draft lacked: there IS hop money on this position, so the
// comparison below is not nothing against nothing.
m := readMoney(t, bookPath)
hopUSD := 0.0
for _, u := range m.checkpoints {
if u.Escalation {
hopUSD += u.CostUSD
}
}
if hopUSD <= 0 {
t.Fatalf("premise broken: the hop cost nothing, so this fixture is comparing an empty position. "+
"checkpoints=%d", len(m.checkpoints))
}
assertRowsMatchTheLedger(t, bookPath)
}
// TestABurnedKeysMoneyReachesTheRow is the other half of the same invariant, on the ordinary path. The
// funnel walks past a burned key and buys the work again one index up; the row must carry BOTH — the
// money that bought nothing and the money that bought the text. The walk's total was being overwritten
// by the fresh call's cost rather than added to it, so a position that was cut and then paid for read as
// if only the second call had happened.
func TestABurnedKeysMoneyReachesTheRow(t *testing.T) {
dir := t.TempDir()
var calls atomic.Int32
srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
// TWO drops, because one is not a burn: a lost connection is retried inside the transport, so a
// single drop followed by an answer never reaches the store as a cut. The delivered-cut cap makes
// the SECOND one terminal — that is what settles the key as money with no result.
if calls.Add(1) <= 2 {
dropAfterHeaders(t, w) // acknowledged, then cut
return
}
answerWhole(w)
})
bookPath := setupCutProject(t, dir, srv.srv.URL)
// Run 1 ends on the cut and leaves the burned key behind; run 2 walks past it and buys one index up.
if err := runOnce(t, context.Background(), bookPath); err == nil {
t.Fatal("premise broken: run 1 was supposed to end on a delivered cut")
}
if err := runOnce(t, context.Background(), bookPath); err != nil {
t.Fatalf("run 2 must complete: the burn is re-asked at the SAME budget one index up: %v", err)
}
m := readMoney(t, bookPath)
burned := 0
for _, u := range m.checkpoints {
if u.CostUSD > 0 && strings.TrimSpace(u.UsageJSON) == "{}" {
burned++
}
}
if burned == 0 {
t.Fatalf("premise broken: no burned checkpoint was written, so this fixture is measuring an "+
"ordinary run. checkpoints=%d", len(m.checkpoints))
}
assertRowsMatchTheLedger(t, bookPath)
}
// TestAStoppedRunCarriesTheRepairsMoneyToTheRow is the repair twin of the hop case. The repair sub-step
// runs INSIDE the shipping stage and returns its money together with its error, and the row that records
// the position is written on that error exit — so the same ordering defect lives here, and the lens that
// found it measured a ledger of 0.001303 against rows adding to 0.000240.
//
// The money is made inevitable the same way: run 1 burns the repair key, so run 2's walk carries that
// cost whatever the stop does to the fresh call.
func TestAStoppedRunCarriesTheRepairsMoneyToTheRow(t *testing.T) {
var repairCalls atomic.Int32
repairHeld := make(chan struct{})
var once sync.Once
var srv *cutServer
srv = newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) {
if isRepairBody(lastBody(srv)) {
if repairCalls.Add(1) <= 2 {
dropAfterHeaders(t, w) // run 1: two drops burn the repair's key
return
}
once.Do(func() { close(repairHeld) })
hold(req)
return
}
if isEditBody(lastBody(srv)) {
answerJSON(w, "Он прождал полчаса и вошёл внутрь.") // carries the defect the repair class detects
return
}
answerJSON(w, "ЧЕРНОВИК ПЕРЕВОДА.")
})
bookPath := setupRepairProject(t, srv.srv.URL)
if err := runOnce(t, context.Background(), bookPath); err == nil {
t.Fatal("premise broken: run 1's repair was supposed to end on a delivered cut and burn its key")
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
<-repairHeld
cancel()
}()
_ = runOnce(t, ctx, bookPath)
m := readMoney(t, bookPath)
repairUSD := 0.0
for _, u := range m.checkpoints {
if u.Role == roleRepair {
repairUSD += u.CostUSD
}
}
if repairUSD <= 0 {
t.Fatalf("premise broken: the repair cost nothing, so this fixture is comparing an empty position. "+
"checkpoints=%d", len(m.checkpoints))
}
assertRowsMatchTheLedger(t, bookPath)
}
// TestAStoppedPositionIsNotADecidedUnit pins the fourth carrier of the same question. A `cancelled` row
// records a position the engine was stopped over — the work was never done and the resume re-does it and
// pays again — so it decides nothing about the unit. Reading it as decided puts the unit into the
// extrapolation's denominator, and the projection then shows a book further along and cheaper than it
// is; the re-bill consent threshold, min($0.50, 5% × projected), shrinks with it.
//
// Both directions, because a predicate that answered «not decided» to everything would pass the first
// half alone and quietly turn the flagged state off for every real defect.
func TestAStoppedPositionIsNotADecidedUnit(t *testing.T) {
rows := func(reason FlagReason) []store.ChunkStatus {
return []store.ChunkStatus{
{Chapter: 1, ChunkIdx: 0, Stage: "draft", Disposition: string(DispOK), CostUSD: 0.001},
{Chapter: 1, ChunkIdx: 0, Stage: "edit", Disposition: string(DispFlagged),
FlagReason: string(reason), CostUSD: 0.002},
}
}
stopped := resolveChunkState(rows(FlagCancelled), 2)
if stopped.State == ChunkFlagged {
t.Fatalf("a stopped position decides nothing: the work was never done and the resume re-does it "+
"and pays again. Counting it as a decided unit puts it in the projection's denominator, so one "+
"stop makes the book look further along and cheaper than it is. got state=%v reason=%q",
stopped.State, stopped.Reason)
}
if stopped.State != ChunkInProgress {
t.Fatalf("a stopped position is still IN PROGRESS — that is what it is. got %v", stopped.State)
}
// The historical money is a fact and stays on the resolution whatever the state says.
if stopped.CostUSD <= 0 {
t.Fatalf("the money already spent on the position must still be reported: %.6f", stopped.CostUSD)
}
// The control: an ordinary flag still decides the unit, or this change has simply switched the
// flagged state off.
flagged := resolveChunkState(rows(FlagLength), 2)
if flagged.State != ChunkFlagged || flagged.Reason != string(FlagLength) {
t.Fatalf("a real content flag must still decide the unit: state=%v reason=%q", flagged.State, flagged.Reason)
}
}
// TestAStopOnAStagesFirstCallStillReportsAnAttempt is the count's own case. Every other fixture here has
// a call that succeeded before the stop, so the loop had already recorded a number; this one is stopped
// on the stage's FIRST call, which is where `attempts=0` beside real money was measured.
//
// The money is made inevitable the same way as the rest: run 1 burns the key, so run 2's walk carries a
// cost whatever the stop does to the fresh call.
func TestAStopOnAStagesFirstCallStillReportsAnAttempt(t *testing.T) {
dir := t.TempDir()
var calls atomic.Int32
held := make(chan struct{})
var once sync.Once
srv := newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) {
if calls.Add(1) <= 2 {
dropAfterHeaders(t, w) // run 1: two drops burn the first key
return
}
once.Do(func() { close(held) }) // run 2: held, and the run is stopped over it
hold(req)
})
bookPath := setupCutProject(t, dir, srv.srv.URL)
if err := runOnce(t, context.Background(), bookPath); err == nil {
t.Fatal("premise broken: run 1 was supposed to end on a delivered cut and burn the key")
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
<-held
cancel()
}()
_ = runOnce(t, ctx, bookPath)
m := readMoney(t, bookPath)
marked := false
for _, cs := range m.statuses {
if cs.FlagReason == string(FlagCancelled) {
marked = true
if cs.CostUSD <= 0 {
t.Fatalf("premise broken: the stopped position carries no money, so the count below proves "+
"nothing. row=%+v", cs)
}
if cs.Attempts == 0 {
t.Fatalf("the position PAID %.6f and reports attempts=0 — the stop landed on the stage's "+
"first call, and the count is written after the error check, so a position that was "+
"cut reads as one that never started", cs.CostUSD)
}
}
}
if !marked {
t.Fatalf("premise broken: no cancelled row was written, so this fixture measured nothing. rows=%d",
len(m.statuses))
}
assertRowsMatchTheLedger(t, bookPath)
}
// TestTheAskedNTimesNoteReachesTheReader pins the operator sentence at its source. Its two properties
// are invisible from the call site and both were wrong: the note sat behind a transport error longer
// than the column's own bound, and it claimed an estimate was booked on chains where nothing was.
func TestTheAskedNTimesNoteReachesTheReader(t *testing.T) {
long := errors.New(strings.Repeat("провайдер разорвал соединение на середине тела ответа. ", 6))
paid := cutErrLine(long, 2, 0.001056)
if !strings.HasPrefix(paid, "[the provider was asked 2 times;") {
t.Fatalf("the note must come FIRST — the column keeps 120 bytes and the error in front of it is "+
"longer than that, so a note at the end reaches nobody. got %q", paid)
}
if !strings.Contains(paid, "ONE estimate is booked") {
t.Fatalf("a chain that WAS billed must say one estimate covers it: %q", paid)
}
free := cutErrLine(long, 3, 0)
if strings.Contains(free, "ONE estimate is booked") {
t.Fatalf("nothing was booked for this chain, so the line must not claim an estimate was: %q", free)
}
if !strings.Contains(free, "NOTHING is booked") {
t.Fatalf("the line must say what actually happened to the money: %q", free)
}
// The control: the error itself is still carried, or the note would have replaced the diagnosis
// instead of leading it.
if !strings.Contains(paid, "разорвал соединение") {
t.Fatalf("the transport error must still be in the line: %q", paid)
}
}
// TestAStopBehindAPaidBreakStillMarksThePosition is the hole §4.2 forbids «at any moment», and it opens
// exactly where two correct mechanisms meet. The retry chain hands up the cut with the strongest MONEY
// claim, which is deliberately the earlier `connection_lost` when the later one is free — so a run a
// person stopped arrives at the mark carrying a cause that is not «stopped». The guard read that first
// cut's cause, decided this was not a stop, and returned in silence: money settled, no row, and the
// export shows a gap nobody can explain.
//
// The money is not what is lost here and the fixture says so: the ledger stays whole and the resume
// re-does the position correctly. What is lost is the position's visible mark — which is the one thing
// the ratified estimate was conditioned on.
func TestAStopBehindAPaidBreakStillMarksThePosition(t *testing.T) {
dir := t.TempDir()
var calls atomic.Int32
held := make(chan struct{})
var once sync.Once
srv := newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) {
if calls.Add(1) == 1 {
// Attempt 1: acknowledged, then the socket dies — retryable, and the money of the chain.
dropAfterHeaders(t, w)
return
}
once.Do(func() { close(held) }) // attempt 2 is in flight when the person presses stop
hold(req)
})
bookPath := setupCutProject(t, dir, srv.srv.URL)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
<-held
cancel()
}()
_ = runOnce(t, ctx, bookPath)
m := readMoney(t, bookPath)
// The premise, and it is the whole point: the chain really did carry a PAID break rather than the
// stop. Without it the fixture would pass on the ordinary «stopped over the first call» path, which
// was never broken.
paid := 0.0
for _, u := range m.checkpoints {
paid += u.CostUSD
}
if paid <= 0 {
t.Fatalf("premise broken: the chain must have settled a PAID break before the stop, else this "+
"fixture measures the case that already worked. checkpoints=%d", len(m.checkpoints))
}
marked := false
for _, cs := range m.statuses {
if cs.FlagReason == string(FlagCancelled) {
marked = true
}
}
if !marked {
t.Fatalf("a person stopped this run and its money is settled, but the position carries NO mark: "+
"the guard read the cause of whichever cut came first — a paid `connection_lost` the chain "+
"kept for its money — and decided a stop was not a stop. §4.2 forbids a hole with no mark at "+
"any moment. rows=%d, settled=%.6f", len(m.statuses), paid)
}
assertRowsMatchTheLedger(t, bookPath)
}
// TestTheCutRowPublishesItsEstimateAndItsGap pins the two carriers of DISCLOSURE on a cut row, and both
// were held by nothing. The ratified estimate is charged on one condition — that the row says it is an
// estimate (D39.230 п.1) — so these are not decoration: they are what the owner's word rests on.
//
// ⚠ WHY THE TABLE OF NINE OUTCOMES DID NOT COVER THEM. Its `wantEstimated` column reads the CHECKPOINT's
// usage through estimatedSpend, not `request_log.estimated`. In every fixture the two are true together,
// so the assertion could not tell them apart — the «two different quantities agreed» shape. This one
// reads the row it is talking about.
func TestTheCutRowPublishesItsEstimateAndItsGap(t *testing.T) {
dir := t.TempDir()
var calls atomic.Int32
srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
// Two drops: one is retried inside the transport, the second is terminal — so the chain delivered
// TWICE and one estimate covers both, which is exactly the gap the row must publish.
if calls.Add(1) <= 2 {
dropAfterHeaders(t, w)
return
}
answerWhole(w)
})
bookPath := setupCutProject(t, dir, srv.srv.URL)
if err := runOnce(t, context.Background(), bookPath); err == nil {
t.Fatal("premise broken: the run was supposed to end on a delivered cut")
}
r, err := NewReadOnlyRunner(bookPath, obs.NewLogger())
if err != nil {
t.Fatal(err)
}
defer r.Close()
rows, err := r.Store.RequestLogRows(r.Book.BookID)
if err != nil {
t.Fatal(err)
}
var cutRows int
for _, row := range rows {
if row.CostUSD <= 0 {
continue
}
cutRows++
if row.Estimated != 1 {
t.Fatalf("a row whose cost is a RESERVATION ESTIMATE must say so: the owner's word charges it "+
"on that condition, and a silent $%.6f is indistinguishable from a provider-reported one. "+
"row finish=%q estimated=%d", row.CostUSD, row.FinishReason, row.Estimated)
}
if row.EstTokens <= 0 {
t.Fatalf("an estimated row publishes the token figure the estimate was built from; 0 leaves "+
"the reader with a number and no way to weigh it. row finish=%q", row.FinishReason)
}
if !strings.Contains(row.Err, "asked 2 times") {
t.Fatalf("the provider was asked twice and ONE estimate is booked for both — the row is the "+
"only place that gap is published, and it says %q", row.Err)
}
}
// The control: there WAS a paid row to inspect. «No violations» over an empty set and over a real one
// print the same on a green test.
if cutRows == 0 {
t.Fatalf("premise broken: no paid row was written, so nothing above was checked. rows=%d", len(rows))
}
}