1193 lines
54 KiB
Go
1193 lines
54 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/chunk"
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/llm"
|
||
"textmachine/backend/internal/obs"
|
||
"textmachine/backend/internal/store"
|
||
"textmachine/backend/internal/terminology"
|
||
)
|
||
|
||
// burnedpregates_test.go: THE PAID-CALL CONTRACT read before the funnel — who asks «is this call already
|
||
// paid for», what a BURNED key does to their answer, and what the answer is allowed to decide.
|
||
//
|
||
// A burned checkpoint records money and NO result (a delivered call the engine cut short), so runAttempt
|
||
// cannot replay it — it walks past and buys the work again. A pre-gate that reads such a row as «already
|
||
// paid» therefore does not skip a purchase, it lets one through: the fresh call happens with the gate's
|
||
// own budget check jumped over. The bank's probe learned this when cut calls started settling at the
|
||
// batch key; these two gates ask the same question about repair and about the escalation hop.
|
||
//
|
||
// Each pair is deliberate. The first fixture holds the property (a burned key must not buy), the second
|
||
// holds its opposite (a real key must still replay for free) — because a predicate that simply answered
|
||
// «not paid» to everything would pass the first alone while turning the free-replay contract off.
|
||
|
||
// gateServer is this file's own provider stub. It exists rather than reusing cutServer because these
|
||
// fixtures need to tell the REPAIR call apart from every other one — the whole assertion is «how many
|
||
// repair calls were bought» — and because a handler that must both read the body and then hold the
|
||
// connection cannot share a helper that drains the body before it is handed the request.
|
||
type gateServer struct {
|
||
mu sync.Mutex
|
||
// counted is how many calls of the kind THIS fixture is about have landed — repair calls for
|
||
// newGateServer, hop calls for newHopServer. One field, because a fixture watches one gate.
|
||
counted int
|
||
srv *httptest.Server
|
||
}
|
||
|
||
// newGateServer answers every non-repair call with a draft carrying the defect the repair class detects,
|
||
// and hands the repair call to `onRepair` — with its 1-based ordinal, so a fixture can make the first
|
||
// call behave differently from the ones that follow it.
|
||
func newGateServer(t *testing.T, onRepair func(i int, w http.ResponseWriter, req *http.Request)) *gateServer {
|
||
t.Helper()
|
||
gs := &gateServer{}
|
||
gs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||
body, _ := io.ReadAll(req.Body)
|
||
if !isRepairBody(string(body)) {
|
||
answerJSON(w, "Он прождал полчаса и вошёл внутрь.")
|
||
return
|
||
}
|
||
gs.mu.Lock()
|
||
gs.counted++
|
||
i := gs.counted
|
||
gs.mu.Unlock()
|
||
onRepair(i, w, req)
|
||
}))
|
||
t.Cleanup(gs.srv.Close)
|
||
return gs
|
||
}
|
||
|
||
func (g *gateServer) repairCalls() int { g.mu.Lock(); defer g.mu.Unlock(); return g.counted }
|
||
|
||
// newHopServer is the escalation twin of newGateServer: every primary-model call is answered with a
|
||
// draft, and only the call addressed to the FALLBACK model — the hop — is counted and handed to the
|
||
// fixture. Counting by model rather than by ordinal matters: the primary's own attempts share the
|
||
// connection and would otherwise be indistinguishable from the hop in the tally.
|
||
func newHopServer(t *testing.T, onHop func(i int, w http.ResponseWriter, req *http.Request)) *gateServer {
|
||
t.Helper()
|
||
gs := &gateServer{}
|
||
gs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||
body, _ := io.ReadAll(req.Body)
|
||
if !isHopBody(string(body)) {
|
||
answerJSON(w, "ЧЕРНОВИК ПЕРЕВОДА.")
|
||
return
|
||
}
|
||
gs.mu.Lock()
|
||
gs.counted++
|
||
i := gs.counted
|
||
gs.mu.Unlock()
|
||
onHop(i, w, req)
|
||
}))
|
||
t.Cleanup(gs.srv.Close)
|
||
return gs
|
||
}
|
||
|
||
func (g *gateServer) hopCalls() int { return g.repairCalls() }
|
||
|
||
// answerJSON writes one ordinary finished reply.
|
||
func answerJSON(w http.ResponseWriter, text string) {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%q},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500}}`, text)
|
||
}
|
||
|
||
// repairBudgetLine is the pipeline.yaml line the fixtures rewrite to exhaust the repair sub-budget
|
||
// between the two runs. It is matched exactly so a change to repairGateYAML breaks this loudly instead
|
||
// of silently leaving the budget wide open and the assertions vacuous.
|
||
const repairBudgetLine = " budget_usd: 1.0\n"
|
||
|
||
// exhaustRepairBudget rewrites the fixture's repair budget to a value no call can fit under, so the ONLY
|
||
// thing that can still produce a repair call is a pre-gate answering «already paid».
|
||
func exhaustRepairBudget(t *testing.T, bookPath string) {
|
||
t.Helper()
|
||
p := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(p)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(string(raw), repairBudgetLine) {
|
||
t.Fatalf("premise broken: %s no longer carries %q, so this fixture would run with an unbounded "+
|
||
"repair budget and prove nothing", p, strings.TrimSpace(repairBudgetLine))
|
||
}
|
||
writeFile(t, p, strings.Replace(string(raw), repairBudgetLine, " budget_usd: 0.0000001\n", 1))
|
||
}
|
||
|
||
// assertRepairBudgetExhausted fails LOUDLY when the fixture's own premise is not in place. Without it a
|
||
// broken exhaust-helper leaves the burned fixtures accusing the pre-gate of buying past the sub-budget
|
||
// when the budget was simply never exhausted — the right colour under the wrong text, which the next
|
||
// shift reads as a money defect and goes to fix in repair.go.
|
||
func assertRepairBudgetExhausted(t *testing.T, r *Runner, spent float64) {
|
||
t.Helper()
|
||
if b := r.Pipeline.Gates.Repair.BudgetUSD; spent <= b {
|
||
t.Fatalf("premise broken: the repair sub-budget is NOT exhausted (spent %.6f, budget %.6f), so "+
|
||
"nothing below is measuring what it claims", spent, b)
|
||
}
|
||
}
|
||
|
||
// repairJob sets up the snapshot and job a direct maybeRepair call needs, and returns the edit stage it
|
||
// runs over. maybeRepair is called directly rather than through TranslateBook because the property is
|
||
// about ONE gate's decision, and a whole-book run would reach it through a resume path that has its own
|
||
// short-circuits.
|
||
func repairJob(t *testing.T, r *Runner) (config.Stage, chunk.Chunk, string, *store.Job) {
|
||
t.Helper()
|
||
st := r.Pipeline.Stages[len(r.Pipeline.Stages)-1]
|
||
if st.Name != "edit" {
|
||
t.Fatalf("premise broken: the repair gate runs on the SHIPPING stage; the fixture's last stage is %q", st.Name)
|
||
}
|
||
ch := chunk.Chunk{Chapter: 1, ChunkIdx: 0, Text: "他等了半个时辰。"}
|
||
const snapID = "snapshot-under-test"
|
||
if err := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The waves build the provider clients eagerly before any worker runs; a direct call into one stage
|
||
// has to do the same or the first request finds no client and never reaches the wire — which would
|
||
// leave every «no call was made» assertion below true for the wrong reason.
|
||
if err := r.buildClients(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return st, ch, snapID, job
|
||
}
|
||
|
||
// TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget is the money property of repair.go's pre-gate.
|
||
//
|
||
// The sequence is the production one, not a hand-written row: the first run's repair call IS delivered
|
||
// and IS cut (the operator stops the run while it is in flight), so the engine settles it at the repair
|
||
// request's own key — money, no text. The second run then finds that key with the sub-budget already
|
||
// exhausted. `repairCheckpointExists` answering «paid» skips the budget comparison entirely and walks
|
||
// into runRepairAttempt, which cannot replay a burned row and buys the repair again — a fresh paid call
|
||
// the only budget bounding repair spend never saw.
|
||
func TestABurnedRepairKeyDoesNotBuyARepairOutsideTheSubBudget(t *testing.T) {
|
||
// ⛔ THE BURN MUST BE A BILLABLE ONE, and how it is produced is the fixture's only delicate part. Our
|
||
// money predicate charges a cut call only once the provider has ACKNOWLEDGED it with a reply, so a
|
||
// server that merely holds a silent connection leaves a $0 row — real, but not the row this gate is
|
||
// about. The handler therefore answers 200, flushes the first bytes of a body, and kills the socket:
|
||
// money owed for a generation, no result to show for it.
|
||
//
|
||
// The cause is the CONNECTION rather than a stop button on purpose. An externally-timed cancel has to
|
||
// be fired after the client has parsed the response headers, and the handler cannot know when that
|
||
// happened — flushing only pushes bytes at the socket. Measured on the first draft of this fixture:
|
||
// 2 runs in 5 cancelled early enough that the call read as unanswered and settled for $0, which would
|
||
// have made the assertions below pass while measuring nothing. A server-driven drop needs no
|
||
// synchronisation at all, and `connection_lost` burns the key exactly the same way.
|
||
//
|
||
// Only the FIRST run's calls are dropped. A later one is answered, so that if the gate does buy a
|
||
// repair it cannot afford, the fixture fails on the MONEY assertion — «a call was bought outside the
|
||
// sub-budget» — rather than on whatever the transport happened to say about the second drop.
|
||
dropUntil := 0
|
||
srv := newGateServer(t, func(i int, w http.ResponseWriter, _ *http.Request) {
|
||
if dropUntil == 0 || i <= dropUntil {
|
||
dropAfterHeaders(t, w)
|
||
return
|
||
}
|
||
answerJSON(w, "Он прождал час и вошёл внутрь.")
|
||
})
|
||
bookPath := setupRepairProject(t, srv.srv.URL)
|
||
|
||
// Run 1: the repair call is cut in flight, so its key holds money and no result.
|
||
r1 := newRunner(t, bookPath)
|
||
st, ch, snapID, job := repairJob(t, r1)
|
||
const drafted = "Он прождал полчаса и вошёл внутрь."
|
||
_, _, mrErr := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil)
|
||
if mrErr == nil {
|
||
t.Fatal("premise broken: the repair call was supposed to be cut in flight, and it returned cleanly")
|
||
}
|
||
burned, err := r1.Store.RepairSpentUSD(r1.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if burned <= 0 {
|
||
t.Fatalf("premise broken: a cut repair call must be SETTLED at the repair key — that row is the "+
|
||
"whole subject of this fixture. repair spend %.6f", burned)
|
||
}
|
||
// The count is recorded, not asserted to be one: a lost connection is retried once inside the
|
||
// transport, so run 1 legitimately asks twice under ONE request key and settles ONE row. What the
|
||
// second run must add is zero, and that is the assertion below.
|
||
repairsAfterRun1 := srv.repairCalls()
|
||
if repairsAfterRun1 == 0 {
|
||
t.Fatal("premise broken: run 1 never reached the provider, so there is no burned key to test")
|
||
}
|
||
dropUntil = repairsAfterRun1 // from here on the provider answers; only the money may still object
|
||
r1.Close()
|
||
|
||
// Run 2: the sub-budget can no longer fit a call. Nothing may buy one.
|
||
exhaustRepairBudget(t, bookPath)
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
st2, ch2, snapID2, job2 := repairJob(t, r2)
|
||
assertRepairBudgetExhausted(t, r2, burned)
|
||
if _, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil); err != nil {
|
||
t.Fatalf("a repair the budget cannot afford is SKIPPED, not an error: %v", err)
|
||
}
|
||
if got := srv.repairCalls(); got != repairsAfterRun1 {
|
||
t.Fatalf("a burned key is money with NO result: runAttempt cannot replay it and buys the repair "+
|
||
"again, so a pre-gate answering «already paid» lets that purchase jump the sub-budget check. "+
|
||
"want %d repair calls, got %d — one of them was bought outside gates.repair.budget_usd",
|
||
repairsAfterRun1, got)
|
||
}
|
||
spent, err := r2.Store.RepairSpentUSD(r2.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if spent != burned {
|
||
t.Fatalf("the exhausted sub-budget still moved: %.6f → %.6f", burned, spent)
|
||
}
|
||
}
|
||
|
||
// TestARealRepairKeyStillReplaysFreeOnAnExhaustedBudget is the other half, and it is what stops the fix
|
||
// above from being «answer no to everything». An ANSWERED repair key must replay for $0 even when the
|
||
// sub-budget is gone: the money was already spent, the bytes already exist, and re-deciding them would
|
||
// make a resumed run ship different text than the run that paid for it.
|
||
func TestARealRepairKeyStillReplaysFreeOnAnExhaustedBudget(t *testing.T) {
|
||
srv := newGateServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
|
||
answerJSON(w, "Он прождал час и вошёл внутрь.")
|
||
})
|
||
bookPath := setupRepairProject(t, srv.srv.URL)
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
st, ch, snapID, job := repairJob(t, r1)
|
||
const drafted = "Он прождал полчаса и вошёл внутрь."
|
||
fixed, _, err := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(fixed, "час") || strings.Contains(fixed, "полчаса") {
|
||
t.Fatalf("premise broken: run 1 must have APPLIED a repair, got %q", fixed)
|
||
}
|
||
paidSpend, err := r1.Store.RepairSpentUSD(r1.Book.BookID)
|
||
if err != nil || paidSpend <= 0 {
|
||
t.Fatalf("premise broken: the answered repair must be booked, got %.6f %v", paidSpend, err)
|
||
}
|
||
repairsAfterRun1 := srv.repairCalls()
|
||
r1.Close()
|
||
|
||
exhaustRepairBudget(t, bookPath)
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
st2, ch2, snapID2, job2 := repairJob(t, r2)
|
||
assertRepairBudgetExhausted(t, r2, paidSpend)
|
||
again, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if again != fixed {
|
||
t.Fatalf("an already-paid repair must re-serve the SAME bytes:\n first: %q\nsecond: %q", fixed, again)
|
||
}
|
||
if got := srv.repairCalls(); got != repairsAfterRun1 {
|
||
t.Fatalf("an already-paid repair replays for $0 and asks nobody: want %d repair calls, got %d",
|
||
repairsAfterRun1, got)
|
||
}
|
||
if spent, serr := r2.Store.RepairSpentUSD(r2.Book.BookID); serr != nil || spent != paidSpend {
|
||
t.Fatalf("a free replay must not move the sub-budget spend: %.6f → %.6f (%v)", paidSpend, spent, serr)
|
||
}
|
||
}
|
||
|
||
// exhaustEscalationBudget rewrites the fixture's escalation budget below what run 1 already spent, so
|
||
// escalationBudgetRemains() answers «no» and the ONLY thing that can still buy a hop is the idempotency
|
||
// probe answering «already paid».
|
||
func exhaustEscalationBudget(t *testing.T, bookPath string) {
|
||
t.Helper()
|
||
p := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(p)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
i := strings.Index(string(raw), "escalation: { budget_usd:")
|
||
if i < 0 {
|
||
t.Fatalf("premise broken: %s carries no escalation budget line, so this fixture would run with the "+
|
||
"gate wide open and prove nothing", p)
|
||
}
|
||
j := strings.Index(string(raw)[i:], "\n")
|
||
writeFile(t, p, string(raw)[:i]+"escalation: { budget_usd: 0.0000001 }"+string(raw)[i+j:])
|
||
}
|
||
|
||
// isHopBody reports whether a mock request is the ESCALATION hop: it is the only call addressed to the
|
||
// fallback model, which the wire body names.
|
||
func isHopBody(body string) bool { return strings.Contains(body, `"model":"fake-fallback"`) }
|
||
|
||
// escalationJob prepares a direct maybeEscalate call over the draft stage, which is the one carrying
|
||
// `escalate_to` in the fixture.
|
||
func escalationJob(t *testing.T, r *Runner) (config.Stage, chunk.Chunk, string, *store.Job) {
|
||
t.Helper()
|
||
st := r.Pipeline.Stages[0]
|
||
if st.ResolvedHop == "" {
|
||
t.Fatalf("premise broken: stage %q has no resolved hop, so maybeEscalate returns before it decides "+
|
||
"anything and every assertion below would be vacuous", st.Name)
|
||
}
|
||
ch := chunk.Chunk{Chapter: 1, ChunkIdx: 0, Text: "静かな図書館の朝。"}
|
||
const snapID = "snapshot-under-test"
|
||
if err := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
job, err := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := r.buildClients(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return st, ch, snapID, job
|
||
}
|
||
|
||
// hopMessages is the prompt both runs hand maybeEscalate. It is a constant of the fixture because the
|
||
// hop's request hash is derived from it: two runs that differed here would address two different keys
|
||
// and the second would find nothing, passing for the wrong reason.
|
||
var hopMessages = []llm.Message{{Role: "user", Content: "静かな図書館の朝。"}}
|
||
|
||
const hopBaseMaxTokens = 512
|
||
|
||
// TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget is the money property of escalation.go's
|
||
// idempotency probe, and it has a second half the repair gate does not: the fresh-hop branch is also
|
||
// where escMu is taken. A burned key read as «already paid» skips BOTH — the budget check and the
|
||
// serialisation — so the hop it then buys is unbounded AND unserialised against the other draft workers.
|
||
func TestABurnedHopKeyDoesNotBuyAHopOutsideTheEscalationBudget(t *testing.T) {
|
||
dropUntil := 0
|
||
srv := newHopServer(t, func(i int, w http.ResponseWriter, _ *http.Request) {
|
||
if dropUntil == 0 || i <= dropUntil {
|
||
dropAfterHeaders(t, w) // delivered, acknowledged, then the socket dies: money, no result
|
||
return
|
||
}
|
||
answerJSON(w, "Тихое утро в библиотеке.")
|
||
})
|
||
bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0)
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
st, ch, snapID, job := escalationJob(t, r1)
|
||
primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}}
|
||
if _, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false); err == nil {
|
||
t.Fatal("premise broken: the hop was supposed to be cut by the connection, and it returned cleanly")
|
||
}
|
||
burned, err := r1.Store.EscalationSpentUSD(r1.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if burned <= 0 {
|
||
t.Fatalf("premise broken: a cut hop must be SETTLED at the hop key — that row is the subject of "+
|
||
"this fixture. escalation spend %.6f", burned)
|
||
}
|
||
hopsAfterRun1 := srv.hopCalls()
|
||
if hopsAfterRun1 == 0 {
|
||
t.Fatal("premise broken: run 1 never reached the provider, so there is no burned key to test")
|
||
}
|
||
dropUntil = hopsAfterRun1
|
||
r1.Close()
|
||
|
||
exhaustEscalationBudget(t, bookPath)
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
st2, ch2, snapID2, job2 := escalationJob(t, r2)
|
||
if remains, rerr := r2.escalationBudgetRemains(); rerr != nil || remains {
|
||
t.Fatalf("premise broken: the escalation budget must be exhausted for run 2, remains=%t err=%v", remains, rerr)
|
||
}
|
||
if _, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false); err != nil {
|
||
t.Fatalf("a hop the budget cannot afford is SKIPPED, not an error: %v", err)
|
||
}
|
||
if got := srv.hopCalls(); got != hopsAfterRun1 {
|
||
t.Fatalf("a burned key is money with NO result: runAttempt cannot replay it and buys the hop again, "+
|
||
"so a probe answering «already paid» sends that purchase past escalation.budget_usd AND past "+
|
||
"escMu. want %d hop calls, got %d", hopsAfterRun1, got)
|
||
}
|
||
if spent, serr := r2.Store.EscalationSpentUSD(r2.Book.BookID); serr != nil || spent != burned {
|
||
t.Fatalf("the exhausted escalation budget still moved: %.6f → %.6f (%v)", burned, spent, serr)
|
||
}
|
||
}
|
||
|
||
// TestARealHopKeyStillReplaysFreeOnAnExhaustedBudget is the other half. escalation.go's doccomment
|
||
// promises a paid hop replays «regardless of the budget» — a crash between the hop's settle and the
|
||
// chunk_status write must not discard a paid, successful translation and flip the verdict OK→flagged.
|
||
// Teaching the probe about burned keys must not cost that promise.
|
||
func TestARealHopKeyStillReplaysFreeOnAnExhaustedBudget(t *testing.T) {
|
||
srv := newHopServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
|
||
answerJSON(w, "Тихое утро в библиотеке.")
|
||
})
|
||
bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0)
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
st, ch, snapID, job := escalationJob(t, r1)
|
||
primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}}
|
||
out1, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !out1.attempted {
|
||
t.Fatal("premise broken: run 1 must have made the hop")
|
||
}
|
||
paid, err := r1.Store.EscalationSpentUSD(r1.Book.BookID)
|
||
if err != nil || paid <= 0 {
|
||
t.Fatalf("premise broken: the answered hop must be booked, got %.6f %v", paid, err)
|
||
}
|
||
hopsAfterRun1 := srv.hopCalls()
|
||
r1.Close()
|
||
|
||
exhaustEscalationBudget(t, bookPath)
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
st2, ch2, snapID2, job2 := escalationJob(t, r2)
|
||
if remains, rerr := r2.escalationBudgetRemains(); rerr != nil || remains {
|
||
t.Fatalf("premise broken: the escalation budget must be exhausted for run 2, remains=%t err=%v", remains, rerr)
|
||
}
|
||
out2, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !out2.attempted {
|
||
t.Fatal("an already-paid hop must still be REPLAYED on an exhausted budget: discarding it flips a " +
|
||
"paid, successful translation to flagged on every resume")
|
||
}
|
||
if out2.fb.text != out1.fb.text {
|
||
t.Fatalf("the replayed hop must re-serve the SAME bytes:\n first: %q\nsecond: %q", out1.fb.text, out2.fb.text)
|
||
}
|
||
if got := srv.hopCalls(); got != hopsAfterRun1 {
|
||
t.Fatalf("an already-paid hop replays for $0 and asks nobody: want %d hop calls, got %d", hopsAfterRun1, got)
|
||
}
|
||
if spent, serr := r2.Store.EscalationSpentUSD(r2.Book.BookID); serr != nil || spent != paid {
|
||
t.Fatalf("a free replay must not move the escalation spend: %.6f → %.6f (%v)", paid, spent, serr)
|
||
}
|
||
}
|
||
|
||
// TestAPaidHopBehindABurnedKeyStillReplaysFree is the case both halves of the pair above miss, and it is
|
||
// the one that makes the probe's SHAPE matter rather than its answer.
|
||
//
|
||
// The funnel does not stop at a burned key — it walks to the next attempt index at the same budget and
|
||
// buys there. So after a stopped run the position looks like this: attempt 0 burned, attempt 1 PAID and
|
||
// answered. A probe that asks about a FIXED index sees only the burn, answers «not paid», and hands the
|
||
// decision to the budget — which, being exhausted, discards a translation that was already bought. The
|
||
// probe must ask what the FUNNEL will ask: walk the burns, then look.
|
||
//
|
||
// Three runs, because the state needs two of them to exist: run 1 burns attempt 0, run 2 buys the answer
|
||
// at attempt 1, run 3 arrives with no budget left and must still serve it for $0.
|
||
func TestAPaidHopBehindABurnedKeyStillReplaysFree(t *testing.T) {
|
||
dropFirst := true
|
||
srv := newHopServer(t, func(i int, w http.ResponseWriter, _ *http.Request) {
|
||
if dropFirst {
|
||
dropAfterHeaders(t, w) // run 1: money, no result — the key at attempt 0 is burned
|
||
return
|
||
}
|
||
answerJSON(w, "Тихое утро в библиотеке.")
|
||
})
|
||
bookPath := setupTwoChapterEscalation(t, srv.srv.URL, 1.0)
|
||
primary := stageAttempt{cls: classification{Reason: FlagCJKArtifact}}
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
st, ch, snapID, job := escalationJob(t, r1)
|
||
if _, err := r1.maybeEscalate(context.Background(), st, snapID, ch, job, hopBaseMaxTokens, hopMessages, primary, false); err == nil {
|
||
t.Fatal("premise broken: run 1's hop was supposed to be cut by the connection")
|
||
}
|
||
r1.Close()
|
||
|
||
// Run 2: the budget is still there, so the funnel walks past the burn and buys the answer one index up.
|
||
dropFirst = false
|
||
r2 := newRunner(t, bookPath)
|
||
st2, ch2, snapID2, job2 := escalationJob(t, r2)
|
||
out2, err := r2.maybeEscalate(context.Background(), st2, snapID2, ch2, job2, hopBaseMaxTokens, hopMessages, primary, false)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !out2.attempted || out2.fb.text == "" {
|
||
t.Fatalf("premise broken: run 2 must have BOUGHT the hop past the burn, got attempted=%t text=%q", out2.attempted, out2.fb.text)
|
||
}
|
||
paid, err := r2.Store.EscalationSpentUSD(r2.Book.BookID)
|
||
if err != nil || paid <= 0 {
|
||
t.Fatalf("premise broken: run 2's hop must be booked, got %.6f %v", paid, err)
|
||
}
|
||
hopsAfterRun2 := srv.hopCalls()
|
||
r2.Close()
|
||
|
||
// Run 3: no budget left. The hop was PAID and ANSWERED — discarding it now throws away bought work
|
||
// and flips the unit's verdict on every later resume.
|
||
exhaustEscalationBudget(t, bookPath)
|
||
r3 := newRunner(t, bookPath)
|
||
defer r3.Close()
|
||
st3, ch3, snapID3, job3 := escalationJob(t, r3)
|
||
if remains, rerr := r3.escalationBudgetRemains(); rerr != nil || remains {
|
||
t.Fatalf("premise broken: the escalation budget must be exhausted for run 3, remains=%t err=%v", remains, rerr)
|
||
}
|
||
out3, err := r3.maybeEscalate(context.Background(), st3, snapID3, ch3, job3, hopBaseMaxTokens, hopMessages, primary, false)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !out3.attempted || out3.fb.text != out2.fb.text {
|
||
t.Fatalf("A PAID, SUCCESSFUL HOP WAS DISCARDED because a BURNED key sits in front of it. The probe "+
|
||
"asks a fixed attempt index; the funnel walks burns and answers one index up, so «is this paid» "+
|
||
"must be asked the funnel's way. run 2 bought %q, run 3 served attempted=%t %q",
|
||
out2.fb.text, out3.attempted, out3.fb.text)
|
||
}
|
||
// The control: serving it must have cost nothing — otherwise this test would pass by re-buying.
|
||
if got := srv.hopCalls(); got != hopsAfterRun2 {
|
||
t.Fatalf("the replay must ask nobody: want %d hop calls, got %d", hopsAfterRun2, got)
|
||
}
|
||
if spent, serr := r3.Store.EscalationSpentUSD(r3.Book.BookID); serr != nil || spent != paid {
|
||
t.Fatalf("a free replay must not move the escalation spend: %.6f → %.6f (%v)", paid, spent, serr)
|
||
}
|
||
}
|
||
|
||
// TestAPaidRepairBehindABurnedKeyStillReplaysFree is the repair twin of the hop case above. Its own
|
||
// contract is stated at repair.go's budget block: an already-paid repair replays regardless of the
|
||
// remaining budget, because a crash between the paid call and the chunk_status write would otherwise
|
||
// make the resumed run ship DIFFERENT bytes than the run that paid for them.
|
||
func TestAPaidRepairBehindABurnedKeyStillReplaysFree(t *testing.T) {
|
||
dropFirst := true
|
||
srv := newGateServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
|
||
if dropFirst {
|
||
dropAfterHeaders(t, w) // run 1: the repair key at the candidate's own index is burned
|
||
return
|
||
}
|
||
answerJSON(w, "Он прождал час и вошёл внутрь.")
|
||
})
|
||
bookPath := setupRepairProject(t, srv.srv.URL)
|
||
const drafted = "Он прождал полчаса и вошёл внутрь."
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
st, ch, snapID, job := repairJob(t, r1)
|
||
if _, _, err := r1.maybeRepair(context.Background(), st, snapID, ch, job, drafted, drafted, nil); err == nil {
|
||
t.Fatal("premise broken: run 1's repair was supposed to be cut by the connection")
|
||
}
|
||
r1.Close()
|
||
|
||
dropFirst = false
|
||
r2 := newRunner(t, bookPath)
|
||
st2, ch2, snapID2, job2 := repairJob(t, r2)
|
||
fixed, _, err := r2.maybeRepair(context.Background(), st2, snapID2, ch2, job2, drafted, drafted, nil)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(fixed, "час") || strings.Contains(fixed, "полчаса") {
|
||
t.Fatalf("premise broken: run 2 must have BOUGHT the repair past the burn, got %q", fixed)
|
||
}
|
||
paid, err := r2.Store.RepairSpentUSD(r2.Book.BookID)
|
||
if err != nil || paid <= 0 {
|
||
t.Fatalf("premise broken: run 2's repair must be booked, got %.6f %v", paid, err)
|
||
}
|
||
repairsAfterRun2 := srv.repairCalls()
|
||
r2.Close()
|
||
|
||
exhaustRepairBudget(t, bookPath)
|
||
r3 := newRunner(t, bookPath)
|
||
defer r3.Close()
|
||
st3, ch3, snapID3, job3 := repairJob(t, r3)
|
||
assertRepairBudgetExhausted(t, r3, paid)
|
||
again, _, err := r3.maybeRepair(context.Background(), st3, snapID3, ch3, job3, drafted, drafted, nil)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if again != fixed {
|
||
t.Fatalf("A PAID REPAIR WAS DISCARDED because a BURNED key sits in front of it, so the resumed run "+
|
||
"ships DIFFERENT bytes than the run that paid:\n paid for: %q\n now ships: %q", fixed, again)
|
||
}
|
||
if got := srv.repairCalls(); got != repairsAfterRun2 {
|
||
t.Fatalf("the replay must ask nobody: want %d repair calls, got %d", repairsAfterRun2, got)
|
||
}
|
||
if spent, serr := r3.Store.RepairSpentUSD(r3.Book.BookID); serr != nil || spent != paid {
|
||
t.Fatalf("a free replay must not move the sub-budget spend: %.6f → %.6f (%v)", paid, spent, serr)
|
||
}
|
||
}
|
||
|
||
// TestTheBankProbeFindsThePaidBatchBehindABurnedKey is the third member of the family, pinned directly
|
||
// rather than end-to-end: the bank probe is a pure function of the store, and the two runs the other two
|
||
// fixtures need exist here only to create rows. What it holds is the same contract — a burn in front of
|
||
// an answer must not hide the answer.
|
||
func TestTheBankProbeFindsThePaidBatchBehindABurnedKey(t *testing.T) {
|
||
dir := t.TempDir()
|
||
srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { answerWhole(w) })
|
||
bookPath := setupCutProject(t, dir, srv.srv.URL)
|
||
r, err := NewRunner(bookPath, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer r.Close()
|
||
|
||
st := r.Pipeline.Stages[0]
|
||
ch := chunk.Chunk{Chapter: 0, ChunkIdx: 0}
|
||
msgs := []llm.Message{{Role: "user", Content: "термины"}}
|
||
const snapID = "snapshot-under-test"
|
||
if serr := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); serr != nil {
|
||
t.Fatal(serr)
|
||
}
|
||
job, jerr := r.Store.EnsureJob(r.Book.BookID, ch.Chapter, st.Name, snapID)
|
||
if jerr != nil {
|
||
t.Fatal(jerr)
|
||
}
|
||
_, maxTokens := r.bankCallBudget(st.Model, msgs)
|
||
hashAt := func(attempt int) string {
|
||
return RequestHash(r.attemptRequest(st, st.Model, snapID, ch, attempt, maxTokens, msgs))
|
||
}
|
||
write := func(hash, finish, text string) {
|
||
t.Helper()
|
||
resv, verdict, rerr := r.Store.Reserve(r.Book.BookID, 0.001, store.Ceilings{BookUSD: 100, DayUSD: 100})
|
||
if rerr != nil || verdict != store.ReserveOK {
|
||
t.Fatalf("reserve: %v %v", verdict, rerr)
|
||
}
|
||
if serr := r.Store.SettleWithCheckpoint(resv, 0.001, store.Checkpoint{
|
||
RequestHash: hash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: 0,
|
||
Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: st.Model,
|
||
ResponseText: text, UsageJSON: "{}", CostUSD: 0.001, FinishReason: finish,
|
||
}, nil); serr != nil {
|
||
t.Fatalf("settle: %v", serr)
|
||
}
|
||
}
|
||
|
||
// The control first: with only the burn there, the batch is NOT paid — the same assertion the probe's
|
||
// own fixture makes, repeated here so a probe that answered «paid» to everything would fail before
|
||
// reaching the case this test is about.
|
||
write(hashAt(0), cancelledFinish, "")
|
||
if paid, perr := r.bankCheckpointExists(st, snapID, ch, msgs); perr != nil || paid {
|
||
t.Fatalf("a lone burned key is not a paid batch: paid=%t err=%v", paid, perr)
|
||
}
|
||
// And now the answer the funnel bought one index up, exactly where a stopped run leaves it.
|
||
write(hashAt(1), "stop", "термин\tterm")
|
||
if paid, perr := r.bankCheckpointExists(st, snapID, ch, msgs); perr != nil || !paid {
|
||
t.Fatalf("the batch WAS bought and answered at the index the funnel walked to; a probe that only "+
|
||
"looks at the starting key hides it, and the role sub-budget then refuses to serve work it "+
|
||
"already paid for. paid=%t err=%v", paid, perr)
|
||
}
|
||
}
|
||
|
||
// TestAnUnaffordableBatchDoesNotTakeThePaidBatchesBehindIt pins the admission SHAPE of the bank pass.
|
||
//
|
||
// Admission is not a prefix. An already-paid batch costs nothing and is admitted whatever the budget
|
||
// says — the loop's own comment promises exactly that — so a batch the budget refuses can sit in FRONT of
|
||
// batches that are free to serve. While the decision was a prefix bound, one refused batch took every
|
||
// paid batch behind it: their replay was $0, their result was already bought, and dropping them bought
|
||
// nothing while losing a consolidated bank.
|
||
func TestAnUnaffordableBatchDoesNotTakeThePaidBatchesBehindIt(t *testing.T) {
|
||
dir := t.TempDir()
|
||
srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) { answerWhole(w) })
|
||
bookPath := setupCutProject(t, dir, srv.srv.URL)
|
||
r, err := NewRunner(bookPath, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer r.Close()
|
||
|
||
const snapID = "snapshot-under-test"
|
||
if serr := r.Store.UpsertSnapshot(snapID, r.Book.BriefHash(), "{}"); serr != nil {
|
||
t.Fatal(serr)
|
||
}
|
||
st := r.bankStage(roleTerminologist)
|
||
job, jerr := r.Store.EnsureJob(r.Book.BookID, 0, st.Name, snapID)
|
||
if jerr != nil {
|
||
t.Fatal(jerr)
|
||
}
|
||
// Two batches. The messages function is the fixture's own, so the request identity below is built the
|
||
// same way the pass builds it — re-deriving it any other way would address a different key and this
|
||
// test would pass by measuring nothing.
|
||
batches := [][]terminology.Candidate{{{Key: "甲", Src: "甲"}}, {{Key: "乙", Src: "乙"}}}
|
||
msgsOf := func(b []terminology.Candidate) ([]llm.Message, error) {
|
||
return []llm.Message{{Role: "user", Content: "batch " + b[0].Key}}, nil
|
||
}
|
||
plan := bankRolePlan{role: roleTerminologist, budgetUSD: 0.0000001, messages: msgsOf}
|
||
|
||
// Batch 1 is ALREADY PAID: a real answered checkpoint at its own key. Batch 0 is not, and the budget
|
||
// cannot fit it — so batch 0 is refused and batch 1 must still be served for $0.
|
||
msgs1, _ := msgsOf(batches[1])
|
||
_, maxTokens := r.bankCallBudget(st.Model, msgs1)
|
||
hash1 := RequestHash(r.attemptRequest(st, st.Model, snapID, chunk.Chunk{Chapter: 0, ChunkIdx: 1}, 0, maxTokens, msgs1))
|
||
resv, verdict, rerr := r.Store.Reserve(r.Book.BookID, 0.001, store.Ceilings{BookUSD: 100, DayUSD: 100})
|
||
if rerr != nil || verdict != store.ReserveOK {
|
||
t.Fatalf("reserve: %v %v", verdict, rerr)
|
||
}
|
||
if serr := r.Store.SettleWithCheckpoint(resv, 0.001, store.Checkpoint{
|
||
RequestHash: hash1, JobID: job.ID, ChunkIdx: 1, Attempt: 0,
|
||
Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: st.Model,
|
||
ResponseText: "乙\tvtoroy", UsageJSON: "{}", CostUSD: 0.001, FinishReason: "stop",
|
||
}, nil); serr != nil {
|
||
t.Fatalf("settle: %v", serr)
|
||
}
|
||
if err := r.buildClients(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
callsBefore := srv.calls()
|
||
|
||
run, err := r.runBankRoleBatches(context.Background(), snapID, plan, batches, "render")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The premise: batch 0 really was refused, or the assertion below would hold for the wrong reason.
|
||
if run.ran[0] {
|
||
t.Fatalf("premise broken: batch 0 was supposed to be unaffordable under a budget of %g", plan.budgetUSD)
|
||
}
|
||
if !run.ran[1] || run.texts[1] == "" {
|
||
t.Fatalf("an ALREADY-PAID batch sitting behind a refused one was dropped. Its replay costs $0 and "+
|
||
"its result was already bought, so refusing it buys nothing and loses a consolidated bank. "+
|
||
"ran=%v texts=%q dropped=%d", run.ran, run.texts, run.dropped)
|
||
}
|
||
// And the control: serving it asked the provider nothing.
|
||
if got := srv.calls(); got != callsBefore {
|
||
t.Fatalf("a paid batch must replay for $0: the provider was asked %d times", got-callsBefore)
|
||
}
|
||
if run.dropped != 1 {
|
||
t.Fatalf("exactly one batch was refused, so the report must say one: dropped=%d", run.dropped)
|
||
}
|
||
}
|
||
|
||
// assertRowsMatchTheLedger holds the invariant this family is about: a position's chunk_status row
|
||
// records what that position COST, and the checkpoints of that position are what it cost. The two are
|
||
// written by different code on different exits — the funnel, the escalation hop, the repair sub-step,
|
||
// the cut settle — and every one of them can walk out early. A row below its own ledger is money the
|
||
// book spent and the projection a person decides on does not show.
|
||
//
|
||
// Bank roles are excluded because they write no chunk_status at all (synthetic stage, chapter 0): they
|
||
// are a different accounting axis, and counting them here would compare a row against a sum that
|
||
// includes rows it never claimed.
|
||
func assertRowsMatchTheLedger(t *testing.T, bookPath string) {
|
||
t.Helper()
|
||
m := readMoney(t, bookPath)
|
||
type key struct {
|
||
ch, chunk int
|
||
stage string
|
||
}
|
||
ledger := map[key]float64{}
|
||
bankRoles := map[string]bool{roleTerminologist: true, roleClassifier: true}
|
||
counted := 0
|
||
for _, u := range m.checkpoints {
|
||
if bankRoles[u.Role] {
|
||
continue
|
||
}
|
||
ledger[key{u.Chapter, u.ChunkIdx, u.Stage}] += u.CostUSD
|
||
counted++
|
||
}
|
||
// The control: the instrument must have been given something to compare. «No mismatch» over an empty
|
||
// ledger and «no mismatch» over a real one read identically in a pass.
|
||
if counted == 0 || len(m.statuses) == 0 {
|
||
t.Fatalf("premise broken: nothing to compare — %d checkpoints outside the bank roles, %d status rows",
|
||
counted, len(m.statuses))
|
||
}
|
||
const cent = 0.0000005 // half a micro-dollar: the columns are float64 sums of the same values
|
||
for _, cs := range m.statuses {
|
||
// A row that cost money must say how many attempts bought it. `attempts=0` beside a non-zero
|
||
// cost is a row that reports work nobody did — measured at 0 attempts against $0.001056 — and it
|
||
// is the number an operator reads to tell «one call was cut» from «the position never started».
|
||
if cs.CostUSD > cent && cs.Attempts == 0 {
|
||
t.Fatalf("chunk_status ch%d/chunk%d/%s cost %.6f and reports attempts=0: the count is written "+
|
||
"after the error check, so a position that PAID reads as one that never started",
|
||
cs.Chapter, cs.ChunkIdx, cs.Stage, cs.CostUSD)
|
||
}
|
||
want := ledger[key{cs.Chapter, cs.ChunkIdx, cs.Stage}]
|
||
if diff := cs.CostUSD - want; diff > cent || diff < -cent {
|
||
t.Fatalf("chunk_status ch%d/chunk%d/%s says it cost %.6f while its own checkpoints add up to "+
|
||
"%.6f. The row is what the projection reads; money that reaches the ledger and not the row "+
|
||
"is money the book spent and nobody is shown.",
|
||
cs.Chapter, cs.ChunkIdx, cs.Stage, cs.CostUSD, want)
|
||
}
|
||
}
|
||
t.Logf("ledger invariant held over %d rows against %d checkpoints", len(m.statuses), counted)
|
||
}
|
||
|
||
// TestAStoppedRunLeavesEveryRowMatchingItsOwnLedger is the family's contract, pinned end to end: the
|
||
// position's money comes from settles written by different code on different exits, and the row that
|
||
// records it is written by a third piece on an error exit.
|
||
//
|
||
// ⚠ THE HOP'S MONEY IS MADE INEVITABLE RATHER THAN RACED FOR, and the first draft of this fixture got
|
||
// that wrong. It stopped the run over a hop held on the wire and relied on the stop landing AFTER the
|
||
// client had parsed the hop's headers — only an acknowledged call is billable. Measured on the mutated
|
||
// tree, where it must fail every time: **2 reds in 8 runs**. The other six the stop won the race, the cut
|
||
// settled for $0, and the row matched the ledger at nothing to nothing — a pin that passed by measuring
|
||
// an empty scenario.
|
||
//
|
||
// So the money is put there BEFORE the race can matter: run 1 burns the hop's key (two drops — one is
|
||
// retried inside the transport), and run 2's funnel walks that burn and carries its cost whatever
|
||
// happens to the fresh call. Whether the stop beats the headers or not, `esc.fb.cumCost` is non-zero,
|
||
// and the only question left is the one being asked — does that money reach the row.
|
||
func TestAStoppedRunLeavesEveryRowMatchingItsOwnLedger(t *testing.T) {
|
||
dir := t.TempDir()
|
||
var hopCalls atomic.Int32
|
||
hopHeld := make(chan struct{})
|
||
var once sync.Once
|
||
var srv *cutServer
|
||
srv = newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) {
|
||
if strings.Contains(lastBody(srv), `"fake-hop"`) {
|
||
if hopCalls.Add(1) <= 2 {
|
||
dropAfterHeaders(t, w) // run 1: two drops burn the hop's key — money, no result
|
||
return
|
||
}
|
||
once.Do(func() { close(hopHeld) }) // run 2: held, and the run is stopped over it
|
||
hold(req)
|
||
return
|
||
}
|
||
// The primary draft: a content filter — deterministic and escalatable, so a hop follows.
|
||
w.Header().Set("Content-Type", "application/json")
|
||
fmt.Fprint(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":"нет"},"finish_reason":"content_filter"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":10}}`)
|
||
})
|
||
bookPath := setupHopProject(t, dir, srv.srv.URL)
|
||
|
||
if err := runOnce(t, context.Background(), bookPath); err == nil {
|
||
t.Fatal("premise broken: run 1's hop was supposed to end on a delivered cut and burn its key")
|
||
}
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
defer cancel()
|
||
go func() {
|
||
<-hopHeld
|
||
cancel()
|
||
}()
|
||
_ = runOnce(t, ctx, bookPath)
|
||
|
||
// The premise, and it is what the first draft lacked: there IS hop money on this position, so the
|
||
// comparison below is not nothing against nothing.
|
||
m := readMoney(t, bookPath)
|
||
hopUSD := 0.0
|
||
for _, u := range m.checkpoints {
|
||
if u.Escalation {
|
||
hopUSD += u.CostUSD
|
||
}
|
||
}
|
||
if hopUSD <= 0 {
|
||
t.Fatalf("premise broken: the hop cost nothing, so this fixture is comparing an empty position. "+
|
||
"checkpoints=%d", len(m.checkpoints))
|
||
}
|
||
assertRowsMatchTheLedger(t, bookPath)
|
||
}
|
||
|
||
// TestABurnedKeysMoneyReachesTheRow is the other half of the same invariant, on the ordinary path. The
|
||
// funnel walks past a burned key and buys the work again one index up; the row must carry BOTH — the
|
||
// money that bought nothing and the money that bought the text. The walk's total was being overwritten
|
||
// by the fresh call's cost rather than added to it, so a position that was cut and then paid for read as
|
||
// if only the second call had happened.
|
||
func TestABurnedKeysMoneyReachesTheRow(t *testing.T) {
|
||
dir := t.TempDir()
|
||
var calls atomic.Int32
|
||
srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
|
||
// TWO drops, because one is not a burn: a lost connection is retried inside the transport, so a
|
||
// single drop followed by an answer never reaches the store as a cut. The delivered-cut cap makes
|
||
// the SECOND one terminal — that is what settles the key as money with no result.
|
||
if calls.Add(1) <= 2 {
|
||
dropAfterHeaders(t, w) // acknowledged, then cut
|
||
return
|
||
}
|
||
answerWhole(w)
|
||
})
|
||
bookPath := setupCutProject(t, dir, srv.srv.URL)
|
||
|
||
// Run 1 ends on the cut and leaves the burned key behind; run 2 walks past it and buys one index up.
|
||
if err := runOnce(t, context.Background(), bookPath); err == nil {
|
||
t.Fatal("premise broken: run 1 was supposed to end on a delivered cut")
|
||
}
|
||
if err := runOnce(t, context.Background(), bookPath); err != nil {
|
||
t.Fatalf("run 2 must complete: the burn is re-asked at the SAME budget one index up: %v", err)
|
||
}
|
||
m := readMoney(t, bookPath)
|
||
burned := 0
|
||
for _, u := range m.checkpoints {
|
||
if u.CostUSD > 0 && strings.TrimSpace(u.UsageJSON) == "{}" {
|
||
burned++
|
||
}
|
||
}
|
||
if burned == 0 {
|
||
t.Fatalf("premise broken: no burned checkpoint was written, so this fixture is measuring an "+
|
||
"ordinary run. checkpoints=%d", len(m.checkpoints))
|
||
}
|
||
assertRowsMatchTheLedger(t, bookPath)
|
||
}
|
||
|
||
// TestAStoppedRunCarriesTheRepairsMoneyToTheRow is the repair twin of the hop case. The repair sub-step
|
||
// runs INSIDE the shipping stage and returns its money together with its error, and the row that records
|
||
// the position is written on that error exit — so the same ordering defect lives here, and the lens that
|
||
// found it measured a ledger of 0.001303 against rows adding to 0.000240.
|
||
//
|
||
// The money is made inevitable the same way: run 1 burns the repair key, so run 2's walk carries that
|
||
// cost whatever the stop does to the fresh call.
|
||
func TestAStoppedRunCarriesTheRepairsMoneyToTheRow(t *testing.T) {
|
||
var repairCalls atomic.Int32
|
||
repairHeld := make(chan struct{})
|
||
var once sync.Once
|
||
var srv *cutServer
|
||
srv = newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) {
|
||
if isRepairBody(lastBody(srv)) {
|
||
if repairCalls.Add(1) <= 2 {
|
||
dropAfterHeaders(t, w) // run 1: two drops burn the repair's key
|
||
return
|
||
}
|
||
once.Do(func() { close(repairHeld) })
|
||
hold(req)
|
||
return
|
||
}
|
||
if isEditBody(lastBody(srv)) {
|
||
answerJSON(w, "Он прождал полчаса и вошёл внутрь.") // carries the defect the repair class detects
|
||
return
|
||
}
|
||
answerJSON(w, "ЧЕРНОВИК ПЕРЕВОДА.")
|
||
})
|
||
bookPath := setupRepairProject(t, srv.srv.URL)
|
||
|
||
if err := runOnce(t, context.Background(), bookPath); err == nil {
|
||
t.Fatal("premise broken: run 1's repair was supposed to end on a delivered cut and burn its key")
|
||
}
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
defer cancel()
|
||
go func() {
|
||
<-repairHeld
|
||
cancel()
|
||
}()
|
||
_ = runOnce(t, ctx, bookPath)
|
||
|
||
m := readMoney(t, bookPath)
|
||
repairUSD := 0.0
|
||
for _, u := range m.checkpoints {
|
||
if u.Role == roleRepair {
|
||
repairUSD += u.CostUSD
|
||
}
|
||
}
|
||
if repairUSD <= 0 {
|
||
t.Fatalf("premise broken: the repair cost nothing, so this fixture is comparing an empty position. "+
|
||
"checkpoints=%d", len(m.checkpoints))
|
||
}
|
||
assertRowsMatchTheLedger(t, bookPath)
|
||
}
|
||
|
||
// TestAStoppedPositionIsNotADecidedUnit pins the fourth carrier of the same question. A `cancelled` row
|
||
// records a position the engine was stopped over — the work was never done and the resume re-does it and
|
||
// pays again — so it decides nothing about the unit. Reading it as decided puts the unit into the
|
||
// extrapolation's denominator, and the projection then shows a book further along and cheaper than it
|
||
// is; the re-bill consent threshold, min($0.50, 5% × projected), shrinks with it.
|
||
//
|
||
// Both directions, because a predicate that answered «not decided» to everything would pass the first
|
||
// half alone and quietly turn the flagged state off for every real defect.
|
||
func TestAStoppedPositionIsNotADecidedUnit(t *testing.T) {
|
||
rows := func(reason FlagReason) []store.ChunkStatus {
|
||
return []store.ChunkStatus{
|
||
{Chapter: 1, ChunkIdx: 0, Stage: "draft", Disposition: string(DispOK), CostUSD: 0.001},
|
||
{Chapter: 1, ChunkIdx: 0, Stage: "edit", Disposition: string(DispFlagged),
|
||
FlagReason: string(reason), CostUSD: 0.002},
|
||
}
|
||
}
|
||
stopped := resolveChunkState(rows(FlagCancelled), 2)
|
||
if stopped.State == ChunkFlagged {
|
||
t.Fatalf("a stopped position decides nothing: the work was never done and the resume re-does it "+
|
||
"and pays again. Counting it as a decided unit puts it in the projection's denominator, so one "+
|
||
"stop makes the book look further along and cheaper than it is. got state=%v reason=%q",
|
||
stopped.State, stopped.Reason)
|
||
}
|
||
if stopped.State != ChunkInProgress {
|
||
t.Fatalf("a stopped position is still IN PROGRESS — that is what it is. got %v", stopped.State)
|
||
}
|
||
// The historical money is a fact and stays on the resolution whatever the state says.
|
||
if stopped.CostUSD <= 0 {
|
||
t.Fatalf("the money already spent on the position must still be reported: %.6f", stopped.CostUSD)
|
||
}
|
||
// The control: an ordinary flag still decides the unit, or this change has simply switched the
|
||
// flagged state off.
|
||
flagged := resolveChunkState(rows(FlagLength), 2)
|
||
if flagged.State != ChunkFlagged || flagged.Reason != string(FlagLength) {
|
||
t.Fatalf("a real content flag must still decide the unit: state=%v reason=%q", flagged.State, flagged.Reason)
|
||
}
|
||
}
|
||
|
||
// TestAStopOnAStagesFirstCallStillReportsAnAttempt is the count's own case. Every other fixture here has
|
||
// a call that succeeded before the stop, so the loop had already recorded a number; this one is stopped
|
||
// on the stage's FIRST call, which is where `attempts=0` beside real money was measured.
|
||
//
|
||
// The money is made inevitable the same way as the rest: run 1 burns the key, so run 2's walk carries a
|
||
// cost whatever the stop does to the fresh call.
|
||
func TestAStopOnAStagesFirstCallStillReportsAnAttempt(t *testing.T) {
|
||
dir := t.TempDir()
|
||
var calls atomic.Int32
|
||
held := make(chan struct{})
|
||
var once sync.Once
|
||
srv := newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) {
|
||
if calls.Add(1) <= 2 {
|
||
dropAfterHeaders(t, w) // run 1: two drops burn the first key
|
||
return
|
||
}
|
||
once.Do(func() { close(held) }) // run 2: held, and the run is stopped over it
|
||
hold(req)
|
||
})
|
||
bookPath := setupCutProject(t, dir, srv.srv.URL)
|
||
|
||
if err := runOnce(t, context.Background(), bookPath); err == nil {
|
||
t.Fatal("premise broken: run 1 was supposed to end on a delivered cut and burn the key")
|
||
}
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
defer cancel()
|
||
go func() {
|
||
<-held
|
||
cancel()
|
||
}()
|
||
_ = runOnce(t, ctx, bookPath)
|
||
|
||
m := readMoney(t, bookPath)
|
||
marked := false
|
||
for _, cs := range m.statuses {
|
||
if cs.FlagReason == string(FlagCancelled) {
|
||
marked = true
|
||
if cs.CostUSD <= 0 {
|
||
t.Fatalf("premise broken: the stopped position carries no money, so the count below proves "+
|
||
"nothing. row=%+v", cs)
|
||
}
|
||
if cs.Attempts == 0 {
|
||
t.Fatalf("the position PAID %.6f and reports attempts=0 — the stop landed on the stage's "+
|
||
"first call, and the count is written after the error check, so a position that was "+
|
||
"cut reads as one that never started", cs.CostUSD)
|
||
}
|
||
}
|
||
}
|
||
if !marked {
|
||
t.Fatalf("premise broken: no cancelled row was written, so this fixture measured nothing. rows=%d",
|
||
len(m.statuses))
|
||
}
|
||
assertRowsMatchTheLedger(t, bookPath)
|
||
}
|
||
|
||
// TestTheAskedNTimesNoteReachesTheReader pins the operator sentence at its source. Its two properties
|
||
// are invisible from the call site and both were wrong: the note sat behind a transport error longer
|
||
// than the column's own bound, and it claimed an estimate was booked on chains where nothing was.
|
||
func TestTheAskedNTimesNoteReachesTheReader(t *testing.T) {
|
||
long := errors.New(strings.Repeat("провайдер разорвал соединение на середине тела ответа. ", 6))
|
||
|
||
paid := cutErrLine(long, 2, 0.001056)
|
||
if !strings.HasPrefix(paid, "[the provider was asked 2 times;") {
|
||
t.Fatalf("the note must come FIRST — the column keeps 120 bytes and the error in front of it is "+
|
||
"longer than that, so a note at the end reaches nobody. got %q", paid)
|
||
}
|
||
if !strings.Contains(paid, "ONE estimate is booked") {
|
||
t.Fatalf("a chain that WAS billed must say one estimate covers it: %q", paid)
|
||
}
|
||
free := cutErrLine(long, 3, 0)
|
||
if strings.Contains(free, "ONE estimate is booked") {
|
||
t.Fatalf("nothing was booked for this chain, so the line must not claim an estimate was: %q", free)
|
||
}
|
||
if !strings.Contains(free, "NOTHING is booked") {
|
||
t.Fatalf("the line must say what actually happened to the money: %q", free)
|
||
}
|
||
// The control: the error itself is still carried, or the note would have replaced the diagnosis
|
||
// instead of leading it.
|
||
if !strings.Contains(paid, "разорвал соединение") {
|
||
t.Fatalf("the transport error must still be in the line: %q", paid)
|
||
}
|
||
}
|
||
|
||
// TestAStopBehindAPaidBreakStillMarksThePosition is the hole §4.2 forbids «at any moment», and it opens
|
||
// exactly where two correct mechanisms meet. The retry chain hands up the cut with the strongest MONEY
|
||
// claim, which is deliberately the earlier `connection_lost` when the later one is free — so a run a
|
||
// person stopped arrives at the mark carrying a cause that is not «stopped». The guard read that first
|
||
// cut's cause, decided this was not a stop, and returned in silence: money settled, no row, and the
|
||
// export shows a gap nobody can explain.
|
||
//
|
||
// The money is not what is lost here and the fixture says so: the ledger stays whole and the resume
|
||
// re-does the position correctly. What is lost is the position's visible mark — which is the one thing
|
||
// the ratified estimate was conditioned on.
|
||
func TestAStopBehindAPaidBreakStillMarksThePosition(t *testing.T) {
|
||
dir := t.TempDir()
|
||
var calls atomic.Int32
|
||
held := make(chan struct{})
|
||
var once sync.Once
|
||
srv := newCutServer(t, func(_ int, w http.ResponseWriter, req *http.Request) {
|
||
if calls.Add(1) == 1 {
|
||
// Attempt 1: acknowledged, then the socket dies — retryable, and the money of the chain.
|
||
dropAfterHeaders(t, w)
|
||
return
|
||
}
|
||
once.Do(func() { close(held) }) // attempt 2 is in flight when the person presses stop
|
||
hold(req)
|
||
})
|
||
bookPath := setupCutProject(t, dir, srv.srv.URL)
|
||
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
defer cancel()
|
||
go func() {
|
||
<-held
|
||
cancel()
|
||
}()
|
||
_ = runOnce(t, ctx, bookPath)
|
||
|
||
m := readMoney(t, bookPath)
|
||
// The premise, and it is the whole point: the chain really did carry a PAID break rather than the
|
||
// stop. Without it the fixture would pass on the ordinary «stopped over the first call» path, which
|
||
// was never broken.
|
||
paid := 0.0
|
||
for _, u := range m.checkpoints {
|
||
paid += u.CostUSD
|
||
}
|
||
if paid <= 0 {
|
||
t.Fatalf("premise broken: the chain must have settled a PAID break before the stop, else this "+
|
||
"fixture measures the case that already worked. checkpoints=%d", len(m.checkpoints))
|
||
}
|
||
marked := false
|
||
for _, cs := range m.statuses {
|
||
if cs.FlagReason == string(FlagCancelled) {
|
||
marked = true
|
||
}
|
||
}
|
||
if !marked {
|
||
t.Fatalf("a person stopped this run and its money is settled, but the position carries NO mark: "+
|
||
"the guard read the cause of whichever cut came first — a paid `connection_lost` the chain "+
|
||
"kept for its money — and decided a stop was not a stop. §4.2 forbids a hole with no mark at "+
|
||
"any moment. rows=%d, settled=%.6f", len(m.statuses), paid)
|
||
}
|
||
assertRowsMatchTheLedger(t, bookPath)
|
||
}
|
||
|
||
// TestTheCutRowPublishesItsEstimateAndItsGap pins the two carriers of DISCLOSURE on a cut row, and both
|
||
// were held by nothing. The ratified estimate is charged on one condition — that the row says it is an
|
||
// estimate (D39.230 п.1) — so these are not decoration: they are what the owner's word rests on.
|
||
//
|
||
// ⚠ WHY THE TABLE OF NINE OUTCOMES DID NOT COVER THEM. Its `wantEstimated` column reads the CHECKPOINT's
|
||
// usage through estimatedSpend, not `request_log.estimated`. In every fixture the two are true together,
|
||
// so the assertion could not tell them apart — the «two different quantities agreed» shape. This one
|
||
// reads the row it is talking about.
|
||
func TestTheCutRowPublishesItsEstimateAndItsGap(t *testing.T) {
|
||
dir := t.TempDir()
|
||
var calls atomic.Int32
|
||
srv := newCutServer(t, func(_ int, w http.ResponseWriter, _ *http.Request) {
|
||
// Two drops: one is retried inside the transport, the second is terminal — so the chain delivered
|
||
// TWICE and one estimate covers both, which is exactly the gap the row must publish.
|
||
if calls.Add(1) <= 2 {
|
||
dropAfterHeaders(t, w)
|
||
return
|
||
}
|
||
answerWhole(w)
|
||
})
|
||
bookPath := setupCutProject(t, dir, srv.srv.URL)
|
||
if err := runOnce(t, context.Background(), bookPath); err == nil {
|
||
t.Fatal("premise broken: the run was supposed to end on a delivered cut")
|
||
}
|
||
|
||
r, err := NewReadOnlyRunner(bookPath, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer r.Close()
|
||
rows, err := r.Store.RequestLogRows(r.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var cutRows int
|
||
for _, row := range rows {
|
||
if row.CostUSD <= 0 {
|
||
continue
|
||
}
|
||
cutRows++
|
||
if row.Estimated != 1 {
|
||
t.Fatalf("a row whose cost is a RESERVATION ESTIMATE must say so: the owner's word charges it "+
|
||
"on that condition, and a silent $%.6f is indistinguishable from a provider-reported one. "+
|
||
"row finish=%q estimated=%d", row.CostUSD, row.FinishReason, row.Estimated)
|
||
}
|
||
if row.EstTokens <= 0 {
|
||
t.Fatalf("an estimated row publishes the token figure the estimate was built from; 0 leaves "+
|
||
"the reader with a number and no way to weigh it. row finish=%q", row.FinishReason)
|
||
}
|
||
if !strings.Contains(row.Err, "asked 2 times") {
|
||
t.Fatalf("the provider was asked twice and ONE estimate is booked for both — the row is the "+
|
||
"only place that gap is published, and it says %q", row.Err)
|
||
}
|
||
}
|
||
// The control: there WAS a paid row to inspect. «No violations» over an empty set and over a real one
|
||
// print the same on a green test.
|
||
if cutRows == 0 {
|
||
t.Fatalf("premise broken: no paid row was written, so nothing above was checked. rows=%d", len(rows))
|
||
}
|
||
}
|