textmachine/backend/internal/pipeline/lowereffort_test.go

484 lines
21 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package pipeline
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// lowereffort_test.go: the retry that reads WHICH failure it is answering — `retries.lower_effort_on_empty`
// (config.Retries.LowerEffortOnEmpty), spent in stagerun.go's attempt loop.
//
// The engine had one remedy for two failures. A `length` cut with text in it ran out of room while the
// answer was being written, and a bigger budget is the cure (D2.3). An `empty` reply filled max_tokens
// before the answer began — on a subset-billing provider that budget went to thinking — and a bigger
// budget there is the move D39.86 falsified: at the vendor default effort, doubling 8496 to 16992 grew
// reasoning_content from 15 424 to 21 528 characters and the reply stayed empty.
//
// The paid run of 11.09 is the shape these tests model: four double-payments worth $0.106472 = 25.4% of
// the book, three of them `empty` at the ceiling (8496 · 8496 · 16000 completion tokens, zero characters
// of text) and one a truncated draft.
// burnsTheWholeCeiling answers every draft call the way the cold run's failures did: finish_reason
// `length`, no content at all, and completion_tokens EQUAL TO THE CEILING IT WAS GIVEN. That last part
// is the measured fact the money rests on — on all four failures of the run, completion_tokens was the
// max_tokens granted — and it is what makes a doubled retry cost exactly twice as much as the attempt
// it is repeating.
//
// When answerWhenEffortIsExplicit is set, a call that carries a reasoning_effort key answers normally;
// that arm is a MODEL of «less thinking leaves room for the answer», not evidence for it, and it is
// used only where a test is about what the engine does with a recovery, never about whether one happens.
// loweredDraftText is what ONLY the lowered draft attempt answers. It shares no substring with the
// fixture's editor reply, so an assertion about the draft cannot be satisfied by the editor's output.
const loweredDraftText = "ДРАФТ НА СНИЖЕННОЙ СТУПЕНИ"
type burnsTheWholeCeiling struct {
mu sync.Mutex
drafts int
answerWhenEffortIsExplicit bool
}
func (b *burnsTheWholeCeiling) handler(rec *reqRec) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
raw, _ := io.ReadAll(r.Body)
body := string(raw)
rec.record(body)
var req struct {
Model string `json:"model"`
MaxTokens int `json:"max_tokens"`
ReasoningEffort string `json:"reasoning_effort"`
}
_ = json.Unmarshal(raw, &req)
if req.Model == "" {
req.Model = "fake-model"
}
if isEditBody(body) {
fmt.Fprintf(w, `{"id":"f","model":%q,"choices":[{"message":{"content":"ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД"},"finish_reason":"stop"}],
"usage":{"prompt_tokens":100,"completion_tokens":200,"total_tokens":300}}`, req.Model)
return
}
b.mu.Lock()
b.drafts++
b.mu.Unlock()
if b.answerWhenEffortIsExplicit && req.ReasoningEffort != "" {
fmt.Fprintf(w, `{"id":"f","model":%q,"choices":[{"message":{"content":%q},"finish_reason":"stop"}],
"usage":{"prompt_tokens":100,"completion_tokens":200,"total_tokens":300,
"completion_tokens_details":{"reasoning_tokens":40}}}`, req.Model, loweredDraftText)
return
}
// The whole ceiling, bought and thrown away: no text, and every token of the budget spent
// thinking.
fmt.Fprintf(w, `{"id":"f","model":%q,"choices":[{"message":{"content":""},"finish_reason":"length"}],
"usage":{"prompt_tokens":100,"completion_tokens":%d,"total_tokens":%d,
"completion_tokens_details":{"reasoning_tokens":%d}}}`,
req.Model, req.MaxTokens, req.MaxTokens+100, req.MaxTokens)
}
}
func (b *burnsTheWholeCeiling) count() int { b.mu.Lock(); defer b.mu.Unlock(); return b.drafts }
// setupLowerEffort writes a one-chunk book whose draft rides the PROVIDER's own default thinking level
// — `reasoning: "off"` on a model with no reasoning capability is an absent key, which is the shape the
// shipping editor stage has (pipeline-c1.yaml) and the one the run's most expensive single failure sat
// on ($0.071009 for zero characters).
func setupLowerEffort(t *testing.T, providerURL string, lowerOnEmpty bool) string {
t.Helper()
dir := t.TempDir()
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
prices_checked: %q
default_model: fake-model
providers:
fake:
kind: openai
base_url: %q
reasoning: subset
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
models:
fake-model:
provider: fake
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
`, time.Now().UTC().Format("2006-01-02"), providerURL))
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
core: C1
version: 1
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
retries: { regenerate_before_escalate: 1, regenerate_echo_before_escalate: 0, lower_effort_on_empty: %t }
stages:
- { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
- { name: edit, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
escalation: { budget_usd: 0 }
`, lowerOnEmpty))
writeFile(t, filepath.Join(dir, "source.txt"), "静かな図書館の朝。")
writeFile(t, filepath.Join(dir, "book.yaml"), `
book_id: test-book
title: Тест
source_lang: ja
target_lang: ru
genre: ранобэ
audience: тест
venuti: 0.5
honorifics: keep
transcription: polivanov
footnotes: minimal
pipeline: pipeline.yaml
models: models.yaml
source_file: source.txt
ceilings: { book_usd: 5.0, day_usd: 10.0 }
`)
return filepath.Join(dir, "book.yaml")
}
// draftBodies returns the recorded request bodies of the DRAFT calls, in order.
func draftBodies(rec *reqRec) []string {
var out []string
for _, b := range rec.all() {
if !isEditBody(b) {
out = append(out, b)
}
}
return out
}
func bodyField(t *testing.T, body string) (maxTokens int, effort string) {
t.Helper()
var req struct {
MaxTokens int `json:"max_tokens"`
ReasoningEffort string `json:"reasoning_effort"`
}
if err := json.Unmarshal([]byte(body), &req); err != nil {
t.Fatalf("request body is not JSON: %v", err)
}
return req.MaxTokens, req.ReasoningEffort
}
// TestTheRetryForAnEmptyReplyBuysLessThinkingNotMoreBudget is the pack's claim measured on the wire and
// on the ledger at once, with NO assumption that lowering effort recovers anything: the provider burns
// whatever ceiling it is handed and returns nothing in both arms, so both flag, and the only difference
// is what the SECOND attempt was allowed to spend.
//
// The evidence for «the effort was lowered» is read off the SECOND request's own bytes, not off a field
// carried over from the first: an assertion satisfied by the previous attempt's state would be green on
// an engine that changed nothing.
func TestTheRetryForAnEmptyReplyBuysLessThinkingNotMoreBudget(t *testing.T) {
type arm struct {
lowerOnEmpty bool
wantMaxTok int // what the SECOND draft call was granted
wantEffort string // what the SECOND draft call asked for
}
var spend [2]float64
for i, a := range []arm{
{lowerOnEmpty: false, wantMaxTok: 1024, wantEffort: ""}, // the behaviour that shipped: twice the budget, same thinking
{lowerOnEmpty: true, wantMaxTok: 512, wantEffort: "low"}, // the remedy that matches the cause
} {
t.Run(fmt.Sprintf("lower_effort_on_empty=%t", a.lowerOnEmpty), func(t *testing.T) {
prov := &burnsTheWholeCeiling{}
rec := &reqRec{}
srv := httptest.NewServer(prov.handler(rec))
defer srv.Close()
r := newRunner(t, setupLowerEffort(t, srv.URL, a.lowerOnEmpty))
defer r.Close()
res, err := r.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
if len(res.Chunks) != 1 {
t.Fatalf("want 1 chunk, got %d", len(res.Chunks))
}
// Both arms end the same way, which is what keeps this test about the PRICE of the retry
// rather than about whether a retry helps.
if got := res.Chunks[0].FlagReason; got != FlagEmpty {
t.Fatalf("both arms must end flagged empty, got %q", got)
}
if got := prov.count(); got != 2 {
t.Fatalf("draft calls = %d, want 2 — one attempt and one regeneration, the budget the "+
"config grants either way", got)
}
bodies := draftBodies(rec)
if len(bodies) != 2 {
t.Fatalf("recorded draft bodies = %d, want 2", len(bodies))
}
if mt, eff := bodyField(t, bodies[0]); mt != 512 || eff != "" {
t.Fatalf("the FIRST attempt must be untouched by the knob: max_tokens=%d effort=%q, want 512 and the configured no-key", mt, eff)
}
mt, eff := bodyField(t, bodies[1])
if mt != a.wantMaxTok {
t.Fatalf("second attempt max_tokens = %d, want %d", mt, a.wantMaxTok)
}
if eff != a.wantEffort {
t.Fatalf("second attempt reasoning_effort = %q, want %q", eff, a.wantEffort)
}
committed, _, err := r.Store.SpentUSD(r.Book.BookID)
if err != nil {
t.Fatal(err)
}
if committed <= 0 {
t.Fatalf("the fixture must really spend money, else the comparison below is two zeros: %v", committed)
}
spend[i] = committed
})
}
// The money is structural, not incidental: the failing attempt bills its whole ceiling, so doubling
// the ceiling doubles what the failure costs. The direction is the knob's property; the ratio is
// this fixture's price table.
if !(spend[1] < spend[0]) {
t.Fatalf("answering an empty reply with less thinking must cost LESS than answering it with twice "+
"the budget, got lower_effort=%.8f doubling=%.8f", spend[1], spend[0])
}
}
// TestTheLoweredAttemptIsARealAttempt closes the half the wire assertion cannot see: the differently
// shaped request must be a normal paid attempt whose answer is USED — checkpointed, classified and
// shipped — and not a probe whose result is discarded.
func TestTheLoweredAttemptIsARealAttempt(t *testing.T) {
prov := &burnsTheWholeCeiling{answerWhenEffortIsExplicit: true}
rec := &reqRec{}
srv := httptest.NewServer(prov.handler(rec))
defer srv.Close()
r := newRunner(t, setupLowerEffort(t, srv.URL, true))
defer r.Close()
res, err := r.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
oc := res.Chunks[0]
if oc.Disposition != DispOK {
t.Fatalf("the recovered chunk must ship, got %s/%s", oc.Disposition, oc.FlagReason)
}
// ⚠ THE DRAFT STAGE'S OWN TEXT, NOT FinalText. The last stage is the editor, and this fixture's
// editor answers the same words unconditionally — so an assertion on the shipped text is satisfied
// by the EDITOR even when the lowered draft returned something else entirely, which is the check
// believing a different call than the one it names.
if len(oc.Stages) == 0 || oc.Stages[0].Stage != "draft" {
t.Fatalf("the fixture must put the draft first, got %+v", oc.Stages)
}
if got := oc.Stages[0].Text; got != loweredDraftText {
t.Fatalf("the draft stage must carry the LOWERED attempt's own words, got %q want %q", got, loweredDraftText)
}
// The FIRST failure keeps its durable trace even though the unit recovered — without it a book that
// paid twice reports as clean, which is how the mini-run of 25.07 reported echo_draft=0.0%.
cs, err := r.Store.GetChunkStatus(r.Book.BookID, 1, 0, "draft")
if err != nil || cs == nil {
t.Fatalf("the recovered unit must have a durable row: %v %v", cs, err)
}
if cs.FirstFlagReason != string(FlagEmpty) {
t.Fatalf("the recovered unit must still remember what it recovered FROM, got %q", cs.FirstFlagReason)
}
if got := prov.count(); got != 2 {
t.Fatalf("draft calls = %d, want 2", got)
}
}
// TestLowerEffortKnobMovesNoSnapshot is the gate that lets this land on books already in flight. The
// knob lives in Retries, which is deliberately outside buildSnapshotID, so turning it on must not
// invalidate a single paid checkpoint. Grepping snapshot.go for the field name would prove only that a
// NAME is absent; this renders the ids from two real configs that differ in that key and nothing else.
//
// ⚠ THE LOAD PATH IS MEASURED FIRST, and the ordering is load-bearing (the lesson of the sibling gate,
// echoregen_test.go): most snapshot inputs are resolved at LOAD, before a Runner exists, so a fold
// derived from the same YAML key would sail straight through an in-memory poke while two BOOKS
// differing only in that key rendered different ids.
//
// ⚠ WHAT IT DOES NOT SAY. Turning the knob on changes the SHAPE of attempt ≥ 1 — a different effort and
// a different budget are both in RequestHash — so a unit that already holds a stored regeneration at
// the doubled budget will not find it and will buy one more call. Attempt 0 is untouched, so nothing
// already delivered is re-bought; the cost of flipping this mid-book is one call per unit that had
// already been retried, and it is named here rather than discovered on a bill.
func TestLowerEffortKnobMovesNoSnapshot(t *testing.T) {
prov := &burnsTheWholeCeiling{answerWhenEffortIsExplicit: true}
rec := &reqRec{}
srv := httptest.NewServer(prov.handler(rec))
defer srv.Close()
renderFor := func(t *testing.T, lower bool) (string, string) {
t.Helper()
rr := newRunner(t, setupLowerEffort(t, srv.URL, lower))
defer rr.Close()
if got := rr.Pipeline.Retries.LowerEffortOnEmpty; got != lower {
t.Fatalf("the fixture's YAML did not reach the loaded config: want %t, got %t", lower, got)
}
d, _, err := rr.snapshotIDForWave(waveDraft)
if err != nil {
t.Fatalf("render the draft snapshot at lower=%t: %v", lower, err)
}
e, _, err := rr.snapshotIDForWave(waveEdit)
if err != nil {
t.Fatalf("render the edit snapshot at lower=%t: %v", lower, err)
}
return d, e
}
dOff, eOff := renderFor(t, false)
dOn, eOn := renderFor(t, true)
if dOn != dOff || eOn != eOff {
t.Fatalf("the knob moved a wave snapshot — every paid checkpoint of every book would be re-bought "+
"(--resnapshot).\n draft %s -> %s\n edit %s -> %s", dOff, dOn, eOff, eOn)
}
// The in-memory arm, as the cheap second axis: the field itself must not reach the id either.
r := newRunner(t, setupLowerEffort(t, srv.URL, false))
defer r.Close()
read := func(what string) (string, string) {
t.Helper()
d, _, err := r.snapshotIDForWave(waveDraft)
if err != nil {
t.Fatalf("render the draft snapshot (%s): %v", what, err)
}
e, _, err := r.snapshotIDForWave(waveEdit)
if err != nil {
t.Fatalf("render the edit snapshot (%s): %v", what, err)
}
return d, e
}
draftOff, editOff := read("knob off")
r.Pipeline.Retries.LowerEffortOnEmpty = true
draftOn, editOn := read("knob on")
if draftOn != draftOff || editOn != editOff {
t.Fatalf("the knob's field reached a wave snapshot.\n draft %s -> %s\n edit %s -> %s",
draftOff, draftOn, editOff, editOn)
}
}
// TestATruncatedAnswerStillBuysMoreBudget is the other half of the split, and without it the pack would
// have replaced one blind remedy with another. A `length` cut that carried TEXT ran out of room while
// the answer was being written — D2.3's own case — so it must keep getting a bigger budget even with
// the knob on, and its effort must not move.
func TestATruncatedAnswerStillBuysMoreBudget(t *testing.T) {
rec := &reqRec{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, _ := io.ReadAll(r.Body)
body := string(raw)
rec.record(body)
if isEditBody(body) {
fmt.Fprint(w, `{"id":"f","model":"fake-model","choices":[{"message":{"content":"ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД"},"finish_reason":"stop"}],
"usage":{"prompt_tokens":100,"completion_tokens":200,"total_tokens":300}}`)
return
}
// A draft cut mid-sentence: text IS there, the budget ran out around it.
fmt.Fprint(w, `{"id":"f","model":"fake-model","choices":[{"message":{"content":"ЧЕРНОВИК ПЕРЕВОДА, оборванный на середине фразы и"},"finish_reason":"length"}],
"usage":{"prompt_tokens":100,"completion_tokens":512,"total_tokens":612}}`)
}))
defer srv.Close()
r := newRunner(t, setupLowerEffort(t, srv.URL, true)) // the knob is ON, and must still not fire here
defer r.Close()
if _, err := r.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
bodies := draftBodies(rec)
if len(bodies) != 2 {
t.Fatalf("recorded draft bodies = %d, want 2", len(bodies))
}
mt, eff := bodyField(t, bodies[1])
if mt != 1024 {
t.Fatalf("a truncated ANSWER must be re-asked with more room: second attempt max_tokens = %d, want 1024", mt)
}
if eff != "" {
t.Fatalf("a truncated answer says nothing about thinking; the effort must stay as configured, got %q", eff)
}
}
// TestTheLoweringIsAnnouncedWithBothLevels pins the WARN that is the ONLY durable trace of WHICH effort
// an attempt was bought at: the level lives in the request hash and nowhere else, so a run on disk can
// show that an attempt thought less and never what it was asked for.
//
// ⚠ THE OPERATOR-MESSAGE CATALOGUE DOES NOT COVER THIS. That gate compares the literals present in the
// SOURCE against a file; a line can keep its literal, lose its fields, or never be reached at all, and
// the catalogue stays green. Measured: deleting `"effort", effort, "next_effort", lower` from this call
// left the whole package green before this test existed.
func TestTheLoweringIsAnnouncedWithBothLevels(t *testing.T) {
prov := &burnsTheWholeCeiling{answerWhenEffortIsExplicit: true}
rec := &reqRec{}
srv := httptest.NewServer(prov.handler(rec))
defer srv.Close()
var log bytes.Buffer
r := newRunner(t, setupLowerEffort(t, srv.URL, true))
defer r.Close()
r.Log = slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelWarn}))
if _, err := r.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
line := warnLine(t, log.String(), "regenerating with less thinking")
// BOTH levels, because either alone is unreadable: «next_effort=low» does not say what was given up,
// and «effort=off» does not say what was asked for instead.
for _, want := range []string{`effort=off`, `next_effort=low`, `max_tokens=512`, `reason=empty`} {
if !strings.Contains(line, want) {
t.Fatalf("the lowering must announce %q — it is the only place the ASKED-FOR level is ever\nrecorded: %s", want, line)
}
}
}
// TestASubstitutedPriceIsAnnouncedOnTheCallThatWasBilled is the other half of the same gap: the price
// basis is computed at settle and stored nowhere, so this line is the live path's only voice. §4.5 of
// the pack claims «the live path warns»; measured, `if false && basis.Substituted()` left the package
// green before this test.
func TestASubstitutedPriceIsAnnouncedOnTheCallThatWasBilled(t *testing.T) {
// The provider answers under a slug the catalogue does not carry, which is the measured shape:
// deepseek-v4-flash was asked for and `deepseek-flash` answered.
rec := &reqRec{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, _ := io.ReadAll(r.Body)
rec.record(string(raw))
fmt.Fprint(w, `{"id":"f","model":"fake-model-0813","choices":[{"message":{"content":"ЧЕРНОВИК ПЕРЕВОДА"},"finish_reason":"stop"}],
"usage":{"prompt_tokens":100,"completion_tokens":200,"total_tokens":300}}`)
}))
defer srv.Close()
var log bytes.Buffer
r := newRunner(t, setupLowerEffort(t, srv.URL, false))
defer r.Close()
r.Log = slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelWarn}))
if _, err := r.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
line := warnLine(t, log.String(), "priced by a model that did not answer")
for _, want := range []string{`requested=fake-model`, `answered=fake-model-0813`, `priced_by=requested`} {
if !strings.Contains(line, want) {
t.Fatalf("the substitution must name %q — both slugs and the rate that was used, or the\nreader cannot tell which direction the bill went: %s", want, line)
}
}
// The control, without which the test above proves nothing: a call priced by the model that ANSWERED
// must stay silent, else the line fires on every call and stops being a signal.
var quiet bytes.Buffer
prov := &burnsTheWholeCeiling{answerWhenEffortIsExplicit: true}
srv2 := httptest.NewServer(prov.handler(&reqRec{})) // answers under the model it was asked for
defer srv2.Close()
r2 := newRunner(t, setupLowerEffort(t, srv2.URL, false))
defer r2.Close()
r2.Log = slog.New(slog.NewTextHandler(&quiet, &slog.HandlerOptions{Level: slog.LevelWarn}))
if _, err := r2.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
if strings.Contains(quiet.String(), "priced by a model that did not answer") {
t.Fatalf("a call priced by its own answerer must say nothing:\n%s", quiet.String())
}
}
// warnLine returns the first logged line containing needle, so an assertion about that line's FIELDS
// cannot be satisfied by attributes of some other message in the stream.
func warnLine(t *testing.T, out, needle string) string {
t.Helper()
for _, l := range strings.Split(out, "\n") {
if strings.Contains(l, needle) {
return l
}
}
t.Fatalf("no logged line contains %q — the message never sounded:\n%s", needle, out)
return ""
}