484 lines
21 KiB
Go
484 lines
21 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log/slog"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// lowereffort_test.go: the retry that reads WHICH failure it is answering — `retries.lower_effort_on_empty`
|
||
// (config.Retries.LowerEffortOnEmpty), spent in stagerun.go's attempt loop.
|
||
//
|
||
// The engine had one remedy for two failures. A `length` cut with text in it ran out of room while the
|
||
// answer was being written, and a bigger budget is the cure (D2.3). An `empty` reply filled max_tokens
|
||
// before the answer began — on a subset-billing provider that budget went to thinking — and a bigger
|
||
// budget there is the move D39.86 falsified: at the vendor default effort, doubling 8496 to 16992 grew
|
||
// reasoning_content from 15 424 to 21 528 characters and the reply stayed empty.
|
||
//
|
||
// The paid run of 11.09 is the shape these tests model: four double-payments worth $0.106472 = 25.4% of
|
||
// the book, three of them `empty` at the ceiling (8496 · 8496 · 16000 completion tokens, zero characters
|
||
// of text) and one a truncated draft.
|
||
|
||
// burnsTheWholeCeiling answers every draft call the way the cold run's failures did: finish_reason
|
||
// `length`, no content at all, and completion_tokens EQUAL TO THE CEILING IT WAS GIVEN. That last part
|
||
// is the measured fact the money rests on — on all four failures of the run, completion_tokens was the
|
||
// max_tokens granted — and it is what makes a doubled retry cost exactly twice as much as the attempt
|
||
// it is repeating.
|
||
//
|
||
// When answerWhenEffortIsExplicit is set, a call that carries a reasoning_effort key answers normally;
|
||
// that arm is a MODEL of «less thinking leaves room for the answer», not evidence for it, and it is
|
||
// used only where a test is about what the engine does with a recovery, never about whether one happens.
|
||
// loweredDraftText is what ONLY the lowered draft attempt answers. It shares no substring with the
|
||
// fixture's editor reply, so an assertion about the draft cannot be satisfied by the editor's output.
|
||
const loweredDraftText = "ДРАФТ НА СНИЖЕННОЙ СТУПЕНИ"
|
||
|
||
type burnsTheWholeCeiling struct {
|
||
mu sync.Mutex
|
||
drafts int
|
||
answerWhenEffortIsExplicit bool
|
||
}
|
||
|
||
func (b *burnsTheWholeCeiling) handler(rec *reqRec) http.HandlerFunc {
|
||
return func(w http.ResponseWriter, r *http.Request) {
|
||
raw, _ := io.ReadAll(r.Body)
|
||
body := string(raw)
|
||
rec.record(body)
|
||
var req struct {
|
||
Model string `json:"model"`
|
||
MaxTokens int `json:"max_tokens"`
|
||
ReasoningEffort string `json:"reasoning_effort"`
|
||
}
|
||
_ = json.Unmarshal(raw, &req)
|
||
if req.Model == "" {
|
||
req.Model = "fake-model"
|
||
}
|
||
if isEditBody(body) {
|
||
fmt.Fprintf(w, `{"id":"f","model":%q,"choices":[{"message":{"content":"ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":200,"total_tokens":300}}`, req.Model)
|
||
return
|
||
}
|
||
b.mu.Lock()
|
||
b.drafts++
|
||
b.mu.Unlock()
|
||
if b.answerWhenEffortIsExplicit && req.ReasoningEffort != "" {
|
||
fmt.Fprintf(w, `{"id":"f","model":%q,"choices":[{"message":{"content":%q},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":200,"total_tokens":300,
|
||
"completion_tokens_details":{"reasoning_tokens":40}}}`, req.Model, loweredDraftText)
|
||
return
|
||
}
|
||
// The whole ceiling, bought and thrown away: no text, and every token of the budget spent
|
||
// thinking.
|
||
fmt.Fprintf(w, `{"id":"f","model":%q,"choices":[{"message":{"content":""},"finish_reason":"length"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":%d,"total_tokens":%d,
|
||
"completion_tokens_details":{"reasoning_tokens":%d}}}`,
|
||
req.Model, req.MaxTokens, req.MaxTokens+100, req.MaxTokens)
|
||
}
|
||
}
|
||
|
||
func (b *burnsTheWholeCeiling) count() int { b.mu.Lock(); defer b.mu.Unlock(); return b.drafts }
|
||
|
||
// setupLowerEffort writes a one-chunk book whose draft rides the PROVIDER's own default thinking level
|
||
// — `reasoning: "off"` on a model with no reasoning capability is an absent key, which is the shape the
|
||
// shipping editor stage has (pipeline-c1.yaml) and the one the run's most expensive single failure sat
|
||
// on ($0.071009 for zero characters).
|
||
func setupLowerEffort(t *testing.T, providerURL string, lowerOnEmpty bool) string {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
|
||
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q
|
||
reasoning: subset
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||
retries: { regenerate_before_escalate: 1, regenerate_echo_before_escalate: 0, lower_effort_on_empty: %t }
|
||
stages:
|
||
- { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
||
- { name: edit, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||
escalation: { budget_usd: 0 }
|
||
`, lowerOnEmpty))
|
||
writeFile(t, filepath.Join(dir, "source.txt"), "静かな図書館の朝。")
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), `
|
||
book_id: test-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.txt
|
||
ceilings: { book_usd: 5.0, day_usd: 10.0 }
|
||
`)
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
// draftBodies returns the recorded request bodies of the DRAFT calls, in order.
|
||
func draftBodies(rec *reqRec) []string {
|
||
var out []string
|
||
for _, b := range rec.all() {
|
||
if !isEditBody(b) {
|
||
out = append(out, b)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func bodyField(t *testing.T, body string) (maxTokens int, effort string) {
|
||
t.Helper()
|
||
var req struct {
|
||
MaxTokens int `json:"max_tokens"`
|
||
ReasoningEffort string `json:"reasoning_effort"`
|
||
}
|
||
if err := json.Unmarshal([]byte(body), &req); err != nil {
|
||
t.Fatalf("request body is not JSON: %v", err)
|
||
}
|
||
return req.MaxTokens, req.ReasoningEffort
|
||
}
|
||
|
||
// TestTheRetryForAnEmptyReplyBuysLessThinkingNotMoreBudget is the pack's claim measured on the wire and
|
||
// on the ledger at once, with NO assumption that lowering effort recovers anything: the provider burns
|
||
// whatever ceiling it is handed and returns nothing in both arms, so both flag, and the only difference
|
||
// is what the SECOND attempt was allowed to spend.
|
||
//
|
||
// The evidence for «the effort was lowered» is read off the SECOND request's own bytes, not off a field
|
||
// carried over from the first: an assertion satisfied by the previous attempt's state would be green on
|
||
// an engine that changed nothing.
|
||
func TestTheRetryForAnEmptyReplyBuysLessThinkingNotMoreBudget(t *testing.T) {
|
||
type arm struct {
|
||
lowerOnEmpty bool
|
||
wantMaxTok int // what the SECOND draft call was granted
|
||
wantEffort string // what the SECOND draft call asked for
|
||
}
|
||
var spend [2]float64
|
||
for i, a := range []arm{
|
||
{lowerOnEmpty: false, wantMaxTok: 1024, wantEffort: ""}, // the behaviour that shipped: twice the budget, same thinking
|
||
{lowerOnEmpty: true, wantMaxTok: 512, wantEffort: "low"}, // the remedy that matches the cause
|
||
} {
|
||
t.Run(fmt.Sprintf("lower_effort_on_empty=%t", a.lowerOnEmpty), func(t *testing.T) {
|
||
prov := &burnsTheWholeCeiling{}
|
||
rec := &reqRec{}
|
||
srv := httptest.NewServer(prov.handler(rec))
|
||
defer srv.Close()
|
||
|
||
r := newRunner(t, setupLowerEffort(t, srv.URL, a.lowerOnEmpty))
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(res.Chunks) != 1 {
|
||
t.Fatalf("want 1 chunk, got %d", len(res.Chunks))
|
||
}
|
||
// Both arms end the same way, which is what keeps this test about the PRICE of the retry
|
||
// rather than about whether a retry helps.
|
||
if got := res.Chunks[0].FlagReason; got != FlagEmpty {
|
||
t.Fatalf("both arms must end flagged empty, got %q", got)
|
||
}
|
||
if got := prov.count(); got != 2 {
|
||
t.Fatalf("draft calls = %d, want 2 — one attempt and one regeneration, the budget the "+
|
||
"config grants either way", got)
|
||
}
|
||
bodies := draftBodies(rec)
|
||
if len(bodies) != 2 {
|
||
t.Fatalf("recorded draft bodies = %d, want 2", len(bodies))
|
||
}
|
||
if mt, eff := bodyField(t, bodies[0]); mt != 512 || eff != "" {
|
||
t.Fatalf("the FIRST attempt must be untouched by the knob: max_tokens=%d effort=%q, want 512 and the configured no-key", mt, eff)
|
||
}
|
||
mt, eff := bodyField(t, bodies[1])
|
||
if mt != a.wantMaxTok {
|
||
t.Fatalf("second attempt max_tokens = %d, want %d", mt, a.wantMaxTok)
|
||
}
|
||
if eff != a.wantEffort {
|
||
t.Fatalf("second attempt reasoning_effort = %q, want %q", eff, a.wantEffort)
|
||
}
|
||
committed, _, err := r.Store.SpentUSD(r.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if committed <= 0 {
|
||
t.Fatalf("the fixture must really spend money, else the comparison below is two zeros: %v", committed)
|
||
}
|
||
spend[i] = committed
|
||
})
|
||
}
|
||
// The money is structural, not incidental: the failing attempt bills its whole ceiling, so doubling
|
||
// the ceiling doubles what the failure costs. The direction is the knob's property; the ratio is
|
||
// this fixture's price table.
|
||
if !(spend[1] < spend[0]) {
|
||
t.Fatalf("answering an empty reply with less thinking must cost LESS than answering it with twice "+
|
||
"the budget, got lower_effort=%.8f doubling=%.8f", spend[1], spend[0])
|
||
}
|
||
}
|
||
|
||
// TestTheLoweredAttemptIsARealAttempt closes the half the wire assertion cannot see: the differently
|
||
// shaped request must be a normal paid attempt whose answer is USED — checkpointed, classified and
|
||
// shipped — and not a probe whose result is discarded.
|
||
func TestTheLoweredAttemptIsARealAttempt(t *testing.T) {
|
||
prov := &burnsTheWholeCeiling{answerWhenEffortIsExplicit: true}
|
||
rec := &reqRec{}
|
||
srv := httptest.NewServer(prov.handler(rec))
|
||
defer srv.Close()
|
||
|
||
r := newRunner(t, setupLowerEffort(t, srv.URL, true))
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
oc := res.Chunks[0]
|
||
if oc.Disposition != DispOK {
|
||
t.Fatalf("the recovered chunk must ship, got %s/%s", oc.Disposition, oc.FlagReason)
|
||
}
|
||
// ⚠ THE DRAFT STAGE'S OWN TEXT, NOT FinalText. The last stage is the editor, and this fixture's
|
||
// editor answers the same words unconditionally — so an assertion on the shipped text is satisfied
|
||
// by the EDITOR even when the lowered draft returned something else entirely, which is the check
|
||
// believing a different call than the one it names.
|
||
if len(oc.Stages) == 0 || oc.Stages[0].Stage != "draft" {
|
||
t.Fatalf("the fixture must put the draft first, got %+v", oc.Stages)
|
||
}
|
||
if got := oc.Stages[0].Text; got != loweredDraftText {
|
||
t.Fatalf("the draft stage must carry the LOWERED attempt's own words, got %q want %q", got, loweredDraftText)
|
||
}
|
||
// The FIRST failure keeps its durable trace even though the unit recovered — without it a book that
|
||
// paid twice reports as clean, which is how the mini-run of 25.07 reported echo_draft=0.0%.
|
||
cs, err := r.Store.GetChunkStatus(r.Book.BookID, 1, 0, "draft")
|
||
if err != nil || cs == nil {
|
||
t.Fatalf("the recovered unit must have a durable row: %v %v", cs, err)
|
||
}
|
||
if cs.FirstFlagReason != string(FlagEmpty) {
|
||
t.Fatalf("the recovered unit must still remember what it recovered FROM, got %q", cs.FirstFlagReason)
|
||
}
|
||
if got := prov.count(); got != 2 {
|
||
t.Fatalf("draft calls = %d, want 2", got)
|
||
}
|
||
}
|
||
|
||
// TestLowerEffortKnobMovesNoSnapshot is the gate that lets this land on books already in flight. The
|
||
// knob lives in Retries, which is deliberately outside buildSnapshotID, so turning it on must not
|
||
// invalidate a single paid checkpoint. Grepping snapshot.go for the field name would prove only that a
|
||
// NAME is absent; this renders the ids from two real configs that differ in that key and nothing else.
|
||
//
|
||
// ⚠ THE LOAD PATH IS MEASURED FIRST, and the ordering is load-bearing (the lesson of the sibling gate,
|
||
// echoregen_test.go): most snapshot inputs are resolved at LOAD, before a Runner exists, so a fold
|
||
// derived from the same YAML key would sail straight through an in-memory poke while two BOOKS
|
||
// differing only in that key rendered different ids.
|
||
//
|
||
// ⚠ WHAT IT DOES NOT SAY. Turning the knob on changes the SHAPE of attempt ≥ 1 — a different effort and
|
||
// a different budget are both in RequestHash — so a unit that already holds a stored regeneration at
|
||
// the doubled budget will not find it and will buy one more call. Attempt 0 is untouched, so nothing
|
||
// already delivered is re-bought; the cost of flipping this mid-book is one call per unit that had
|
||
// already been retried, and it is named here rather than discovered on a bill.
|
||
func TestLowerEffortKnobMovesNoSnapshot(t *testing.T) {
|
||
prov := &burnsTheWholeCeiling{answerWhenEffortIsExplicit: true}
|
||
rec := &reqRec{}
|
||
srv := httptest.NewServer(prov.handler(rec))
|
||
defer srv.Close()
|
||
|
||
renderFor := func(t *testing.T, lower bool) (string, string) {
|
||
t.Helper()
|
||
rr := newRunner(t, setupLowerEffort(t, srv.URL, lower))
|
||
defer rr.Close()
|
||
if got := rr.Pipeline.Retries.LowerEffortOnEmpty; got != lower {
|
||
t.Fatalf("the fixture's YAML did not reach the loaded config: want %t, got %t", lower, got)
|
||
}
|
||
d, _, err := rr.snapshotIDForWave(waveDraft)
|
||
if err != nil {
|
||
t.Fatalf("render the draft snapshot at lower=%t: %v", lower, err)
|
||
}
|
||
e, _, err := rr.snapshotIDForWave(waveEdit)
|
||
if err != nil {
|
||
t.Fatalf("render the edit snapshot at lower=%t: %v", lower, err)
|
||
}
|
||
return d, e
|
||
}
|
||
dOff, eOff := renderFor(t, false)
|
||
dOn, eOn := renderFor(t, true)
|
||
if dOn != dOff || eOn != eOff {
|
||
t.Fatalf("the knob moved a wave snapshot — every paid checkpoint of every book would be re-bought "+
|
||
"(--resnapshot).\n draft %s -> %s\n edit %s -> %s", dOff, dOn, eOff, eOn)
|
||
}
|
||
|
||
// The in-memory arm, as the cheap second axis: the field itself must not reach the id either.
|
||
r := newRunner(t, setupLowerEffort(t, srv.URL, false))
|
||
defer r.Close()
|
||
read := func(what string) (string, string) {
|
||
t.Helper()
|
||
d, _, err := r.snapshotIDForWave(waveDraft)
|
||
if err != nil {
|
||
t.Fatalf("render the draft snapshot (%s): %v", what, err)
|
||
}
|
||
e, _, err := r.snapshotIDForWave(waveEdit)
|
||
if err != nil {
|
||
t.Fatalf("render the edit snapshot (%s): %v", what, err)
|
||
}
|
||
return d, e
|
||
}
|
||
draftOff, editOff := read("knob off")
|
||
r.Pipeline.Retries.LowerEffortOnEmpty = true
|
||
draftOn, editOn := read("knob on")
|
||
if draftOn != draftOff || editOn != editOff {
|
||
t.Fatalf("the knob's field reached a wave snapshot.\n draft %s -> %s\n edit %s -> %s",
|
||
draftOff, draftOn, editOff, editOn)
|
||
}
|
||
}
|
||
|
||
// TestATruncatedAnswerStillBuysMoreBudget is the other half of the split, and without it the pack would
|
||
// have replaced one blind remedy with another. A `length` cut that carried TEXT ran out of room while
|
||
// the answer was being written — D2.3's own case — so it must keep getting a bigger budget even with
|
||
// the knob on, and its effort must not move.
|
||
func TestATruncatedAnswerStillBuysMoreBudget(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
raw, _ := io.ReadAll(r.Body)
|
||
body := string(raw)
|
||
rec.record(body)
|
||
if isEditBody(body) {
|
||
fmt.Fprint(w, `{"id":"f","model":"fake-model","choices":[{"message":{"content":"ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":200,"total_tokens":300}}`)
|
||
return
|
||
}
|
||
// A draft cut mid-sentence: text IS there, the budget ran out around it.
|
||
fmt.Fprint(w, `{"id":"f","model":"fake-model","choices":[{"message":{"content":"ЧЕРНОВИК ПЕРЕВОДА, оборванный на середине фразы и"},"finish_reason":"length"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":512,"total_tokens":612}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
r := newRunner(t, setupLowerEffort(t, srv.URL, true)) // the knob is ON, and must still not fire here
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
bodies := draftBodies(rec)
|
||
if len(bodies) != 2 {
|
||
t.Fatalf("recorded draft bodies = %d, want 2", len(bodies))
|
||
}
|
||
mt, eff := bodyField(t, bodies[1])
|
||
if mt != 1024 {
|
||
t.Fatalf("a truncated ANSWER must be re-asked with more room: second attempt max_tokens = %d, want 1024", mt)
|
||
}
|
||
if eff != "" {
|
||
t.Fatalf("a truncated answer says nothing about thinking; the effort must stay as configured, got %q", eff)
|
||
}
|
||
}
|
||
|
||
// TestTheLoweringIsAnnouncedWithBothLevels pins the WARN that is the ONLY durable trace of WHICH effort
|
||
// an attempt was bought at: the level lives in the request hash and nowhere else, so a run on disk can
|
||
// show that an attempt thought less and never what it was asked for.
|
||
//
|
||
// ⚠ THE OPERATOR-MESSAGE CATALOGUE DOES NOT COVER THIS. That gate compares the literals present in the
|
||
// SOURCE against a file; a line can keep its literal, lose its fields, or never be reached at all, and
|
||
// the catalogue stays green. Measured: deleting `"effort", effort, "next_effort", lower` from this call
|
||
// left the whole package green before this test existed.
|
||
func TestTheLoweringIsAnnouncedWithBothLevels(t *testing.T) {
|
||
prov := &burnsTheWholeCeiling{answerWhenEffortIsExplicit: true}
|
||
rec := &reqRec{}
|
||
srv := httptest.NewServer(prov.handler(rec))
|
||
defer srv.Close()
|
||
|
||
var log bytes.Buffer
|
||
r := newRunner(t, setupLowerEffort(t, srv.URL, true))
|
||
defer r.Close()
|
||
r.Log = slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
line := warnLine(t, log.String(), "regenerating with less thinking")
|
||
// BOTH levels, because either alone is unreadable: «next_effort=low» does not say what was given up,
|
||
// and «effort=off» does not say what was asked for instead.
|
||
for _, want := range []string{`effort=off`, `next_effort=low`, `max_tokens=512`, `reason=empty`} {
|
||
if !strings.Contains(line, want) {
|
||
t.Fatalf("the lowering must announce %q — it is the only place the ASKED-FOR level is ever\nrecorded: %s", want, line)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestASubstitutedPriceIsAnnouncedOnTheCallThatWasBilled is the other half of the same gap: the price
|
||
// basis is computed at settle and stored nowhere, so this line is the live path's only voice. §4.5 of
|
||
// the pack claims «the live path warns»; measured, `if false && basis.Substituted()` left the package
|
||
// green before this test.
|
||
func TestASubstitutedPriceIsAnnouncedOnTheCallThatWasBilled(t *testing.T) {
|
||
// The provider answers under a slug the catalogue does not carry, which is the measured shape:
|
||
// deepseek-v4-flash was asked for and `deepseek-flash` answered.
|
||
rec := &reqRec{}
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
raw, _ := io.ReadAll(r.Body)
|
||
rec.record(string(raw))
|
||
fmt.Fprint(w, `{"id":"f","model":"fake-model-0813","choices":[{"message":{"content":"ЧЕРНОВИК ПЕРЕВОДА"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":200,"total_tokens":300}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
var log bytes.Buffer
|
||
r := newRunner(t, setupLowerEffort(t, srv.URL, false))
|
||
defer r.Close()
|
||
r.Log = slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
line := warnLine(t, log.String(), "priced by a model that did not answer")
|
||
for _, want := range []string{`requested=fake-model`, `answered=fake-model-0813`, `priced_by=requested`} {
|
||
if !strings.Contains(line, want) {
|
||
t.Fatalf("the substitution must name %q — both slugs and the rate that was used, or the\nreader cannot tell which direction the bill went: %s", want, line)
|
||
}
|
||
}
|
||
|
||
// The control, without which the test above proves nothing: a call priced by the model that ANSWERED
|
||
// must stay silent, else the line fires on every call and stops being a signal.
|
||
var quiet bytes.Buffer
|
||
prov := &burnsTheWholeCeiling{answerWhenEffortIsExplicit: true}
|
||
srv2 := httptest.NewServer(prov.handler(&reqRec{})) // answers under the model it was asked for
|
||
defer srv2.Close()
|
||
r2 := newRunner(t, setupLowerEffort(t, srv2.URL, false))
|
||
defer r2.Close()
|
||
r2.Log = slog.New(slog.NewTextHandler(&quiet, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||
if _, err := r2.TranslateBook(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if strings.Contains(quiet.String(), "priced by a model that did not answer") {
|
||
t.Fatalf("a call priced by its own answerer must say nothing:\n%s", quiet.String())
|
||
}
|
||
}
|
||
|
||
// warnLine returns the first logged line containing needle, so an assertion about that line's FIELDS
|
||
// cannot be satisfied by attributes of some other message in the stream.
|
||
func warnLine(t *testing.T, out, needle string) string {
|
||
t.Helper()
|
||
for _, l := range strings.Split(out, "\n") {
|
||
if strings.Contains(l, needle) {
|
||
return l
|
||
}
|
||
}
|
||
t.Fatalf("no logged line contains %q — the message never sounded:\n%s", needle, out)
|
||
return ""
|
||
}
|