372 lines
17 KiB
Go
372 lines
17 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// echoregen_test.go: the OPT-IN echo re-generation — `retries.regenerate_echo_before_escalate`
|
||
// (config.Retries.RegenerateEchoBeforeEscalate), read at stagerun.go's `echoRegen`.
|
||
//
|
||
// WHY THIS FILE EXISTS AT ALL. The knob shipped with D39.64 and, until this file, had ZERO tests:
|
||
// `grep -rn RegenerateEchoBeforeEscalate --include=*_test.go` returned nothing. It is a knob on the
|
||
// money path — it decides whether an echo costs a same-model re-generation or a hop to the expensive
|
||
// model — and nothing held it. Turning it on in the shipping configs without a test would have been
|
||
// changing a paid path on the strength of a code read.
|
||
//
|
||
// The two tests are the two halves a default change needs: the knob does what it claims (the
|
||
// re-generation replaces the hop), and turning it on moves NO snapshot, so nothing already bought is
|
||
// re-bought by the change.
|
||
|
||
// echoOnFirstDraft answers an OpenAI-compatible completion as a provider whose echo is STOCHASTIC PER
|
||
// CALL (D39.61): the first PRIMARY draft call returns the CJK source verbatim — an echo, which classify
|
||
// reads as cjk_artifact — and every later primary call translates cleanly. That is the measured shape
|
||
// the knob exists for: on such a provider a re-generation recovers the chunk without the hop, and the
|
||
// premise the knob inverts ("a same-model retry just re-produces the echo", disposition.go) is false.
|
||
type echoOnFirstDraft struct {
|
||
mu sync.Mutex
|
||
primaryDrafts int
|
||
}
|
||
|
||
func (e *echoOnFirstDraft) respond(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
}
|
||
if strings.Contains(body, "fake-fallback") {
|
||
return "Тихое утро в библиотеке (дорогая модель).", "stop"
|
||
}
|
||
e.mu.Lock()
|
||
defer e.mu.Unlock()
|
||
e.primaryDrafts++
|
||
if e.primaryDrafts == 1 {
|
||
return "静かな図書館の朝。", "stop" // the echo
|
||
}
|
||
return "Тихое утро в библиотеке.", "stop"
|
||
}
|
||
|
||
func (e *echoOnFirstDraft) count() int { e.mu.Lock(); defer e.mu.Unlock(); return e.primaryDrafts }
|
||
|
||
// newPricedProvider is newJSONProvider with ONE difference, and the difference is the point: it answers
|
||
// with the model the REQUEST asked for instead of the constant "fake-model".
|
||
//
|
||
// Money is priced off the model that actually ANSWERED (internal/ledger.PriceForResponse, «цена — по
|
||
// фактически ответившей модели», backend/README.md invariant 1), so a fake that always names one model
|
||
// prices an escalation hop at the primary's rate. On the shared fixture that is harmless; here it would
|
||
// silently erase the only thing this file measures — the price difference between a hop and a
|
||
// re-generation — and the cost assertion below would pass on two equal numbers. Found by writing the
|
||
// assertion and watching it report regen=escalate=0.00546000 on identical prices.
|
||
func newPricedProvider(rec *reqRec, respond func(body string) (text, finish string)) *httptest.Server {
|
||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
rec.record(string(body))
|
||
var req struct {
|
||
Model string `json:"model"`
|
||
}
|
||
_ = json.Unmarshal(body, &req)
|
||
if req.Model == "" {
|
||
req.Model = "fake-model"
|
||
}
|
||
text, finish := respond(string(body))
|
||
if finish == "" {
|
||
finish = "stop"
|
||
}
|
||
tb, _ := json.Marshal(text)
|
||
mb, _ := json.Marshal(req.Model)
|
||
fmt.Fprintf(w, `{"id":"fake","model":%s,"choices":[{"message":{"content":%s},"finish_reason":%q}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`,
|
||
mb, tb, finish)
|
||
}))
|
||
}
|
||
|
||
// setupEchoRegen writes a ONE-chunk book whose draft escalates to `fake-fallback`, with the echo
|
||
// re-generation budget set to echoRegen.
|
||
//
|
||
// The fallback is priced TEN TIMES the primary on output. That is not decoration: the whole claim of
|
||
// the knob is economic — an escalation hop is the expensive answer to an echo and a re-generation is
|
||
// the cheap one — and a fixture where both models cost the same would pin the ROUTING while saying
|
||
// nothing about the money, which is the half the shipping default is being changed for.
|
||
func setupEchoRegen(t *testing.T, providerURL string, echoRegen int) string {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
|
||
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
fake-fallback:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 20.0 }
|
||
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||
retries: { regenerate_before_escalate: 0, regenerate_echo_before_escalate: %d }
|
||
stages:
|
||
- { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off", escalate_to: fake-fallback }
|
||
- { name: edit, role: editor, model: fake-model, prompt_override: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||
escalation: { budget_usd: 5.0 }
|
||
`, echoRegen))
|
||
writeFile(t, filepath.Join(dir, "source.txt"), "静かな図書館の朝。")
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), `
|
||
book_id: test-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.txt
|
||
ceilings: { book_usd: 5.0, day_usd: 10.0 }
|
||
`)
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
// TestEchoRegenReplacesTheHop is the knob's whole claim, run twice on the SAME provider: with the
|
||
// budget at 0 an echo goes straight to the expensive hop; with it at 1 the same echo is re-generated
|
||
// on the primary and the hop never happens. Both runs ship a clean chunk — the knob changes WHAT WAS
|
||
// BOUGHT, not what the reader gets, which is exactly why it can be defaulted on evidence.
|
||
//
|
||
// Mutation this catches: delete the `att.cls.Reason == FlagCJKArtifact && attempt < echoRegen` branch
|
||
// in stagerun.go and the echoRegen=1 arm escalates → its hop count becomes 1 and the assertion on
|
||
// "no fallback call" goes RED. Weakening it to `attempt <= echoRegen` doubles the primary calls → also RED.
|
||
func TestEchoRegenReplacesTheHop(t *testing.T) {
|
||
type arm struct {
|
||
echoRegen int
|
||
wantPrimary int // fresh calls on fake-model's DRAFT stage
|
||
wantEscalated bool // did the unit ride the hop
|
||
}
|
||
arms := []arm{
|
||
{echoRegen: 0, wantPrimary: 1, wantEscalated: true}, // the shipped-until-now behaviour
|
||
{echoRegen: 1, wantPrimary: 2, wantEscalated: false}, // the behaviour the default change buys
|
||
}
|
||
var spend [2]float64
|
||
for i, a := range arms {
|
||
t.Run(fmt.Sprintf("echo_regen=%d", a.echoRegen), func(t *testing.T) {
|
||
prov := &echoOnFirstDraft{}
|
||
rec := &reqRec{}
|
||
srv := newPricedProvider(rec, prov.respond)
|
||
defer srv.Close()
|
||
bookPath := setupEchoRegen(t, srv.URL, a.echoRegen)
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(res.Chunks) != 1 {
|
||
t.Fatalf("want 1 chunk, got %d", len(res.Chunks))
|
||
}
|
||
oc := res.Chunks[0]
|
||
if oc.Disposition != DispOK {
|
||
t.Fatalf("both arms must SHIP the chunk — the knob changes the route, not the result; got %s/%s",
|
||
oc.Disposition, oc.FlagReason)
|
||
}
|
||
if got := prov.count(); got != a.wantPrimary {
|
||
t.Fatalf("primary draft calls = %d, want %d (echo_regen=%d)", got, a.wantPrimary, a.echoRegen)
|
||
}
|
||
if got := oc.Stages[0].Escalated; got != a.wantEscalated {
|
||
t.Fatalf("draft escalated = %t, want %t (echo_regen=%d) — with a re-generation budget the "+
|
||
"echo must be recovered on the SAME model and the hop must never fire", got, a.wantEscalated, a.echoRegen)
|
||
}
|
||
committed, _, err := r.Store.SpentUSD(r.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
spend[i] = committed
|
||
})
|
||
}
|
||
// The economic half, structural rather than incidental: the fallback costs 10× on output, so the
|
||
// re-generating arm must come out CHEAPER. Asserted as a strict inequality, not as a ratio — the
|
||
// ratio is a property of this fixture's price table, the DIRECTION is the property of the knob.
|
||
if !(spend[1] < spend[0]) {
|
||
t.Fatalf("the re-generating arm must cost LESS than the escalating one, got regen=%.8f escalate=%.8f",
|
||
spend[1], spend[0])
|
||
}
|
||
}
|
||
|
||
// TestEchoRegenBudgetMovesNoSnapshot is the machine gate the default change is allowed by, and it is
|
||
// permanent rather than a one-off measurement: the re-generation budgets are deliberately NOT folded
|
||
// into buildSnapshotID (backend/README.md, «`Retries` НЕ фолдится в снапшот … сознательно так же для
|
||
// ручки `RegenerateEchoBeforeEscalate`, D39.64»), so turning the knob on cannot invalidate a single
|
||
// paid checkpoint. Grepping snapshot.go for `Retries` proves only that a NAME is absent; this renders
|
||
// the ID on a fixture with the knob off and on and compares the bytes.
|
||
//
|
||
// ⛔ IT MEASURES THE CONFIG PATH FIRST, and that ordering is not cosmetic. An earlier version of this
|
||
// test only poked `r.Pipeline.Retries` on a live runner, which pins the STRUCT FIELD — and this codebase
|
||
// computes most snapshot inputs at LOAD (PromptSHA256, LangpackVersion, EmbeddedVersion, MemoryVersion
|
||
// are all resolved before a Runner exists). A fold derived at load from the same YAML key would sail
|
||
// straight through an in-memory poke while two BOOKS differing only in that key rendered different ids,
|
||
// i.e. the «--resnapshot re-buys every paid checkpoint» catastrophe shipping green. Demonstrated by an
|
||
// adversarial pass over this pack, which built exactly that fold and watched the old gate stay green.
|
||
// So the load path is measured on two real fixtures, and the in-memory arm is kept only as the cheap
|
||
// second axis it always was.
|
||
//
|
||
// ⚠ WHAT THIS PINS IS A TRADE, NOT A FREE LUNCH, and the trade is named so a later reader does not
|
||
// read the green as "nothing depends on this". Because the budget is outside the snapshot, two runs
|
||
// that made a DIFFERENT NUMBER of re-generations share one snapshot id and are externally
|
||
// indistinguishable by it. That is sound for MONEY — every attempt carries its own request_hash
|
||
// (attempt is in it), so a resume replays exactly what was paid for and nothing is re-bought — and it
|
||
// is a real gap for COMPARABILITY, whose carrier belongs in the report (the attempt>0 slice of the
|
||
// paid tail), never in the snapshot. Folding it would make every existing book re-payable to buy a
|
||
// number that is already durable in `checkpoints.attempt`.
|
||
//
|
||
// Mutation this catches: fold the KNOB'S VALUE into the payload — `EchoRegen int` set from
|
||
// r.Pipeline.Retries.RegenerateEchoBeforeEscalate, or a string derived from it at load. ⚠ NOT "add any
|
||
// field": a CONSTANT added to the payload shifts both ids identically and this test stays green, which is
|
||
// correct — the question it asks is whether the KNOB reaches the id, not whether the payload changed.
|
||
func TestEchoRegenBudgetMovesNoSnapshot(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
|
||
// --- ARM 1, THE LOAD PATH: two BOOKS whose pipeline.yaml differs in this key and nothing else. ---
|
||
// This is the measurement the pack was ordered to show: the snapshot id before and after the diff the
|
||
// session actually shipped, which is a change to YAML files and not to a struct field.
|
||
renderFor := func(t *testing.T, echoRegen int) (string, string) {
|
||
t.Helper()
|
||
rr := newRunner(t, setupEchoRegen(t, srv.URL, echoRegen))
|
||
defer rr.Close()
|
||
if got := rr.Pipeline.Retries.RegenerateEchoBeforeEscalate; got != echoRegen {
|
||
t.Fatalf("the fixture's YAML did not reach the loaded config: want %d, got %d", echoRegen, got)
|
||
}
|
||
d, _, err := rr.snapshotIDForWave(waveDraft)
|
||
if err != nil {
|
||
t.Fatalf("render the draft snapshot at echo_regen=%d: %v", echoRegen, err)
|
||
}
|
||
e, _, err := rr.snapshotIDForWave(waveEdit)
|
||
if err != nil {
|
||
t.Fatalf("render the edit snapshot at echo_regen=%d: %v", echoRegen, err)
|
||
}
|
||
return d, e
|
||
}
|
||
dOff, eOff := renderFor(t, 0)
|
||
dOn, eOn := renderFor(t, 1)
|
||
if dOn != dOff || eOn != eOff {
|
||
t.Fatalf("the shipped config change moved a wave snapshot — every paid checkpoint of every book "+
|
||
"would be re-bought (--resnapshot).\n draft %s -> %s\n edit %s -> %s", dOff, dOn, eOff, eOn)
|
||
}
|
||
|
||
// --- ARM 2, THE STRUCT FIELD: cheaper, and it also covers the sibling budget. ---
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "静かな図書館の朝。"})
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
if err := r.seedGlossary(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
read := func(what string) (string, string) {
|
||
t.Helper()
|
||
d, _, err := r.snapshotIDForWave(waveDraft)
|
||
if err != nil {
|
||
t.Fatalf("render the draft snapshot (%s): %v", what, err)
|
||
}
|
||
e, _, err := r.snapshotIDForWave(waveEdit)
|
||
if err != nil {
|
||
t.Fatalf("render the edit snapshot (%s): %v", what, err)
|
||
}
|
||
return d, e
|
||
}
|
||
if r.Pipeline.Retries.RegenerateEchoBeforeEscalate != 0 {
|
||
t.Fatalf("fixture drifted: this book must start with the knob OFF, got %d", r.Pipeline.Retries.RegenerateEchoBeforeEscalate)
|
||
}
|
||
draftOff, editOff := read("knob off")
|
||
|
||
// Both re-generation budgets, because both are outside the snapshot for the same reason and a
|
||
// future fold of EITHER is the thing this gate exists to make loud.
|
||
for _, n := range []int{1, 7} {
|
||
r.Pipeline.Retries.RegenerateEchoBeforeEscalate = n
|
||
r.Pipeline.Retries.RegenerateBeforeEscalate = n
|
||
draftOn, editOn := read(fmt.Sprintf("knob=%d", n))
|
||
if draftOn != draftOff || editOn != editOff {
|
||
t.Fatalf("a re-generation budget of %d moved a wave snapshot — turning it on would re-buy every "+
|
||
"paid checkpoint of every book (--resnapshot).\n draft %s -> %s\n edit %s -> %s",
|
||
n, draftOff, draftOn, editOff, editOn)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestEchoRegenFiresONLYForEcho holds the REASON half of the branch, which the sibling test above does
|
||
// not: it varies the budget while the flag is always an echo, so it cannot tell "re-generate the echo"
|
||
// from "re-generate anything".
|
||
//
|
||
// The distinction is money, not tidiness. `cjk_artifact` is the ONE non-retryable flag a same-model call
|
||
// can plausibly recover, precisely because D39.61 measured this provider's echo stochastic per call.
|
||
// Every other non-retryable reason — hard_refusal, soft_refusal, content_filter, loop_degenerate,
|
||
// decode_error — is deterministic for the model that produced it, and disposition.go says what a retry
|
||
// buys there: «a same-model retry would just re-refuse and re-bill» (D2.2). With the knob now ON in five
|
||
// shipping configs, an implementation that re-generated on ANY flag would buy that re-refusal on every
|
||
// refused chunk of every book, and until this test nothing in the repo went red on it.
|
||
//
|
||
// Mutation this catches: drop the `att.cls.Reason == FlagCJKArtifact` half and keep only
|
||
// `attempt < echoRegen` — the whole battery stays green today; here the primary call count goes 1 → 2 → RED.
|
||
func TestEchoRegenFiresONLYForEcho(t *testing.T) {
|
||
var mu sync.Mutex
|
||
primary := 0
|
||
rec := &reqRec{}
|
||
srv := newPricedProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
}
|
||
if strings.Contains(body, "fake-fallback") {
|
||
return "Тихое утро в библиотеке (дорогая модель).", "stop"
|
||
}
|
||
mu.Lock()
|
||
defer mu.Unlock()
|
||
primary++
|
||
// A provider REFUSAL — deterministic for this model, and therefore exactly the reason a
|
||
// same-model re-generation must NOT be spent on.
|
||
return "", "refusal"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupEchoRegen(t, srv.URL, 1) // the knob is ON — that is the point
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(res.Chunks) != 1 {
|
||
t.Fatalf("want 1 chunk, got %d", len(res.Chunks))
|
||
}
|
||
// The premise: the primary really did flag and the run really did take the ESCALATION route — which is
|
||
// the correct answer for a deterministic refusal, and the route a wrong implementation would replace
|
||
// with a same-model re-generation.
|
||
if !res.Chunks[0].Stages[0].Escalated {
|
||
t.Fatal("fixture drifted: a refused primary must escalate, so the hop is what recovers this chunk")
|
||
}
|
||
mu.Lock()
|
||
got := primary
|
||
mu.Unlock()
|
||
if got != 1 {
|
||
t.Fatalf("the echo re-generation budget must be spent on ECHO and nothing else: a refusal is "+
|
||
"deterministic for this model, so a same-model retry only re-refuses and re-bills (D2.2). "+
|
||
"primary calls = %d, want 1", got)
|
||
}
|
||
}
|