131 lines
5.5 KiB
Go
131 lines
5.5 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"path/filepath"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// escalation_budget_test.go closes the D24.1г acceptance caveat: escalation.budget_usd
|
||
// was only exercised in the ">0 → the hop fires" direction; the REFUSAL by exhaustion was
|
||
// never executed. Here a budget that a single hop overshoots must DENY the next flagged
|
||
// chunk's hop — the flag stays honest, no fresh hop is made, and no escalation money is
|
||
// spent beyond the first hop.
|
||
|
||
// setupTwoChapterEscalation writes a 2-chapter epub book whose draft escalates to
|
||
// fake-fallback under budgetUSD. Two spine chapters → two chunks, both of which echo on
|
||
// the primary (cjk_artifact) so both are escalation candidates.
|
||
func setupTwoChapterEscalation(t *testing.T, providerURL string, budgetUSD float64) string {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
|
||
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
fake-fallback:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||
retries: { regenerate_before_escalate: 0 }
|
||
stages:
|
||
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off", escalate_to: fake-fallback }
|
||
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||
escalation: { budget_usd: %g }
|
||
`, budgetUSD))
|
||
chapters := []epubChapter{
|
||
{id: "c1", href: "ch1.xhtml", body: `<p>静かな図書館の朝。</p>`},
|
||
{id: "c2", href: "ch2.xhtml", body: `<p>静かな図書館の夜。</p>`},
|
||
}
|
||
buildEPUBAt(t, filepath.Join(dir, "source.epub"), chapters, []string{"c1", "c2"})
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), `
|
||
book_id: test-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.epub
|
||
ceilings: { book_usd: 5.0, day_usd: 10.0 }
|
||
`)
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
// A budget that one hop overshoots denies the second flagged chunk's hop. The first chunk
|
||
// escalates and resolves OK; the second stays honestly flagged, unescalated, with NO extra
|
||
// escalation spend. Mutation: make escalationBudgetRemains always admit (drop `spent <
|
||
// budget`) and the second chunk escalates → calls jump to 6 and it flips OK → red.
|
||
func TestRunnerEscalationBudgetExhaustionDeniesLaterHop(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean) // primary echoes (cjk_artifact); fake-fallback cleans
|
||
defer srv.Close()
|
||
// budget 0.001 < one hop's cost (fakeCallUSD ≈ 0.00182): chunk1's hop is admitted
|
||
// (spent 0 < budget) and overshoots to 0.00182; chunk2 then sees spent ≥ budget.
|
||
bookPath := setupTwoChapterEscalation(t, srv.URL, 0.001)
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(res.Chunks) != 2 {
|
||
t.Fatalf("want 2 chunks from 2 chapters, got %d", len(res.Chunks))
|
||
}
|
||
|
||
// Calls: ch1 = primary + hop + edit (3); ch2 = primary only, hop denied, edit skipped (1).
|
||
if rec.count() != 4 {
|
||
t.Fatalf("want 4 provider calls (ch1: primary+hop+edit; ch2: primary only, hop denied), got %d", rec.count())
|
||
}
|
||
|
||
ch1, ch2 := res.Chunks[0], res.Chunks[1]
|
||
// Chapter 1: the first hop is admitted and resolves the chunk OK.
|
||
if ch1.Disposition != DispOK || !ch1.Stages[0].Escalated {
|
||
t.Fatalf("chapter 1 must escalate (budget still open) and resolve OK, got disp=%s escalated=%t", ch1.Disposition, ch1.Stages[0].Escalated)
|
||
}
|
||
// Chapter 2: the exhausted budget denies the hop — the flag stays, nothing escalated.
|
||
if ch2.Disposition != DispFlagged || ch2.FlagReason != FlagCJKArtifact {
|
||
t.Fatalf("chapter 2 must stay honestly flagged when the budget is exhausted, got disp=%s reason=%s", ch2.Disposition, ch2.FlagReason)
|
||
}
|
||
if ch2.Stages[0].Escalated {
|
||
t.Fatalf("a budget-denied hop must NOT be recorded as an escalation, got %+v", ch2.Stages[0])
|
||
}
|
||
if ch2.Stages[1].Disposition != DispSkipped {
|
||
t.Fatalf("chapter 2 edit must be skipped after the primary flag stands, got %+v", ch2.Stages[1])
|
||
}
|
||
if res.Flagged != 1 || res.ExitCode() != 2 {
|
||
t.Fatalf("exactly the second chunk stays flagged, want flagged=1 exit=2, got flagged=%d exit=%d", res.Flagged, res.ExitCode())
|
||
}
|
||
|
||
// Money: escalation spend is EXACTLY the one admitted hop — the denied hop cost nothing.
|
||
esc, err := r.Store.EscalationSpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if diff := esc - fakeCallUSD; diff > 1e-12 || diff < -1e-12 {
|
||
t.Fatalf("escalation spend must be exactly one hop (%.6f), got %.6f — the denied hop must spend $0", fakeCallUSD, esc)
|
||
}
|
||
}
|