813 lines
27 KiB
Go
813 lines
27 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/obs"
|
||
|
||
_ "modernc.org/sqlite"
|
||
)
|
||
|
||
// e2e-тесты книжного раннера Вехи 2: temp-проект против httptest-провайдера.
|
||
// Деньги, чекпоинты, resume, disposition и exit-коды проверяются через реальные
|
||
// конфиги/store — то, что демонстрирует приёмка, только на моке.
|
||
|
||
func writeFile(t *testing.T, path, content string) {
|
||
t.Helper()
|
||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// reqRec records every provider request body (thread-safe for -race).
|
||
type reqRec struct {
|
||
mu sync.Mutex
|
||
n int
|
||
bodies []string
|
||
}
|
||
|
||
func (r *reqRec) record(body string) {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
r.n++
|
||
r.bodies = append(r.bodies, body)
|
||
}
|
||
func (r *reqRec) count() int { r.mu.Lock(); defer r.mu.Unlock(); return r.n }
|
||
func (r *reqRec) all() []string {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
return append([]string(nil), r.bodies...)
|
||
}
|
||
|
||
// newJSONProvider serves an OpenAI-compatible completion whose text+finish come
|
||
// from respond(body); usage is fixed. The default draftEdit distinguishes the
|
||
// editor by the presence of the editor-prompt marker in the body.
|
||
func newJSONProvider(rec *reqRec, respond func(body string) (text, finish string)) *httptest.Server {
|
||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
rec.record(string(body))
|
||
text, finish := respond(string(body))
|
||
if finish == "" {
|
||
finish = "stop"
|
||
}
|
||
tb, _ := json.Marshal(text)
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":%q}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`,
|
||
tb, finish)
|
||
}))
|
||
}
|
||
|
||
func isEditBody(body string) bool {
|
||
return strings.Contains(body, "Черновик перевода для редактуры")
|
||
}
|
||
|
||
func draftEdit(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
}
|
||
return "ЧЕРНОВИК ПЕРЕВОДА", "stop"
|
||
}
|
||
|
||
func maxTokensOf(t *testing.T, body string) int {
|
||
t.Helper()
|
||
var m map[string]any
|
||
if err := json.Unmarshal([]byte(body), &m); err != nil {
|
||
t.Fatalf("bad request body: %v", err)
|
||
}
|
||
v, ok := m["max_tokens"].(float64)
|
||
if !ok {
|
||
t.Fatalf("no numeric max_tokens in body: %s", body)
|
||
}
|
||
return int(v)
|
||
}
|
||
|
||
// перстоимость fake-model на вызов: (1000-200)·1 + 200·0.1 + 500·2 за 1M.
|
||
const fakeCallUSD = (800*1.0 + 200*0.1 + 500*2.0) / 1e6
|
||
|
||
type projectOpts struct {
|
||
source string
|
||
regenerate int
|
||
minMaxTokens int
|
||
bookUSD float64
|
||
}
|
||
|
||
func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
|
||
t.Helper()
|
||
if o.source == "" {
|
||
o.source = "静かな図書館の朝。"
|
||
}
|
||
if o.minMaxTokens == 0 {
|
||
o.minMaxTokens = 512
|
||
}
|
||
if o.bookUSD == 0 {
|
||
o.bookUSD = 1.0
|
||
}
|
||
dir := t.TempDir()
|
||
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
|
||
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
|
||
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
||
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: %d }
|
||
retries: { regenerate_before_escalate: %d }
|
||
stages:
|
||
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
||
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||
`, o.minMaxTokens, o.regenerate))
|
||
|
||
writeFile(t, filepath.Join(dir, "source.txt"), o.source)
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(`
|
||
book_id: test-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.txt
|
||
ceilings: { book_usd: %g, day_usd: 2.0 }
|
||
`, o.bookUSD))
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
func setupProject(t *testing.T, providerURL string) string {
|
||
return setupProjectOpts(t, providerURL, projectOpts{regenerate: 1})
|
||
}
|
||
|
||
func newRunner(t *testing.T, bookPath string) *Runner {
|
||
t.Helper()
|
||
r, err := NewRunner(bookPath, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return r
|
||
}
|
||
|
||
// --- existing Фаза-0/Веха-1 guarantees, ported to the book runner -----------
|
||
|
||
func TestRunnerEndToEndWithResume(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("expected 2 provider calls, got %d", rec.count())
|
||
}
|
||
if len(res1.Chunks) != 1 || res1.Flagged != 0 || res1.ExitCode() != 0 {
|
||
t.Fatalf("run1 = %+v", res1)
|
||
}
|
||
ch := res1.Chunks[0]
|
||
if ch.Disposition != DispOK || ch.FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||
t.Fatalf("chunk = %+v", ch)
|
||
}
|
||
if diff := res1.TotalUSD - 2*fakeCallUSD; diff > 1e-12 || diff < -1e-12 {
|
||
t.Fatalf("total = %v, want %v", res1.TotalUSD, 2*fakeCallUSD)
|
||
}
|
||
committed, reserved, err := r1.Store.SpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if reserved != 0 || committed != res1.TotalUSD {
|
||
t.Fatalf("ledger: committed=%v reserved=%v want committed=%v", committed, reserved, res1.TotalUSD)
|
||
}
|
||
r1.Close()
|
||
|
||
// Прогон 2 (новый процесс): обе стадии из чекпоинтов/chunk_status за $0.
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("resume must not call the provider again, calls=%d", rec.count())
|
||
}
|
||
if res2.TotalUSD != 0 {
|
||
t.Fatalf("resume must be $0, got %v", res2.TotalUSD)
|
||
}
|
||
for _, st := range res2.Chunks[0].Stages {
|
||
if !st.FromResume || st.CostUSD != 0 {
|
||
t.Fatalf("stage %s must be served from resume at $0: %+v", st.Stage, st)
|
||
}
|
||
}
|
||
if res2.Chunks[0].FinalText != res1.Chunks[0].FinalText {
|
||
t.Fatalf("resume text differs: %q vs %q", res2.Chunks[0].FinalText, res1.Chunks[0].FinalText)
|
||
}
|
||
committed2, _, _ := r2.Store.SpentUSD("test-book")
|
||
if committed2 != committed {
|
||
t.Fatalf("resume must not add spend: %v -> %v", committed, committed2)
|
||
}
|
||
}
|
||
|
||
func TestRunnerSnapshotPinning(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
if rec.count() != 2 {
|
||
t.Fatalf("run1 calls = %d", rec.count())
|
||
}
|
||
|
||
promptPath := filepath.Join(filepath.Dir(bookPath), "prompts", "translator.md")
|
||
writeFile(t, promptPath, "НОВЫЙ промпт с {{source_lang}}.\n---USER---\n{{text}}")
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
_, err := r2.TranslateBook(ctx)
|
||
r2.Close()
|
||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||
t.Fatalf("changed config must fail loud mentioning --resnapshot, got: %v", err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("denied resume must not call the provider, calls=%d", rec.count())
|
||
}
|
||
|
||
r3 := newRunner(t, bookPath)
|
||
defer r3.Close()
|
||
r3.Resnapshot = true
|
||
res, err := r3.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 4 {
|
||
t.Fatalf("resnapshot run must re-call both stages, calls=%d", rec.count())
|
||
}
|
||
if res.TotalUSD <= 0 {
|
||
t.Fatal("re-translation must be billed")
|
||
}
|
||
}
|
||
|
||
// D5.2/B: правка ТОЛЬКО каппы модели двигает snapshot и роняет resume громко.
|
||
func TestRunnerSnapshotPinsCapability(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
|
||
modelsPath := filepath.Join(filepath.Dir(bookPath), "models.yaml")
|
||
raw, err := os.ReadFile(modelsPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
patched := strings.Replace(string(raw),
|
||
"output_per_m: 2.0 }",
|
||
"output_per_m: 2.0 }\n capabilities: { temperature: { mode: force, value: 0.9 } }", 1)
|
||
if patched == string(raw) {
|
||
t.Fatal("failed to inject capability into models.yaml")
|
||
}
|
||
writeFile(t, modelsPath, patched)
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
_, err = r2.TranslateBook(ctx)
|
||
r2.Close()
|
||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||
t.Fatalf("a capability edit must fail loud mentioning --resnapshot, got: %v", err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("denied resume must not call the provider, calls=%d", rec.count())
|
||
}
|
||
}
|
||
|
||
func TestRunnerZeroUsagePaidSettlesEstimate(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
fmt.Fprint(w, `{"id":"z","model":"deepseek-v4-flash","choices":[{"message":{"content":"перевод"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":0,"completion_tokens":0}}`)
|
||
}))
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.TotalUSD <= 0 {
|
||
t.Fatalf("zero-usage paid 2xx must settle a non-zero estimate, got $%.6f", res.TotalUSD)
|
||
}
|
||
committed, _, _ := r.Store.SpentUSD("test-book")
|
||
if committed != res.TotalUSD {
|
||
t.Fatalf("committed %v != total %v", committed, res.TotalUSD)
|
||
}
|
||
}
|
||
|
||
func TestRunnerRejectsUnrunnableConfig(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, pipePath, string(raw)+"\nfanout: { candidates: 3 }\n")
|
||
|
||
if _, err := NewRunner(bookPath, obs.NewLogger()); err == nil {
|
||
t.Fatal("fanout.candidates>1 must be rejected by NewRunner (Phase-2 mechanics)")
|
||
}
|
||
if rec.count() != 0 {
|
||
t.Fatalf("rejected config must not reach the provider, calls=%d", rec.count())
|
||
}
|
||
}
|
||
|
||
func TestRunnerCeilingDenies(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{bookUSD: 0.0000001})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
_, err := r.TranslateBook(context.Background())
|
||
if err == nil {
|
||
t.Fatal("ceiling must deny the run")
|
||
}
|
||
if rec.count() != 0 {
|
||
t.Fatalf("denied reserve must not reach the provider, calls=%d", rec.count())
|
||
}
|
||
}
|
||
|
||
// --- Веха 2: chunk loop, disposition, resume, exit codes --------------------
|
||
|
||
// Multi-chunk (two chapters via form feed) end-to-end + resume at $0.
|
||
func TestRunnerMultiChunkResume(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ГЛАВАОДИН\fГЛАВАДВА"})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
if len(res1.Chunks) != 2 {
|
||
t.Fatalf("want 2 chunks (2 chapters), got %d", len(res1.Chunks))
|
||
}
|
||
if res1.Chunks[0].Chapter != 1 || res1.Chunks[1].Chapter != 2 {
|
||
t.Fatalf("chapters = %d,%d", res1.Chunks[0].Chapter, res1.Chunks[1].Chapter)
|
||
}
|
||
if rec.count() != 4 { // 2 chunks × (draft+edit)
|
||
t.Fatalf("want 4 calls, got %d", rec.count())
|
||
}
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 4 {
|
||
t.Fatalf("resume must not re-call, calls=%d", rec.count())
|
||
}
|
||
if res2.TotalUSD != 0 || res2.Flagged != 0 || res2.ExitCode() != 0 {
|
||
t.Fatalf("resume run = %+v", res2)
|
||
}
|
||
}
|
||
|
||
// D2 core: a flagged chunk skips its remaining stages and the loop continues to
|
||
// the next chunk; the run completes with exit code 2, and resume does not
|
||
// re-attack the flagged chunk.
|
||
func TestRunnerFlagSkipContinueExit2(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) {
|
||
return "Извините, я не могу перевести это.", "stop" // soft refusal
|
||
}
|
||
return draftEdit(body)
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА"})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("a flagged chunk must NOT be an error (loop continues), got %v", err)
|
||
}
|
||
// draft(ch1)=refusal → edit(ch1) skipped (no call); draft(ch2)+edit(ch2) ok = 3 calls.
|
||
if rec.count() != 3 {
|
||
t.Fatalf("want 3 calls (edit of flagged chunk skipped), got %d", rec.count())
|
||
}
|
||
if res1.Flagged != 1 || res1.ExitCode() != 2 {
|
||
t.Fatalf("want 1 flagged chunk / exit 2, got flagged=%d exit=%d", res1.Flagged, res1.ExitCode())
|
||
}
|
||
c1 := res1.Chunks[0]
|
||
if c1.Disposition != DispFlagged || c1.FlagReason != FlagSoftRefusal || c1.FinalText != "" {
|
||
t.Fatalf("chunk1 = %+v (want flagged/soft_refusal/no text)", c1)
|
||
}
|
||
if c1.Stages[1].Disposition != DispSkipped {
|
||
t.Fatalf("edit of a flagged chunk must be skipped, got %+v", c1.Stages[1])
|
||
}
|
||
if res1.Chunks[1].Disposition != DispOK {
|
||
t.Fatalf("chunk2 must be ok, got %+v", res1.Chunks[1])
|
||
}
|
||
// chunk_status persisted the flag and the skip.
|
||
cs, _ := r1.Store.GetChunkStatus("test-book", 1, 0, "draft")
|
||
if cs == nil || cs.Disposition != "flagged" || cs.FlagReason != "soft_refusal" {
|
||
t.Fatalf("draft chunk_status = %+v", cs)
|
||
}
|
||
skip, _ := r1.Store.GetChunkStatus("test-book", 1, 0, "edit")
|
||
if skip == nil || skip.Disposition != "skipped" {
|
||
t.Fatalf("edit chunk_status = %+v", skip)
|
||
}
|
||
r1.Close()
|
||
|
||
// Resume: the flagged chunk is resolved from chunk_status, NOT re-attacked.
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 3 {
|
||
t.Fatalf("resume must not re-attack a flagged chunk, calls=%d", rec.count())
|
||
}
|
||
if res2.Flagged != 1 || res2.TotalUSD != 0 || res2.ExitCode() != 2 {
|
||
t.Fatalf("resume run = %+v", res2)
|
||
}
|
||
}
|
||
|
||
// F4: a non-empty truncated (finish=length) draft must be FLAGGED, never passed
|
||
// downstream to the editor as OK. With no regenerations it flags immediately.
|
||
func TestRunnerF4TruncatedLengthNotPassedToEdit(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if !isEditBody(body) {
|
||
return "Обрыв на середине предложения", "length" // truncated, non-empty, not a loop
|
||
}
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("a truncated draft must flag, not crash: %v", err)
|
||
}
|
||
if rec.count() != 1 {
|
||
t.Fatalf("edit must NOT run on a flagged draft, calls=%d", rec.count())
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagLength {
|
||
t.Fatalf("chunk must be flagged length, got %+v", ch)
|
||
}
|
||
if ch.Stages[0].Disposition != DispFlagged || ch.Stages[1].Disposition != DispSkipped {
|
||
t.Fatalf("draft flagged + edit skipped expected, got %+v", ch.Stages)
|
||
}
|
||
}
|
||
|
||
// D2.3 + F3 + attempt-axis: a length cut is retried with a DOUBLED budget; the
|
||
// second attempt succeeds, the editor then runs, and cost sums both attempts.
|
||
func TestRunnerLengthRetrySucceedsDoublesBudget(t *testing.T) {
|
||
rec := &reqRec{}
|
||
src := strings.Repeat("字", 500) // EstimateTokens=500 → base max_tokens=1000
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
}
|
||
if maxTokensOf(t, body) <= 1000 { // attempt 0 (base)
|
||
return "Обрыв на середине", "length"
|
||
}
|
||
return "ПОЛНЫЙ ЧЕРНОВИК", "stop" // attempt 1 (doubled budget)
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: src, regenerate: 1, minMaxTokens: 128})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispOK || ch.FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||
t.Fatalf("retry should succeed and edit should run, got %+v", ch)
|
||
}
|
||
draft := ch.Stages[0]
|
||
if draft.Attempts != 2 {
|
||
t.Fatalf("draft must have taken 2 attempts, got %d", draft.Attempts)
|
||
}
|
||
// F3: cost sums BOTH attempts.
|
||
if diff := draft.CumCostUSD - 2*fakeCallUSD; diff > 1e-12 || diff < -1e-12 {
|
||
t.Fatalf("draft cum cost = %v, want 2×call %v (attempts summed)", draft.CumCostUSD, 2*fakeCallUSD)
|
||
}
|
||
// The two draft attempts doubled the budget: 1000 then 2000.
|
||
var draftToks []int
|
||
for _, b := range rec.all() {
|
||
if !isEditBody(b) {
|
||
draftToks = append(draftToks, maxTokensOf(t, b))
|
||
}
|
||
}
|
||
if len(draftToks) != 2 || draftToks[0] != 1000 || draftToks[1] != 2000 {
|
||
t.Fatalf("attempt budgets = %v, want [1000 2000]", draftToks)
|
||
}
|
||
}
|
||
|
||
// D2.5: the editor's max_tokens is sized from the (long Russian) DRAFT, not the
|
||
// (short CJK) source — otherwise it under-budgets and false-triggers a length
|
||
// retry. Broken D2.5 would make the two budgets equal.
|
||
func TestRunnerEditSizedFromDraft(t *testing.T) {
|
||
rec := &reqRec{}
|
||
longDraft := strings.Repeat("Утро было тихим и ясным над рекой. ", 120) // long Russian output
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОК", "stop"
|
||
}
|
||
return longDraft, "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: strings.Repeat("字", 100), minMaxTokens: 128})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var draftTok, editTok int
|
||
for _, b := range rec.all() {
|
||
if isEditBody(b) {
|
||
editTok = maxTokensOf(t, b)
|
||
} else {
|
||
draftTok = maxTokensOf(t, b)
|
||
}
|
||
}
|
||
if editTok <= draftTok {
|
||
t.Fatalf("edit max_tokens (%d) must exceed draft's (%d) — sized from the longer draft (D2.5)", editTok, draftTok)
|
||
}
|
||
}
|
||
|
||
// Anti-wedge #2: an empty completion is FLAGGED and the loop continues — not a
|
||
// crash (the Фаза-0 behaviour was to error out the whole run).
|
||
func TestRunnerEmptyFlaggedNotCrash(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if !isEditBody(body) {
|
||
return "", "stop" // empty draft
|
||
}
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatalf("empty completion must flag, not crash: %v", err)
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagEmpty {
|
||
t.Fatalf("empty draft must be flagged empty, got %+v", ch)
|
||
}
|
||
if res.ExitCode() != 2 {
|
||
t.Fatalf("exit code must be 2, got %d", res.ExitCode())
|
||
}
|
||
// Money for the paid-but-empty call is still accounted.
|
||
committed, _, _ := r.Store.SpentUSD("test-book")
|
||
if committed <= 0 {
|
||
t.Fatalf("the paid empty call must still be billed, committed=%v", committed)
|
||
}
|
||
}
|
||
|
||
// Anti-wedge #2: a billed 2xx with an unreadable body → flagged decode_error
|
||
// with the estimate settled, loop continues (not a crash).
|
||
func TestRunnerDecodeErrorFlagged(t *testing.T) {
|
||
var draftCalls int
|
||
var mu sync.Mutex
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
if !isEditBody(string(body)) {
|
||
mu.Lock()
|
||
draftCalls++
|
||
mu.Unlock()
|
||
fmt.Fprint(w, `NOT JSON AT ALL`) // 200 with a garbage body → BilledDecodeError
|
||
return
|
||
}
|
||
fmt.Fprint(w, `{"id":"e","model":"fake-model","choices":[{"message":{"content":"ред"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
||
}))
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatalf("a billed decode failure must flag, not crash: %v", err)
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagDecodeError {
|
||
t.Fatalf("draft must be flagged decode_error, got %+v", ch)
|
||
}
|
||
if ch.Stages[1].Disposition != DispSkipped {
|
||
t.Fatalf("edit must be skipped after a decode flag, got %+v", ch.Stages[1])
|
||
}
|
||
if res.TotalUSD <= 0 {
|
||
t.Fatal("the conservative estimate must be settled for a billed decode failure")
|
||
}
|
||
}
|
||
|
||
// Determinism content-guard (self-review Веха 2): the source is NOT in the
|
||
// snapshot, so editing a chunk's source between runs must RE-TRANSLATE that
|
||
// position (content-addressed, §3.4), never serve the stale positional
|
||
// chunk_status row. Without the content_hash guard the fast-path would return
|
||
// the OLD translation at $0.
|
||
func TestRunnerSourceEditReTranslates(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "СТАРЫЙИСХОДНИК"})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count()
|
||
|
||
// Edit the source in place (same position, still one chunk). No brief field
|
||
// and not the chunker changed, so the snapshot is unchanged — the ONLY thing
|
||
// that must invalidate resume is the content itself.
|
||
writeFile(t, filepath.Join(filepath.Dir(bookPath), "source.txt"), "НОВЫЙИСХОДНИК")
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Both stages re-render with the new content (the editor sees {{text}} too),
|
||
// miss their old checkpoints, and are re-translated and re-billed.
|
||
if rec.count() != callsAfterRun1+2 {
|
||
t.Fatalf("an edited source must re-translate the chunk, not serve stale: calls %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
if res.TotalUSD <= 0 {
|
||
t.Fatalf("re-translation of an edited source must be billed, got $%.6f", res.TotalUSD)
|
||
}
|
||
}
|
||
|
||
// Anti-wedge #1: a terminally-flagged chunk is resolved from chunk_status BEFORE
|
||
// rendering, so it is NOT re-attacked even when the retry budget is later raised
|
||
// (RegenerateBeforeEscalate is not in the snapshot). Without the chunk_status
|
||
// fast-path the attempt loop would spawn NEW paid attempts up to the new cap.
|
||
func TestRunnerFlaggedNotReattackedWhenRegenRaised(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if !isEditBody(body) {
|
||
return "Обрыв на середине", "length" // always a length cut
|
||
}
|
||
return "ред", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count() // regen=0 → exactly 1 draft attempt, flagged length
|
||
|
||
// Raise the regeneration budget and re-run. The flagged chunk must stay put.
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, pipePath, strings.Replace(string(raw), "regenerate_before_escalate: 0", "regenerate_before_escalate: 2", 1))
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatalf("a terminally-flagged chunk must NOT be re-attacked on a raised budget: calls %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
if res.Chunks[0].Disposition != DispFlagged || res.Chunks[0].FlagReason != FlagLength {
|
||
t.Fatalf("chunk must stay flagged length, got %+v", res.Chunks[0])
|
||
}
|
||
}
|
||
|
||
// Anti-wedge #3: a checkpoint present without its chunk_status row (kill -9 lost
|
||
// the resolve) self-heals — classify the checkpoint text, no re-billing, no
|
||
// crash. Proven by deleting the chunk_status row and re-running with regen=0 so
|
||
// the empty flag cannot spawn a new paid attempt.
|
||
func TestRunnerLegacyEmptyCheckpointSelfHeal(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if !isEditBody(body) {
|
||
return "", "stop" // empty draft → empty checkpoint
|
||
}
|
||
return "ред", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
dbPath := r1.Book.ProjectDB
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count()
|
||
|
||
// Simulate the resolve being lost while the (empty) checkpoint survives.
|
||
db, err := sql.Open("sqlite", "file:"+dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := db.Exec(`DELETE FROM chunk_status WHERE stage = 'draft'`); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
db.Close()
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("legacy empty checkpoint must self-heal, not crash: %v", err)
|
||
}
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatalf("self-heal must NOT re-bill the empty attempt: calls %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagEmpty {
|
||
t.Fatalf("self-heal must re-derive the empty flag, got %+v", ch)
|
||
}
|
||
// chunk_status was rebuilt.
|
||
cs, _ := r2.Store.GetChunkStatus("test-book", 1, 0, "draft")
|
||
if cs == nil || cs.Disposition != "flagged" {
|
||
t.Fatalf("chunk_status must be rebuilt as flagged, got %+v", cs)
|
||
}
|
||
}
|