1706 lines
61 KiB
Go
1706 lines
61 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/obs"
|
||
"textmachine/backend/internal/store"
|
||
|
||
_ "modernc.org/sqlite"
|
||
)
|
||
|
||
// e2e-тесты книжного раннера Вехи 2: temp-проект против httptest-провайдера.
|
||
// Деньги, чекпоинты, resume, disposition и exit-коды проверяются через реальные
|
||
// конфиги/store — то, что демонстрирует приёмка, только на моке.
|
||
|
||
func writeFile(t *testing.T, path, content string) {
|
||
t.Helper()
|
||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// reqRec records every provider request body (thread-safe for -race).
|
||
type reqRec struct {
|
||
mu sync.Mutex
|
||
n int
|
||
bodies []string
|
||
}
|
||
|
||
func (r *reqRec) record(body string) {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
r.n++
|
||
r.bodies = append(r.bodies, body)
|
||
}
|
||
func (r *reqRec) count() int { r.mu.Lock(); defer r.mu.Unlock(); return r.n }
|
||
func (r *reqRec) all() []string {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
return append([]string(nil), r.bodies...)
|
||
}
|
||
|
||
// newJSONProvider serves an OpenAI-compatible completion whose text+finish come
|
||
// from respond(body); usage is fixed. The default draftEdit distinguishes the
|
||
// editor by the presence of the editor-prompt marker in the body.
|
||
func newJSONProvider(rec *reqRec, respond func(body string) (text, finish string)) *httptest.Server {
|
||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
rec.record(string(body))
|
||
text, finish := respond(string(body))
|
||
if finish == "" {
|
||
finish = "stop"
|
||
}
|
||
tb, _ := json.Marshal(text)
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":%q}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`,
|
||
tb, finish)
|
||
}))
|
||
}
|
||
|
||
func isEditBody(body string) bool {
|
||
return strings.Contains(body, "Черновик перевода для редактуры")
|
||
}
|
||
|
||
func draftEdit(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
}
|
||
return "ЧЕРНОВИК ПЕРЕВОДА", "stop"
|
||
}
|
||
|
||
func maxTokensOf(t *testing.T, body string) int {
|
||
t.Helper()
|
||
var m map[string]any
|
||
if err := json.Unmarshal([]byte(body), &m); err != nil {
|
||
t.Fatalf("bad request body: %v", err)
|
||
}
|
||
v, ok := m["max_tokens"].(float64)
|
||
if !ok {
|
||
t.Fatalf("no numeric max_tokens in body: %s", body)
|
||
}
|
||
return int(v)
|
||
}
|
||
|
||
// перстоимость fake-model на вызов: (1000-200)·1 + 200·0.1 + 500·2 за 1M.
|
||
const fakeCallUSD = (800*1.0 + 200*0.1 + 500*2.0) / 1e6
|
||
|
||
type projectOpts struct {
|
||
source string
|
||
epub []epubChapter // when set, the source is an epub (source.epub) instead of txt
|
||
spine []string // spine order for epub
|
||
regenerate int
|
||
minMaxTokens int
|
||
bookUSD float64
|
||
gatesYAML string // optional gates:/... block appended to pipeline.yaml
|
||
glossarySeed string // optional glossary seed YAML content; "" = no glossary
|
||
postcheckGate bool // enable the memory post-check hard gate (assumes gatesYAML is empty)
|
||
banknote bool // enable the banknote channel gate (WS4; assumes gatesYAML/postcheckGate empty)
|
||
waveWorkers int // wave-executor parallelism (0 → default 1, a deterministic sequential-structured run)
|
||
}
|
||
|
||
func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
|
||
t.Helper()
|
||
if o.source == "" {
|
||
o.source = "静かな図書館の朝。"
|
||
}
|
||
if o.minMaxTokens == 0 {
|
||
o.minMaxTokens = 512
|
||
}
|
||
if o.bookUSD == 0 {
|
||
o.bookUSD = 1.0
|
||
}
|
||
dir := t.TempDir()
|
||
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
|
||
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
|
||
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
||
|
||
gatesBlock := o.gatesYAML
|
||
if o.postcheckGate {
|
||
gatesBlock += "\ngates:\n glossary:\n postcheck_gate: true\n"
|
||
}
|
||
if o.banknote {
|
||
gatesBlock += "\ngates:\n banknote:\n enabled: true\n"
|
||
}
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: %d }
|
||
retries: { regenerate_before_escalate: %d }
|
||
context: { glossary_injection: selective, glossary_token_budget: 800 }
|
||
waves: { workers: %d }
|
||
stages:
|
||
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
||
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||
%s`, o.minMaxTokens, o.regenerate, o.waveWorkers, gatesBlock))
|
||
|
||
sourceName := "source.txt"
|
||
if len(o.epub) > 0 {
|
||
sourceName = "source.epub"
|
||
buildEPUBAt(t, filepath.Join(dir, sourceName), o.epub, o.spine)
|
||
} else {
|
||
writeFile(t, filepath.Join(dir, "source.txt"), o.source)
|
||
}
|
||
glossaryLine := ""
|
||
if o.glossarySeed != "" {
|
||
writeFile(t, filepath.Join(dir, "glossary-seed.yaml"), o.glossarySeed)
|
||
glossaryLine = "glossary_seed: glossary-seed.yaml"
|
||
}
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(`
|
||
book_id: test-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: %s
|
||
%s
|
||
ceilings: { book_usd: %g, day_usd: 2.0 }
|
||
`, sourceName, glossaryLine, o.bookUSD))
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
func setupProject(t *testing.T, providerURL string) string {
|
||
return setupProjectOpts(t, providerURL, projectOpts{regenerate: 1})
|
||
}
|
||
|
||
func newRunner(t *testing.T, bookPath string) *Runner {
|
||
t.Helper()
|
||
r, err := NewRunner(bookPath, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return r
|
||
}
|
||
|
||
// --- existing Фаза-0/Веха-1 guarantees, ported to the book runner -----------
|
||
|
||
func TestRunnerEndToEndWithResume(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("expected 2 provider calls, got %d", rec.count())
|
||
}
|
||
if len(res1.Chunks) != 1 || res1.Flagged != 0 || res1.ExitCode() != 0 {
|
||
t.Fatalf("run1 = %+v", res1)
|
||
}
|
||
ch := res1.Chunks[0]
|
||
if ch.Disposition != DispOK || ch.FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||
t.Fatalf("chunk = %+v", ch)
|
||
}
|
||
if diff := res1.TotalUSD - 2*fakeCallUSD; diff > 1e-12 || diff < -1e-12 {
|
||
t.Fatalf("total = %v, want %v", res1.TotalUSD, 2*fakeCallUSD)
|
||
}
|
||
committed, reserved, err := r1.Store.SpentUSD("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if reserved != 0 || committed != res1.TotalUSD {
|
||
t.Fatalf("ledger: committed=%v reserved=%v want committed=%v", committed, reserved, res1.TotalUSD)
|
||
}
|
||
r1.Close()
|
||
|
||
// Прогон 2 (новый процесс): обе стадии из чекпоинтов/chunk_status за $0.
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("resume must not call the provider again, calls=%d", rec.count())
|
||
}
|
||
if res2.TotalUSD != 0 {
|
||
t.Fatalf("resume must be $0, got %v", res2.TotalUSD)
|
||
}
|
||
for _, st := range res2.Chunks[0].Stages {
|
||
if !st.FromResume || st.CostUSD != 0 {
|
||
t.Fatalf("stage %s must be served from resume at $0: %+v", st.Stage, st)
|
||
}
|
||
}
|
||
if res2.Chunks[0].FinalText != res1.Chunks[0].FinalText {
|
||
t.Fatalf("resume text differs: %q vs %q", res2.Chunks[0].FinalText, res1.Chunks[0].FinalText)
|
||
}
|
||
committed2, _, _ := r2.Store.SpentUSD("test-book")
|
||
if committed2 != committed {
|
||
t.Fatalf("resume must not add spend: %v -> %v", committed, committed2)
|
||
}
|
||
}
|
||
|
||
func TestRunnerEPUBEndToEndAndRubyPersist(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
chapters := []epubChapter{
|
||
{id: "c1", href: "ch1.xhtml", body: `<p><ruby>朱雀<rt>すざく</rt></ruby>は南を守る。</p>`},
|
||
{id: "c2", href: "ch2.xhtml", body: `<p><ruby>羅生門<rt>らしょうもん</rt></ruby>の下で待つ。</p>`},
|
||
}
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{epub: chapters, spine: []string{"c1", "c2"}, regenerate: 1})
|
||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Two spine chapters → two chunks (one each) on the chapter axis, both OK.
|
||
if len(res1.Chunks) != 2 {
|
||
t.Fatalf("want 2 chunks from 2 chapters, got %d: %+v", len(res1.Chunks), res1.Chunks)
|
||
}
|
||
if res1.Chunks[0].Chapter != 1 || res1.Chunks[1].Chapter != 2 ||
|
||
res1.Chunks[0].ChunkIdx != 0 || res1.Chunks[1].ChunkIdx != 0 {
|
||
t.Fatalf("chapter/chunk axis wrong: %+v", res1.Chunks)
|
||
}
|
||
if res1.Flagged != 0 || res1.Chunks[0].FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||
t.Fatalf("run = %+v", res1)
|
||
}
|
||
if rec.count() != 4 { // draft+edit per chunk × 2 chunks
|
||
t.Fatalf("want 4 provider calls, got %d", rec.count())
|
||
}
|
||
// Ruby captured on ingest and persisted with the right first_chapter.
|
||
assertRuby := func(s *store.Store, wantRows int) {
|
||
t.Helper()
|
||
rows, err := s.RubyReadingsForBook("test-book")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(rows) != wantRows {
|
||
t.Fatalf("ruby rows = %d, want %d: %#v", len(rows), wantRows, rows)
|
||
}
|
||
got := map[string]store.RubyReading{}
|
||
for _, rr := range rows {
|
||
got[rr.Base] = rr
|
||
}
|
||
if got["朱雀"].Reading != "すざく" || got["朱雀"].FirstChapter != 1 {
|
||
t.Fatalf("ruby 朱雀 = %+v", got["朱雀"])
|
||
}
|
||
if got["羅生門"].Reading != "らしょうもん" || got["羅生門"].FirstChapter != 2 {
|
||
t.Fatalf("ruby 羅生門 = %+v", got["羅生門"])
|
||
}
|
||
}
|
||
assertRuby(r1.Store, 2)
|
||
r1.Close()
|
||
|
||
// Resume (new process): $0, no new provider calls, ruby persist stays idempotent.
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 4 || res2.TotalUSD != 0 {
|
||
t.Fatalf("resume must be $0 with no new calls: calls=%d usd=%v", rec.count(), res2.TotalUSD)
|
||
}
|
||
assertRuby(r2.Store, 2)
|
||
}
|
||
|
||
func TestRunnerSnapshotPinning(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
if rec.count() != 2 {
|
||
t.Fatalf("run1 calls = %d", rec.count())
|
||
}
|
||
|
||
promptPath := filepath.Join(filepath.Dir(bookPath), "prompts", "translator.md")
|
||
writeFile(t, promptPath, "НОВЫЙ промпт с {{source_lang}}.\n---USER---\n{{text}}")
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
_, err := r2.TranslateBook(ctx)
|
||
r2.Close()
|
||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||
t.Fatalf("changed config must fail loud mentioning --resnapshot, got: %v", err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("denied resume must not call the provider, calls=%d", rec.count())
|
||
}
|
||
|
||
r3 := newRunner(t, bookPath)
|
||
defer r3.Close()
|
||
r3.Resnapshot = true
|
||
res, err := r3.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// R1 per-wave --resnapshot is SURGICAL, not all-or-nothing: the changed TRANSLATOR prompt moves only
|
||
// snapshot_W1, so the DRAFT wave re-pins and re-calls (+1). The EDITOR resumes: snapshot_W2 is unchanged
|
||
// AND its wire input (the mock draft "ЧЕРНОВИК ПЕРЕВОДА" is prompt-independent) is byte-identical, so
|
||
// its content-addressed checkpoint hits — re-billing a byte-identical wire request is exactly the D15
|
||
// waste the per-wave snapshot avoids. Total = 2 (fresh) + 1 (draft re-call) = 3. In production a draft
|
||
// prompt change WOULD change the draft output, cascading to the editor via its content-hash (→ 4).
|
||
if rec.count() != 3 {
|
||
t.Fatalf("resnapshot run must re-call only the changed (draft) wave; the editor resumes on an unchanged wire, calls=%d", rec.count())
|
||
}
|
||
if res.TotalUSD <= 0 {
|
||
t.Fatal("re-translation must be billed")
|
||
}
|
||
}
|
||
|
||
// D5.2/B: правка ТОЛЬКО каппы модели двигает snapshot и роняет resume громко.
|
||
func TestRunnerSnapshotPinsCapability(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
|
||
modelsPath := filepath.Join(filepath.Dir(bookPath), "models.yaml")
|
||
raw, err := os.ReadFile(modelsPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
patched := strings.Replace(string(raw),
|
||
"output_per_m: 2.0 }",
|
||
"output_per_m: 2.0 }\n capabilities: { temperature: { mode: force, value: 0.9 } }", 1)
|
||
if patched == string(raw) {
|
||
t.Fatal("failed to inject capability into models.yaml")
|
||
}
|
||
writeFile(t, modelsPath, patched)
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
_, err = r2.TranslateBook(ctx)
|
||
r2.Close()
|
||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||
t.Fatalf("a capability edit must fail loud mentioning --resnapshot, got: %v", err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("denied resume must not call the provider, calls=%d", rec.count())
|
||
}
|
||
}
|
||
|
||
func TestRunnerZeroUsagePaidSettlesEstimate(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
fmt.Fprint(w, `{"id":"z","model":"deepseek-v4-flash","choices":[{"message":{"content":"перевод"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":0,"completion_tokens":0}}`)
|
||
}))
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.TotalUSD <= 0 {
|
||
t.Fatalf("zero-usage paid 2xx must settle a non-zero estimate, got $%.6f", res.TotalUSD)
|
||
}
|
||
committed, _, _ := r.Store.SpentUSD("test-book")
|
||
if committed != res.TotalUSD {
|
||
t.Fatalf("committed %v != total %v", committed, res.TotalUSD)
|
||
}
|
||
}
|
||
|
||
func TestRunnerRejectsUnrunnableConfig(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, pipePath, string(raw)+"\nfanout: { candidates: 3 }\n")
|
||
|
||
if _, err := NewRunner(bookPath, obs.NewLogger()); err == nil {
|
||
t.Fatal("fanout.candidates>1 must be rejected by NewRunner (Phase-2 mechanics)")
|
||
}
|
||
if rec.count() != 0 {
|
||
t.Fatalf("rejected config must not reach the provider, calls=%d", rec.count())
|
||
}
|
||
}
|
||
|
||
func TestRunnerCeilingDenies(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{bookUSD: 0.0000001})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
_, err := r.TranslateBook(context.Background())
|
||
if err == nil {
|
||
t.Fatal("ceiling must deny the run")
|
||
}
|
||
if rec.count() != 0 {
|
||
t.Fatalf("denied reserve must not reach the provider, calls=%d", rec.count())
|
||
}
|
||
}
|
||
|
||
// --- Веха 2: chunk loop, disposition, resume, exit codes --------------------
|
||
|
||
// Multi-chunk (two chapters via form feed) end-to-end + resume at $0.
|
||
func TestRunnerMultiChunkResume(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ГЛАВАОДИН\fГЛАВАДВА"})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
if len(res1.Chunks) != 2 {
|
||
t.Fatalf("want 2 chunks (2 chapters), got %d", len(res1.Chunks))
|
||
}
|
||
if res1.Chunks[0].Chapter != 1 || res1.Chunks[1].Chapter != 2 {
|
||
t.Fatalf("chapters = %d,%d", res1.Chunks[0].Chapter, res1.Chunks[1].Chapter)
|
||
}
|
||
if rec.count() != 4 { // 2 chunks × (draft+edit)
|
||
t.Fatalf("want 4 calls, got %d", rec.count())
|
||
}
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 4 {
|
||
t.Fatalf("resume must not re-call, calls=%d", rec.count())
|
||
}
|
||
if res2.TotalUSD != 0 || res2.Flagged != 0 || res2.ExitCode() != 0 {
|
||
t.Fatalf("resume run = %+v", res2)
|
||
}
|
||
}
|
||
|
||
// D2 core: a flagged chunk skips its remaining stages and the loop continues to
|
||
// the next chunk; the run completes with exit code 2, and resume does not
|
||
// re-attack the flagged chunk.
|
||
func TestRunnerFlagSkipContinueExit2(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) {
|
||
return "Извините, я не могу перевести это.", "stop" // soft refusal
|
||
}
|
||
return draftEdit(body)
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА"})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("a flagged chunk must NOT be an error (loop continues), got %v", err)
|
||
}
|
||
// draft(ch1)=refusal → edit(ch1) skipped (no call); draft(ch2)+edit(ch2) ok = 3 calls.
|
||
if rec.count() != 3 {
|
||
t.Fatalf("want 3 calls (edit of flagged chunk skipped), got %d", rec.count())
|
||
}
|
||
if res1.Flagged != 1 || res1.ExitCode() != 2 {
|
||
t.Fatalf("want 1 flagged chunk / exit 2, got flagged=%d exit=%d", res1.Flagged, res1.ExitCode())
|
||
}
|
||
c1 := res1.Chunks[0]
|
||
if c1.Disposition != DispFlagged || c1.FlagReason != FlagSoftRefusal || c1.FinalText != "" {
|
||
t.Fatalf("chunk1 = %+v (want flagged/soft_refusal/no text)", c1)
|
||
}
|
||
if c1.Stages[1].Disposition != DispSkipped {
|
||
t.Fatalf("edit of a flagged chunk must be skipped, got %+v", c1.Stages[1])
|
||
}
|
||
if res1.Chunks[1].Disposition != DispOK {
|
||
t.Fatalf("chunk2 must be ok, got %+v", res1.Chunks[1])
|
||
}
|
||
// chunk_status persisted the flag and the skip.
|
||
cs, _ := r1.Store.GetChunkStatus("test-book", 1, 0, "draft")
|
||
if cs == nil || cs.Disposition != "flagged" || cs.FlagReason != "soft_refusal" {
|
||
t.Fatalf("draft chunk_status = %+v", cs)
|
||
}
|
||
skip, _ := r1.Store.GetChunkStatus("test-book", 1, 0, "edit")
|
||
if skip == nil || skip.Disposition != "skipped" {
|
||
t.Fatalf("edit chunk_status = %+v", skip)
|
||
}
|
||
r1.Close()
|
||
|
||
// Resume: the flagged chunk is resolved from chunk_status, NOT re-attacked.
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 3 {
|
||
t.Fatalf("resume must not re-attack a flagged chunk, calls=%d", rec.count())
|
||
}
|
||
if res2.Flagged != 1 || res2.TotalUSD != 0 || res2.ExitCode() != 2 {
|
||
t.Fatalf("resume run = %+v", res2)
|
||
}
|
||
}
|
||
|
||
// F4: a non-empty truncated (finish=length) draft must be FLAGGED, never passed
|
||
// downstream to the editor as OK. With no regenerations it flags immediately.
|
||
func TestRunnerF4TruncatedLengthNotPassedToEdit(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if !isEditBody(body) {
|
||
return "Обрыв на середине предложения", "length" // truncated, non-empty, not a loop
|
||
}
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("a truncated draft must flag, not crash: %v", err)
|
||
}
|
||
if rec.count() != 1 {
|
||
t.Fatalf("edit must NOT run on a flagged draft, calls=%d", rec.count())
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagLength {
|
||
t.Fatalf("chunk must be flagged length, got %+v", ch)
|
||
}
|
||
if ch.Stages[0].Disposition != DispFlagged || ch.Stages[1].Disposition != DispSkipped {
|
||
t.Fatalf("draft flagged + edit skipped expected, got %+v", ch.Stages)
|
||
}
|
||
}
|
||
|
||
// D2.3 + F3 + attempt-axis: a length cut is retried with a DOUBLED budget; the
|
||
// second attempt succeeds, the editor then runs, and cost sums both attempts.
|
||
func TestRunnerLengthRetrySucceedsDoublesBudget(t *testing.T) {
|
||
rec := &reqRec{}
|
||
src := strings.Repeat("字", 500) // EstimateTokens=500 → base max_tokens=1000
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
}
|
||
if maxTokensOf(t, body) <= 1000 { // attempt 0 (base)
|
||
return "Обрыв на середине", "length"
|
||
}
|
||
return "ПОЛНЫЙ ЧЕРНОВИК", "stop" // attempt 1 (doubled budget)
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: src, regenerate: 1, minMaxTokens: 128})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispOK || ch.FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||
t.Fatalf("retry should succeed and edit should run, got %+v", ch)
|
||
}
|
||
draft := ch.Stages[0]
|
||
if draft.Attempts != 2 {
|
||
t.Fatalf("draft must have taken 2 attempts, got %d", draft.Attempts)
|
||
}
|
||
// F3: cost sums BOTH attempts.
|
||
if diff := draft.CumCostUSD - 2*fakeCallUSD; diff > 1e-12 || diff < -1e-12 {
|
||
t.Fatalf("draft cum cost = %v, want 2×call %v (attempts summed)", draft.CumCostUSD, 2*fakeCallUSD)
|
||
}
|
||
// The two draft attempts doubled the budget: 1000 then 2000.
|
||
var draftToks []int
|
||
for _, b := range rec.all() {
|
||
if !isEditBody(b) {
|
||
draftToks = append(draftToks, maxTokensOf(t, b))
|
||
}
|
||
}
|
||
if len(draftToks) != 2 || draftToks[0] != 1000 || draftToks[1] != 2000 {
|
||
t.Fatalf("attempt budgets = %v, want [1000 2000]", draftToks)
|
||
}
|
||
}
|
||
|
||
// D2.5: the editor's max_tokens is sized from the (long Russian) DRAFT, not the
|
||
// (short CJK) source — otherwise it under-budgets and false-triggers a length
|
||
// retry. Broken D2.5 would make the two budgets equal.
|
||
func TestRunnerEditSizedFromDraft(t *testing.T) {
|
||
rec := &reqRec{}
|
||
longDraft := strings.Repeat("Утро было тихим и ясным над рекой. ", 120) // long Russian output
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОК", "stop"
|
||
}
|
||
return longDraft, "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: strings.Repeat("字", 100), minMaxTokens: 128})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var draftTok, editTok int
|
||
for _, b := range rec.all() {
|
||
if isEditBody(b) {
|
||
editTok = maxTokensOf(t, b)
|
||
} else {
|
||
draftTok = maxTokensOf(t, b)
|
||
}
|
||
}
|
||
if editTok <= draftTok {
|
||
t.Fatalf("edit max_tokens (%d) must exceed draft's (%d) — sized from the longer draft (D2.5)", editTok, draftTok)
|
||
}
|
||
}
|
||
|
||
// Anti-wedge #2: an empty completion is FLAGGED and the loop continues — not a
|
||
// crash (the Фаза-0 behaviour was to error out the whole run).
|
||
func TestRunnerEmptyFlaggedNotCrash(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if !isEditBody(body) {
|
||
return "", "stop" // empty draft
|
||
}
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatalf("empty completion must flag, not crash: %v", err)
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagEmpty {
|
||
t.Fatalf("empty draft must be flagged empty, got %+v", ch)
|
||
}
|
||
if res.ExitCode() != 2 {
|
||
t.Fatalf("exit code must be 2, got %d", res.ExitCode())
|
||
}
|
||
// Money for the paid-but-empty call is still accounted.
|
||
committed, _, _ := r.Store.SpentUSD("test-book")
|
||
if committed <= 0 {
|
||
t.Fatalf("the paid empty call must still be billed, committed=%v", committed)
|
||
}
|
||
}
|
||
|
||
// Anti-wedge #2: a billed 2xx with an unreadable body → flagged decode_error
|
||
// with the estimate settled, loop continues (not a crash).
|
||
func TestRunnerDecodeErrorFlagged(t *testing.T) {
|
||
var draftCalls int
|
||
var mu sync.Mutex
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
if !isEditBody(string(body)) {
|
||
mu.Lock()
|
||
draftCalls++
|
||
mu.Unlock()
|
||
fmt.Fprint(w, `NOT JSON AT ALL`) // 200 with a garbage body → BilledDecodeError
|
||
return
|
||
}
|
||
fmt.Fprint(w, `{"id":"e","model":"fake-model","choices":[{"message":{"content":"ред"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
||
}))
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatalf("a billed decode failure must flag, not crash: %v", err)
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagDecodeError {
|
||
t.Fatalf("draft must be flagged decode_error, got %+v", ch)
|
||
}
|
||
if ch.Stages[1].Disposition != DispSkipped {
|
||
t.Fatalf("edit must be skipped after a decode flag, got %+v", ch.Stages[1])
|
||
}
|
||
if res.TotalUSD <= 0 {
|
||
t.Fatal("the conservative estimate must be settled for a billed decode failure")
|
||
}
|
||
}
|
||
|
||
// Determinism content-guard (self-review Веха 2): the source is NOT in the
|
||
// snapshot, so editing a chunk's source between runs must RE-TRANSLATE that
|
||
// position (content-addressed, §3.4), never serve the stale positional
|
||
// chunk_status row. Without the content_hash guard the fast-path would return
|
||
// the OLD translation at $0.
|
||
func TestRunnerSourceEditReTranslates(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "СТАРЫЙИСХОДНИК"})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count()
|
||
|
||
// Edit the source in place (same position, still one chunk). No brief field
|
||
// and not the chunker changed, so the snapshot is unchanged — the ONLY thing
|
||
// that must invalidate resume is the content itself.
|
||
writeFile(t, filepath.Join(filepath.Dir(bookPath), "source.txt"), "НОВЫЙИСХОДНИК")
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Both stages re-render with the new content (the editor sees {{text}} too),
|
||
// miss their old checkpoints, and are re-translated and re-billed.
|
||
if rec.count() != callsAfterRun1+2 {
|
||
t.Fatalf("an edited source must re-translate the chunk, not serve stale: calls %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
if res.TotalUSD <= 0 {
|
||
t.Fatalf("re-translation of an edited source must be billed, got $%.6f", res.TotalUSD)
|
||
}
|
||
}
|
||
|
||
// Anti-wedge #1: a terminally-flagged chunk is resolved from chunk_status BEFORE
|
||
// rendering, so it is NOT re-attacked even when the retry budget is later raised
|
||
// (RegenerateBeforeEscalate is not in the snapshot). Without the chunk_status
|
||
// fast-path the attempt loop would spawn NEW paid attempts up to the new cap.
|
||
func TestRunnerFlaggedNotReattackedWhenRegenRaised(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if !isEditBody(body) {
|
||
return "Обрыв на середине", "length" // always a length cut
|
||
}
|
||
return "ред", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count() // regen=0 → exactly 1 draft attempt, flagged length
|
||
|
||
// Raise the regeneration budget and re-run. The flagged chunk must stay put.
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, pipePath, strings.Replace(string(raw), "regenerate_before_escalate: 0", "regenerate_before_escalate: 2", 1))
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatalf("a terminally-flagged chunk must NOT be re-attacked on a raised budget: calls %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
if res.Chunks[0].Disposition != DispFlagged || res.Chunks[0].FlagReason != FlagLength {
|
||
t.Fatalf("chunk must stay flagged length, got %+v", res.Chunks[0])
|
||
}
|
||
}
|
||
|
||
// Anti-wedge #3: a checkpoint present without its chunk_status row (kill -9 lost
|
||
// the resolve) self-heals — classify the checkpoint text, no re-billing, no
|
||
// crash. Proven by deleting the chunk_status row and re-running with regen=0 so
|
||
// the empty flag cannot spawn a new paid attempt.
|
||
func TestRunnerLegacyEmptyCheckpointSelfHeal(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if !isEditBody(body) {
|
||
return "", "stop" // empty draft → empty checkpoint
|
||
}
|
||
return "ред", "stop"
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
dbPath := r1.Book.ProjectDB
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count()
|
||
|
||
// Simulate the resolve being lost while the (empty) checkpoint survives.
|
||
db, err := sql.Open("sqlite", "file:"+dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := db.Exec(`DELETE FROM chunk_status WHERE stage = 'draft'`); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
db.Close()
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("legacy empty checkpoint must self-heal, not crash: %v", err)
|
||
}
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatalf("self-heal must NOT re-bill the empty attempt: calls %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagEmpty {
|
||
t.Fatalf("self-heal must re-derive the empty flag, got %+v", ch)
|
||
}
|
||
// chunk_status was rebuilt.
|
||
cs, _ := r2.Store.GetChunkStatus("test-book", 1, 0, "draft")
|
||
if cs == nil || cs.Disposition != "flagged" {
|
||
t.Fatalf("chunk_status must be rebuilt as flagged, got %+v", cs)
|
||
}
|
||
}
|
||
|
||
// --- Веха 2.5: excision coverage-gate (A, D12 Q3) ---------------------------
|
||
|
||
// coverageGateBlock enables the gate with the boevoy ja-ru corridor; min_chunk_chars
|
||
// is low so the test fixtures qualify.
|
||
const coverageGateBlock = `
|
||
gates:
|
||
coverage:
|
||
enabled: true
|
||
len_ratio_bounds: { ja-ru: [1.4, 2.6] }
|
||
sent_cov_min: 0.75
|
||
min_chunk_chars: 100
|
||
`
|
||
|
||
// A draft that excises a long source into one short sentence must be flagged
|
||
// excision_suspect (non-retryable → edit skipped, exit 2); resume reproduces the
|
||
// verdict from chunk_status without re-attacking.
|
||
func TestRunnerCoverageGateFlagsExcision(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ред", "stop"
|
||
}
|
||
return "Короткий огрызок перевода.", "stop" // 30 source sentences → 1
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||
source: strings.Repeat("これは長い文である。", 30), regenerate: 1, gatesYAML: coverageGateBlock,
|
||
})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatalf("an excised chunk must flag, not crash: %v", err)
|
||
}
|
||
if rec.count() != 1 {
|
||
t.Fatalf("edit must be skipped after an excision flag, calls=%d", rec.count())
|
||
}
|
||
ch := res1.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagExcisionSuspect {
|
||
t.Fatalf("draft must be flagged excision_suspect, got %+v", ch)
|
||
}
|
||
if ch.Stages[0].Attempts != 1 {
|
||
t.Fatalf("excision_suspect is non-retryable — exactly one attempt, got %d", ch.Stages[0].Attempts)
|
||
}
|
||
if res1.ExitCode() != 2 {
|
||
t.Fatalf("exit code must be 2, got %d", res1.ExitCode())
|
||
}
|
||
r1.Close()
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 1 {
|
||
t.Fatalf("resume must not re-attack the excised chunk, calls=%d", rec.count())
|
||
}
|
||
if res2.Chunks[0].FlagReason != FlagExcisionSuspect {
|
||
t.Fatalf("resume verdict differs: %+v", res2.Chunks[0])
|
||
}
|
||
}
|
||
|
||
// A faithful, full-length translation must pass the gate — both stages run, no flag
|
||
// (precision: the gate must not false-flag a healthy chunk).
|
||
func TestRunnerCoverageGatePassesHealthy(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
// The editor is ALSO gated against the original source (excision can happen
|
||
// at edit too), so it must return a full-length polish, not a short stub.
|
||
if isEditBody(body) {
|
||
return strings.Repeat("Правленое предложение перевода. ", 30), "stop"
|
||
}
|
||
return strings.Repeat("Это предложение перевода. ", 30), "stop" // 30 sentences ≈ source
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||
source: strings.Repeat("これは長い文である。", 30), regenerate: 1, gatesYAML: coverageGateBlock,
|
||
})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("a healthy chunk must run both stages, calls=%d", rec.count())
|
||
}
|
||
if res.Flagged != 0 || res.ExitCode() != 0 || res.Chunks[0].Disposition != DispOK {
|
||
t.Fatalf("healthy chunk must pass the gate, got %+v", res.Chunks[0])
|
||
}
|
||
}
|
||
|
||
// The gate config is folded into the snapshot: enabling it on an in-progress book
|
||
// changes the snapshot, so resume fails loud until --resnapshot (else already-done
|
||
// chunks would silently never be re-checked by the now-on gate).
|
||
func TestRunnerCoverageGateEntersSnapshot(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL) // gate OFF
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, pipePath, string(raw)+coverageGateBlock)
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
_, err = r2.TranslateBook(ctx)
|
||
r2.Close()
|
||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||
t.Fatalf("enabling the coverage gate must fail loud mentioning --resnapshot, got: %v", err)
|
||
}
|
||
}
|
||
|
||
// An enabled gate with NO usable thresholds is a silent no-op (passes everything) —
|
||
// LoadPipeline must fail-fast rather than fail-open (against Р7).
|
||
func TestRunnerRejectsCoverageGateWithoutThresholds(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProject(t, srv.URL)
|
||
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, pipePath, string(raw)+"\ngates:\n coverage:\n enabled: true\n")
|
||
|
||
if _, err := NewRunner(bookPath, obs.NewLogger()); err == nil || !strings.Contains(err.Error(), "silently pass") {
|
||
t.Fatalf("an enabled gate with no thresholds must fail-fast, got: %v", err)
|
||
}
|
||
if rec.count() != 0 {
|
||
t.Fatalf("a rejected config must not reach the provider, calls=%d", rec.count())
|
||
}
|
||
}
|
||
|
||
// --- Веха 2.5: single-hop escalation (B, D12) -------------------------------
|
||
|
||
// setupEscalationProject wires a ja→ru book whose draft stage escalates to a second
|
||
// model (fake-fallback) on a deterministic content-failure, under escalation
|
||
// budget_usd. Both models sit on the same fake provider; the mock distinguishes them
|
||
// by the "model" field in the request body. Optional permissive marks the provider
|
||
// permissive and channel=adult on the draft (D4.1 isolation test).
|
||
func setupEscalationProject(t *testing.T, providerURL string, budgetUSD float64, permissive bool, channel string) string {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
|
||
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
|
||
|
||
permLine := ""
|
||
if permissive {
|
||
permLine = "\n permissive: true"
|
||
}
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q%s
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
fake-fallback:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
`, time.Now().UTC().Format("2006-01-02"), providerURL, permLine))
|
||
|
||
chanLine := ""
|
||
if channel != "" {
|
||
chanLine = ", channel: " + channel
|
||
}
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), fmt.Sprintf(`
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||
retries: { regenerate_before_escalate: 0 }
|
||
stages:
|
||
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off", escalate_to: fake-fallback%s }
|
||
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||
escalation: { budget_usd: %g }
|
||
`, chanLine, budgetUSD))
|
||
|
||
writeFile(t, filepath.Join(dir, "source.txt"), "静かな図書館の朝。")
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), `
|
||
book_id: test-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.txt
|
||
ceilings: { book_usd: 5.0, day_usd: 10.0 }
|
||
`)
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
// echoOrClean is the escalation mock: the primary (fake-model) ECHOES the CJK source
|
||
// (cjk_artifact); the fallback (fake-fallback) returns a clean Russian translation.
|
||
func echoOrClean(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
|
||
}
|
||
if strings.Contains(body, "fake-fallback") {
|
||
return "Тихое утро в библиотеке.", "stop"
|
||
}
|
||
return "静かな図書館の朝。", "stop" // echo of the CJK source → cjk_artifact
|
||
}
|
||
|
||
// A deterministic echo (cjk_artifact) is escalated ONCE to the fallback model, which
|
||
// translates cleanly → the chunk is OK, billed to the fallback, and the edit runs.
|
||
func TestRunnerEscalationFixesEcho(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// draft(fake-model)=echo → escalate draft(fake-fallback)=clean → edit = 3 calls.
|
||
if rec.count() != 3 {
|
||
t.Fatalf("want 3 calls (primary + fallback + edit), got %d", rec.count())
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispOK || res.Flagged != 0 || res.ExitCode() != 0 {
|
||
t.Fatalf("escalated chunk must resolve OK, got %+v (flagged=%d)", ch, res.Flagged)
|
||
}
|
||
draft := ch.Stages[0]
|
||
// The mock canonicalizes every response model to "fake-model", so Model reflects
|
||
// that; the escalation is proven by Escalated + EscalationModel + the fallback's text.
|
||
if !draft.Escalated || draft.EscalationModel != "fake-fallback" || draft.Text != "Тихое утро в библиотеке." {
|
||
t.Fatalf("draft must record the successful escalation to fake-fallback, got %+v", draft)
|
||
}
|
||
if ch.FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||
t.Fatalf("final text must come from the edit over the fallback draft, got %q", ch.FinalText)
|
||
}
|
||
esc, err := r.Store.EscalationSpentUSD("test-book")
|
||
if err != nil || esc <= 0 {
|
||
t.Fatalf("the escalation call must be tagged and summable, got %v (err %v)", esc, err)
|
||
}
|
||
}
|
||
|
||
// If the fallback ALSO fails the re-gate, the chunk stays flagged (primary reason),
|
||
// the edit is skipped, and it was exactly ONE hop (primary + fallback, no more).
|
||
func TestRunnerEscalationFallbackAlsoFailsFlags(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ред", "stop"
|
||
}
|
||
return "静かな図書館の朝。", "stop" // BOTH models echo → cjk_artifact
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 2 {
|
||
t.Fatalf("want exactly 2 calls (primary + 1 fallback hop, edit skipped), got %d", rec.count())
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagCJKArtifact || res.ExitCode() != 2 {
|
||
t.Fatalf("a failed escalation must keep the primary flag, got %+v", ch)
|
||
}
|
||
if !ch.Stages[0].Escalated {
|
||
t.Fatalf("the draft must record that escalation was tried, got %+v", ch.Stages[0])
|
||
}
|
||
if ch.Stages[1].Disposition != DispSkipped {
|
||
t.Fatalf("edit must be skipped after a failed escalation, got %+v", ch.Stages[1])
|
||
}
|
||
}
|
||
|
||
// escalation.budget_usd = 0 disables escalation (opt-in): the deterministic flag
|
||
// stays a flag, no fallback call is made.
|
||
func TestRunnerEscalationBudgetZeroDisables(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 0, false, "") // budget 0
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != 1 {
|
||
t.Fatalf("budget=0 must not make a fallback call, got %d calls", rec.count())
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagCJKArtifact || ch.Stages[0].Escalated {
|
||
t.Fatalf("with no budget the echo must stay flagged, unescalated, got %+v", ch.Stages[0])
|
||
}
|
||
}
|
||
|
||
// A successful escalation is resume-safe: the chunk resolves from chunk_status and
|
||
// its final_hash points at the FALLBACK checkpoint, served at $0 with no re-call.
|
||
func TestRunnerEscalationResumeServesFallback(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count()
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatalf("resume must not re-call after a successful escalation, calls %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
if res.TotalUSD != 0 || res.Chunks[0].Disposition != DispOK {
|
||
t.Fatalf("resume must serve the escalated chunk at $0, got %+v", res.Chunks[0])
|
||
}
|
||
if res.Chunks[0].FinalText != "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД" {
|
||
t.Fatalf("resume text differs: %q", res.Chunks[0].FinalText)
|
||
}
|
||
// Escalation attribution survives resume (self-review [11/14]).
|
||
if d := res.Chunks[0].Stages[0]; !d.Escalated || d.EscalationModel != "fake-fallback" {
|
||
t.Fatalf("resume must preserve escalation attribution, got %+v", d)
|
||
}
|
||
}
|
||
|
||
// Only the TRANSLATOR stage is coverage-gated (self-review [6]): a monolingual editor
|
||
// that legitimately merges/tightens sentences (low sent_cov vs the CJK source) must
|
||
// NOT be flagged as excision. Mutation: gate every stage and the short edit flags.
|
||
func TestRunnerCoverageGateOnlyTranslator(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "Одно.", "stop" // a very short "polish" — would flag if the editor were gated
|
||
}
|
||
return strings.Repeat("Это предложение перевода. ", 30), "stop" // faithful translator draft
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||
source: strings.Repeat("これは長い文である。", 30), regenerate: 0, gatesYAML: coverageGateBlock,
|
||
})
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.Flagged != 0 || res.ExitCode() != 0 || res.Chunks[0].Disposition != DispOK {
|
||
t.Fatalf("a short monolingual edit must NOT be excision-flagged (editor is not gated), got %+v", res.Chunks[0])
|
||
}
|
||
}
|
||
|
||
// A successful, already-PAID escalation must be resume-idempotent even when the budget
|
||
// no longer admits a fresh hop (self-review [2/9/13]): a crash between the fallback
|
||
// settle and the chunk_status write must REPLAY the paid checkpoint, not discard it and
|
||
// flip the chunk OK→flagged. Mutation: drop the fbExists check and the chunk flips.
|
||
func TestRunnerEscalationResumeIdempotentUnderBudget(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
// budget < one hop cost: after the first hop, EscalationSpentUSD ≥ budget, so a
|
||
// FRESH hop would be denied on resume — the paid hop must replay regardless.
|
||
bookPath := setupEscalationProject(t, srv.URL, 0.001, false, "")
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil || res1.Chunks[0].Disposition != DispOK {
|
||
t.Fatalf("run1 must escalate to OK, got %+v (err %v)", res1.Chunks[0], err)
|
||
}
|
||
dbPath := r1.Book.ProjectDB
|
||
r1.Close()
|
||
callsAfterRun1 := rec.count()
|
||
|
||
// Simulate the crash window: chunk_status lost, checkpoints (incl. escalation) survive.
|
||
db, err := sql.Open("sqlite", "file:"+dbPath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := db.Exec(`DELETE FROM chunk_status WHERE stage = 'draft'`); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
db.Close()
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatalf("resume must REPLAY the paid escalation checkpoint, not re-call: calls %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
if res2.Chunks[0].Disposition != DispOK {
|
||
t.Fatalf("resume must re-serve the escalated OK chunk, not flip to flagged (budget must not orphan a paid hop), got %+v", res2.Chunks[0])
|
||
}
|
||
}
|
||
|
||
// An OPTIONAL escalation hop that trips the book USD ceiling must degrade to keeping
|
||
// the primary flag, NOT abort the whole book (and re-abort on every resume) — self-
|
||
// review [10]. Mutation: propagate the ceiling error and TranslateBook returns non-nil.
|
||
func TestRunnerEscalationCeilingDegrades(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||
|
||
// Lower the book ceiling so the primary draft call fits but the escalation hop does not.
|
||
bp := filepath.Join(filepath.Dir(bookPath), "book.yaml")
|
||
raw, err := os.ReadFile(bp)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, bp, strings.Replace(string(raw), "book_usd: 5.0", "book_usd: 0.0025", 1))
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(context.Background())
|
||
if err != nil {
|
||
t.Fatalf("an escalation-hop ceiling denial must NOT abort the book, got: %v", err)
|
||
}
|
||
ch := res.Chunks[0]
|
||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagCJKArtifact {
|
||
t.Fatalf("the chunk must keep its primary flag when escalation is ceiling-denied, got %+v", ch)
|
||
}
|
||
if ch.Stages[0].Escalated {
|
||
t.Fatalf("a ceiling-denied hop must not be recorded as an escalation, got %+v", ch.Stages[0])
|
||
}
|
||
}
|
||
|
||
// The fallback model's top-level extra_body is folded into the snapshot (self-review
|
||
// [1]): editing it changes the escalation wire, so resume must fail loud until
|
||
// --resnapshot, never silently serve a stale escalated checkpoint.
|
||
func TestRunnerEscalationFoldsFallbackExtraBody(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
|
||
// Add extra_body to the FALLBACK model only — it changes the escalation wire body.
|
||
mp := filepath.Join(filepath.Dir(bookPath), "models.yaml")
|
||
raw, err := os.ReadFile(mp)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
patched := strings.Replace(string(raw),
|
||
" fake-fallback:\n provider: fake",
|
||
" fake-fallback:\n provider: fake\n extra_body: { top_p: 0.5 }", 1)
|
||
if patched == string(raw) {
|
||
t.Fatal("failed to inject extra_body into fake-fallback")
|
||
}
|
||
writeFile(t, mp, patched)
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
_, err = r2.TranslateBook(ctx)
|
||
r2.Close()
|
||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||
t.Fatalf("editing the fallback's extra_body must fail loud mentioning --resnapshot, got: %v", err)
|
||
}
|
||
}
|
||
|
||
// The fallback model is folded into the snapshot: removing escalate_to changes the
|
||
// snapshot, so resume fails loud until --resnapshot (D5.2 escalation-capability closure).
|
||
func TestRunnerEscalationEntersSnapshot(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, pipePath, strings.Replace(string(raw), ", escalate_to: fake-fallback", "", 1))
|
||
|
||
r2 := newRunner(t, bookPath)
|
||
_, err = r2.TranslateBook(ctx)
|
||
r2.Close()
|
||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||
t.Fatalf("changing the fallback model must fail loud mentioning --resnapshot, got: %v", err)
|
||
}
|
||
}
|
||
|
||
// D12 editor-pinned, enforced structurally (external-review [4]): escalate_to on a
|
||
// NON-translator role is rejected at load — an editor must not fall back to a foreign
|
||
// model. Mutation: drop the role check and this loads instead of failing.
|
||
func TestRunnerRejectsEscalateToOnNonTranslator(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
bookPath := setupEscalationProject(t, srv.URL, 1.0, false, "")
|
||
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Put escalate_to on the EDITOR stage (role=editor) — must be rejected.
|
||
patched := strings.Replace(string(raw),
|
||
`{ name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }`,
|
||
`{ name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off", escalate_to: fake-fallback }`, 1)
|
||
if patched == string(raw) {
|
||
t.Fatal("failed to inject escalate_to onto the editor stage")
|
||
}
|
||
writeFile(t, pipePath, patched)
|
||
|
||
if _, err := NewRunner(bookPath, obs.NewLogger()); err == nil || !strings.Contains(err.Error(), "translator") {
|
||
t.Fatalf("escalate_to on a non-translator role must fail-fast (D12 editor-pinned), got: %v", err)
|
||
}
|
||
}
|
||
|
||
// The intrinsic classify() version AND the local backend tag are folded into the
|
||
// snapshot (external-review [1]/[2]): a threshold or backend-tag change is a loud
|
||
// --resnapshot, not a silent re-verdict / stale-model resume. Mutation: drop either
|
||
// fold and the payload no longer carries it.
|
||
func TestRunnerSnapshotFoldsClassifierAndLocalTag(t *testing.T) {
|
||
dir := t.TempDir()
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: anchor
|
||
providers:
|
||
loc:
|
||
kind: local
|
||
base_url: http://127.0.0.1:11434/v1
|
||
model: my-backend-tag-8b
|
||
max_tokens: 8192
|
||
cloud:
|
||
kind: openai
|
||
base_url: http://x
|
||
models:
|
||
anchor:
|
||
provider: cloud
|
||
price: { input_per_m: 1, cached_per_m: 0, cache_write_per_m: 0, output_per_m: 2 }
|
||
local-m:
|
||
provider: loc
|
||
price: { input_per_m: 0, cached_per_m: 0, cache_write_per_m: 0, output_per_m: 0 }
|
||
`, time.Now().UTC().Format("2006-01-02")))
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), `
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||
retries: { regenerate_before_escalate: 0 }
|
||
stages:
|
||
- { name: draft, role: translator, model: local-m, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
||
`)
|
||
writeFile(t, filepath.Join(dir, "source.txt"), "猫。")
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), `
|
||
book_id: b
|
||
title: T
|
||
source_lang: zh
|
||
target_lang: ru
|
||
genre: g
|
||
audience: a
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: pinyin
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.txt
|
||
ceilings: { book_usd: 1.0, day_usd: 1.0 }
|
||
`)
|
||
|
||
r := newRunner(t, filepath.Join(dir, "book.yaml"))
|
||
defer r.Close()
|
||
_, payload, err := r.snapshotID()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(payload, classifierVersion) {
|
||
t.Fatalf("snapshot must fold the classifier version %q, got: %s", classifierVersion, payload)
|
||
}
|
||
if !strings.Contains(payload, "my-backend-tag-8b") {
|
||
t.Fatalf("snapshot must fold the local backend tag (provider_model), got: %s", payload)
|
||
}
|
||
}
|
||
|
||
// D4.1: channel-B (18+) isolation is enforced by TYPE — a channel=adult stage on a
|
||
// NON-permissive provider is rejected at load, never a silent fall-through.
|
||
func TestRunnerChannelBRequiresPermissive(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, echoOrClean)
|
||
defer srv.Close()
|
||
|
||
// channel=adult, provider NOT permissive → NewRunner must fail.
|
||
nonPerm := setupEscalationProject(t, srv.URL, 1.0, false, "adult")
|
||
if _, err := NewRunner(nonPerm, obs.NewLogger()); err == nil || !strings.Contains(err.Error(), "permissive") {
|
||
t.Fatalf("channel=adult on a non-permissive provider must fail-fast, got: %v", err)
|
||
}
|
||
|
||
// channel=adult, provider permissive → accepted.
|
||
perm := setupEscalationProject(t, srv.URL, 1.0, true, "adult")
|
||
r, err := NewRunner(perm, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatalf("channel=adult on a permissive provider must load, got: %v", err)
|
||
}
|
||
r.Close()
|
||
}
|
||
|
||
// TestReadOnlyRunnerSkipsKeyCheck is the D20.4 read-only-audit fix: `report`/`status` build via
|
||
// NewReadOnlyRunner, which must open a project whose stage provider's API key is UNSET — NewRunner
|
||
// (translate/redrive path) must still fail loud on the same config. Reverting the checkKeys guard
|
||
// makes NewReadOnlyRunner fail here too.
|
||
func TestReadOnlyRunnerSkipsKeyCheck(t *testing.T) {
|
||
dir := t.TempDir()
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"), "T\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"), "E\n---USER---\n{{text}}\n{{draft}}")
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: keyed-model
|
||
providers:
|
||
keyed:
|
||
kind: openai
|
||
base_url: http://127.0.0.1:1/v1
|
||
api_key_env: TM_TEST_DEFINITELY_UNSET_KEY
|
||
timeouts: { attempt_s: 5, max_attempts: 1, backoff_cap_s: 1 }
|
||
models:
|
||
keyed-model:
|
||
provider: keyed
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
`, time.Now().UTC().Format("2006-01-02")))
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), `
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||
stages:
|
||
- { name: draft, role: translator, model: keyed-model, prompt: prompts/translator.md, prompt_version: v, temperature: 0.3, reasoning: "off" }
|
||
`)
|
||
writeFile(t, filepath.Join(dir, "source.txt"), "テキスト。")
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), `
|
||
book_id: test-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.txt
|
||
ceilings: { book_usd: 1.0, day_usd: 2.0 }
|
||
`)
|
||
bookPath := filepath.Join(dir, "book.yaml")
|
||
|
||
if _, err := NewRunner(bookPath, obs.NewLogger()); err == nil || !strings.Contains(err.Error(), "TM_TEST_DEFINITELY_UNSET_KEY") {
|
||
t.Fatalf("NewRunner must fail loud on the missing stage key, got: %v", err)
|
||
}
|
||
r, err := NewReadOnlyRunner(bookPath, obs.NewLogger())
|
||
if err != nil {
|
||
t.Fatalf("NewReadOnlyRunner must open the project without the API key ($0 read), got: %v", err)
|
||
}
|
||
r.Close()
|
||
}
|
||
|
||
// TestRunnerAdditiveReasoningBufferReserved verifies the xAI path (Task 3): a reasoning-ON stage on
|
||
// an ADDITIVE-billing provider reserves the declared reasoning_max_tokens buffer in EstimateUSD, so
|
||
// the additive reasoning spend is NOT invisible to the book ceiling. Method: a huge buffer pushes
|
||
// the FIRST draft reservation past a $1 ceiling — the run aborts loud. Reverting the
|
||
// AdditiveReasoningTokens wiring in runAttempt (reserve 0) lets the reservation fit and the run
|
||
// succeeds, failing this test. A subset provider with the same buffer must NOT reserve it (control).
|
||
func TestRunnerAdditiveReasoningBufferReserved(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
|
||
setup := func(t *testing.T, providerReasoning string) string {
|
||
dir := t.TempDir()
|
||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"),
|
||
"Редактируй перевод.\n---USER---\nИсходник: {{text}}\nЧерновик перевода для редактуры: {{draft}}")
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q
|
||
reasoning: %s
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
`, time.Now().UTC().Format("2006-01-02"), srv.URL, providerReasoning))
|
||
// reasoning:high on the draft + a 10M-token buffer → buffer term = 10M×$2/M = $20 ≫ the $1
|
||
// ceiling; the base reservation (tiny) fits alone. So the buffer alone decides ceiling-trip.
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), `
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||
stages:
|
||
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: high, reasoning_max_tokens: 10000000 }
|
||
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||
`)
|
||
writeFile(t, filepath.Join(dir, "source.txt"), "静かな図書館の朝。")
|
||
writeFile(t, filepath.Join(dir, "book.yaml"), `
|
||
book_id: test-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.txt
|
||
ceilings: { book_usd: 1.0, day_usd: 2.0 }
|
||
`)
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
// Additive provider: the buffer is reserved → the draft reservation trips the $1 ceiling.
|
||
r := newRunner(t, setup(t, "additive"))
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(context.Background()); err == nil || !errors.Is(err, errReserveCeiling) {
|
||
t.Fatalf("additive+reasoning-on must reserve the buffer and trip the book ceiling, got err=%v", err)
|
||
}
|
||
|
||
// Subset provider, identical buffer: reasoning ⊆ completion ≤ maxTokens, so NO buffer is
|
||
// reserved and the run proceeds (the buffer is inert for subset billing).
|
||
rSub := newRunner(t, setup(t, "subset"))
|
||
defer rSub.Close()
|
||
if _, err := rSub.TranslateBook(context.Background()); err != nil {
|
||
t.Fatalf("subset provider must NOT reserve the additive buffer (run should proceed), got err=%v", err)
|
||
}
|
||
}
|