248 lines
11 KiB
Go
248 lines
11 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/pipeline"
|
||
)
|
||
|
||
// rebill_cli_test.go closes the D20.4 «parsed but silently ignored» regression class for
|
||
// --accept-rebill: parsing the flag correctly is worthless if the value never reaches the runner. The
|
||
// invocation tests cover the parse; this one drives the REAL `translate` entry point end-to-end against
|
||
// a mock provider, so the wiring itself is executed, not asserted about.
|
||
|
||
// writeCLIFile writes a fixture file, creating parents.
|
||
func writeCLIFile(t *testing.T, path, content string) {
|
||
t.Helper()
|
||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// setupCLIProject builds the smallest runnable project (one draft stage) pointed at providerURL and
|
||
// returns the path of its book.yaml.
|
||
func setupCLIProject(t *testing.T, providerURL string) string {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
writeCLIFile(t, filepath.Join(dir, "prompts", "translator.md"),
|
||
"Переводи с {{source_lang}} на {{target_lang}}.\n---USER---\n{{text}}")
|
||
writeCLIFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
||
writeCLIFile(t, filepath.Join(dir, "pipeline.yaml"), `
|
||
core: C1
|
||
version: 1
|
||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||
retries: { regenerate_before_escalate: 0 }
|
||
context: { glossary_injection: selective, glossary_token_budget: 800 }
|
||
stages:
|
||
- { name: draft, role: translator, model: fake-model, prompt_override: prompts/translator.md, prompt_version: v-cli, temperature: 0.3, reasoning: "off" }
|
||
`)
|
||
writeCLIFile(t, filepath.Join(dir, "source.txt"), "静かな図書館の朝。")
|
||
writeCLIFile(t, filepath.Join(dir, "book.yaml"), `
|
||
book_id: cli-book
|
||
title: Тест
|
||
source_lang: ja
|
||
target_lang: ru
|
||
genre: ранобэ
|
||
audience: тест
|
||
venuti: 0.5
|
||
honorifics: keep
|
||
transcription: polivanov
|
||
footnotes: minimal
|
||
pipeline: pipeline.yaml
|
||
models: models.yaml
|
||
source_file: source.txt
|
||
ceilings: { book_usd: 1.0, day_usd: 2.0 }
|
||
`)
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
// TestTranslateWiresAcceptRebill: after a config drift, `translate --resnapshot` alone must be REFUSED
|
||
// with the projected sum, and the same call carrying the consent must go through. Dropping
|
||
// `r.AcceptRebill = acceptRebill` in main.go makes the second half fail here — which is the whole point:
|
||
// the flag exists to change what the run does, not what the parser returns.
|
||
func TestTranslateWiresAcceptRebill(t *testing.T) {
|
||
calls := 0
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
_, _ = io.ReadAll(r.Body)
|
||
calls++
|
||
tb, _ := json.Marshal("ЧЕРНОВИК ПЕРЕВОДА")
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`, tb)
|
||
}))
|
||
defer srv.Close()
|
||
bookPath := setupCLIProject(t, srv.URL)
|
||
ctx := context.Background()
|
||
|
||
if err := translate(ctx, bookPath, false, pipeline.RebillConsent{}, false, 0); err != nil {
|
||
t.Fatalf("first run: %v", err)
|
||
}
|
||
if calls != 1 {
|
||
t.Fatalf("setup: the first run must bill one call, got %d", calls)
|
||
}
|
||
|
||
// Drift the config so the stored row's snapshot is superseded.
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeCLIFile(t, pipePath, strings.Replace(string(raw), "prompt_version: v-cli", "prompt_version: v-cli-drift", 1))
|
||
|
||
// --resnapshot WITHOUT consent → refused, nothing called.
|
||
err = translate(ctx, bookPath, true, pipeline.RebillConsent{}, false, 0)
|
||
if err == nil {
|
||
t.Fatal("`translate --resnapshot` without --accept-rebill must be refused over the threshold")
|
||
}
|
||
if !strings.Contains(err.Error(), "--accept-rebill") {
|
||
t.Errorf("the refusal must name the flag; got: %v", err)
|
||
}
|
||
if calls != 1 {
|
||
t.Fatalf("a refused translate must reach no provider, calls=%d", calls)
|
||
}
|
||
|
||
// A ceiling BELOW the projection is still a refusal — proving the CAPPED value is wired, not just
|
||
// the boolean.
|
||
if err := translate(ctx, bookPath, true, pipeline.RebillConsent{Given: true, Capped: true, CapUSD: 0.0001}, false, 0); err == nil {
|
||
t.Fatal("a ceiling below the projection must be refused through the CLI entry point too")
|
||
}
|
||
if calls != 1 {
|
||
t.Fatalf("a cap-refused translate must reach no provider, calls=%d", calls)
|
||
}
|
||
|
||
// Consent given → the run proceeds and really re-pays.
|
||
if err := translate(ctx, bookPath, true, pipeline.RebillConsent{Given: true}, false, 0); err != nil {
|
||
t.Fatalf("`translate --resnapshot --accept-rebill` must proceed: %v", err)
|
||
}
|
||
if calls != 2 {
|
||
t.Fatalf("the consented run must re-call the provider, calls=%d", calls)
|
||
}
|
||
}
|
||
|
||
// TestRedriveWiresAcceptRebill is the same wiring check for the OTHER money-spending command: redrive
|
||
// carries --resnapshot too, so it carries the consent too. A redrive refused for want of consent must
|
||
// also leave the flagged row it was about to reset intact.
|
||
func TestRedriveWiresAcceptRebill(t *testing.T) {
|
||
calls := 0
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
_, _ = io.ReadAll(r.Body)
|
||
calls++
|
||
tb, _ := json.Marshal("Извините, я не могу перевести это.")
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`, tb)
|
||
}))
|
||
defer srv.Close()
|
||
bookPath := setupCLIProject(t, srv.URL)
|
||
ctx := context.Background()
|
||
|
||
// The refusal text flags the only chunk (exit 2 is a sentinel, not an infra failure).
|
||
if err := translate(ctx, bookPath, false, pipeline.RebillConsent{}, false, 0); exitCode(err) != 2 {
|
||
t.Fatalf("setup: the refusal fixture must complete with flags (exit 2), got %v", err)
|
||
}
|
||
callsAfterRun1 := calls
|
||
|
||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||
raw, err := os.ReadFile(pipePath)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeCLIFile(t, pipePath, strings.Replace(string(raw), "prompt_version: v-cli", "prompt_version: v-cli-drift", 1))
|
||
|
||
sel := pipeline.RedriveSelector{Chapter: -1, ChunkIdx: -1}
|
||
if err := redrive(ctx, bookPath, true, pipeline.RebillConsent{}, sel, false, 0); err == nil {
|
||
t.Fatal("`redrive --resnapshot` without consent must be refused over the threshold")
|
||
} else if !strings.Contains(err.Error(), "--accept-rebill") {
|
||
t.Errorf("the redrive refusal must name the flag; got: %v", err)
|
||
}
|
||
if calls != callsAfterRun1 {
|
||
t.Fatalf("a refused redrive must reach no provider, calls=%d", calls)
|
||
}
|
||
|
||
// Consent given → the redrive resets and re-attacks (the mock still refuses, so it ends flagged again
|
||
// — what matters is that it RAN and billed).
|
||
if err := redrive(ctx, bookPath, true, pipeline.RebillConsent{Given: true}, sel, false, 0); exitCode(err) != 2 {
|
||
t.Fatalf("`redrive --resnapshot --accept-rebill` must run (and end flagged), got %v", err)
|
||
}
|
||
if calls <= callsAfterRun1 {
|
||
t.Fatalf("the consented redrive must re-call the provider, calls=%d", calls)
|
||
}
|
||
}
|
||
|
||
// TestTranslateAndRedriveWireTheRunCeiling is the same class as TestTranslateWiresAcceptRebill, for the
|
||
// flag row 145 added: it exists to change what the RUN does, not what the parser returns.
|
||
//
|
||
// It closes an audit finding — deleting `r.CeilingUSD = ceilingUSD` from either command left the whole
|
||
// backend suite green, because TestParseCeilingUSD stops at the parsed invocation and the pipeline-level
|
||
// ceiling tests set the field on the Runner directly. Neither of them touches the wire between the two.
|
||
func TestTranslateAndRedriveWireTheRunCeiling(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
_, _ = io.ReadAll(r.Body)
|
||
tb, _ := json.Marshal("ЧЕРНОВИК ПЕРЕВОДА")
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`, tb)
|
||
}))
|
||
defer srv.Close()
|
||
ctx := context.Background()
|
||
|
||
// A ceiling far below anything the book can cost. The book's own ceiling is comfortable, so ONLY the
|
||
// argument can produce this stop — and only if the command actually hands it to the runner.
|
||
const tiny = 0.000001
|
||
err := translate(ctx, setupCLIProject(t, srv.URL), false, pipeline.RebillConsent{}, false, tiny)
|
||
if err == nil || !strings.Contains(err.Error(), "ceiling reached") {
|
||
t.Fatalf("translate must carry --ceiling-usd into the run; got %v", err)
|
||
}
|
||
if !strings.Contains(err.Error(), "--ceiling-usd") {
|
||
t.Fatalf("the stop must name the ceiling in force: %v", err)
|
||
}
|
||
|
||
// Same for redrive: it re-attacks with real provider calls, so an unwired flag there is a hole in
|
||
// exactly the surface the row closes. A redrive with no targets never reserves and therefore could not
|
||
// show a ceiling at all — so the book is first driven to a FLAGGED chunk, which gives the redrive
|
||
// something to re-attack and makes the reservation (and the ceiling) reachable.
|
||
refuse := true
|
||
srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
text := "ЧЕРНОВИК ПЕРЕВОДА"
|
||
if refuse && !strings.Contains(string(body), "Черновик перевода для редактуры") {
|
||
text = "Извините, я не могу перевести это." // soft refusal → the chunk is flagged
|
||
}
|
||
tb, _ := json.Marshal(text)
|
||
fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`, tb)
|
||
}))
|
||
defer srv2.Close()
|
||
bookPath := setupCLIProject(t, srv2.URL)
|
||
if err := translate(ctx, bookPath, false, pipeline.RebillConsent{}, false, 0); exitCode(err) != 2 {
|
||
t.Fatalf("setup: the run must finish WITH a flagged chunk (exit 2), got %v", err)
|
||
}
|
||
refuse = false
|
||
sel := pipeline.RedriveSelector{Chapter: -1, ChunkIdx: -1}
|
||
err = redrive(ctx, bookPath, false, pipeline.RebillConsent{}, sel, false, tiny)
|
||
if err == nil || !strings.Contains(err.Error(), "ceiling reached") {
|
||
t.Fatalf("redrive must carry --ceiling-usd into the re-attack; got %v", err)
|
||
}
|
||
}
|