262 lines
9.5 KiB
Go
262 lines
9.5 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"math"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
)
|
||
|
||
// TestStatusAndRedrive exercises the D15.3 read-model + targeted re-attack end-to-end against
|
||
// the mock provider ($0, no real LLM): a book with one flagged chapter and one ok chapter →
|
||
// status projects the honest N/M + per-chapter passports + money; redrive dry-run identifies
|
||
// the flagged chunk without touching it; a real redrive resets ONLY the flagged stages, re-runs
|
||
// with a fresh budget (ok work resumes at $0), and the previously-spent money stays committed.
|
||
func TestStatusAndRedrive(t *testing.T) {
|
||
rec := &reqRec{}
|
||
var mu sync.Mutex
|
||
refuse := true // ch1 draft refuses until flipped
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
mu.Lock()
|
||
r := refuse
|
||
mu.Unlock()
|
||
if r && strings.Contains(body, "ОТКАЗНАЯГЛАВА") && !isEditBody(body) {
|
||
return "Извините, я не могу перевести это.", "stop" // soft refusal → flagged, non-retryable
|
||
}
|
||
return draftEdit(body)
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ОТКАЗНАЯГЛАВА\fНОРМАЛЬНАЯГЛАВА", regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
// --- Run 1: ch1 flagged (soft_refusal), ch2 done. ---
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res1.Flagged != 1 {
|
||
t.Fatalf("want 1 flagged chunk, got %d", res1.Flagged)
|
||
}
|
||
callsAfterRun1 := rec.count() // draft(ch1)=refusal, draft(ch2)+edit(ch2)=ok → 3
|
||
if callsAfterRun1 != 3 {
|
||
t.Fatalf("want 3 provider calls after run1, got %d", callsAfterRun1)
|
||
}
|
||
|
||
// --- STATUS projection (read-only). ---
|
||
rep, err := r1.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.TotalChunks != 2 || rep.Done != 1 || rep.Flagged != 1 || rep.Pending != 0 || rep.InProgress != 0 {
|
||
t.Fatalf("status counts wrong: %+v", rep)
|
||
}
|
||
if math.Abs(rep.PercentDone-50) > 1e-9 {
|
||
t.Errorf("percent_done = %v, want 50", rep.PercentDone)
|
||
}
|
||
if len(rep.Chapters) != 2 {
|
||
t.Fatalf("want 2 chapter passports, got %d", len(rep.Chapters))
|
||
}
|
||
if rep.Chapters[0].Verdict != "attention" || rep.Chapters[0].WorstFlagReason != string(FlagSoftRefusal) || rep.Chapters[0].ChunksFlagged != 1 {
|
||
t.Errorf("ch1 passport = %+v (want attention/soft_refusal/1 flagged)", rep.Chapters[0])
|
||
}
|
||
if rep.Chapters[1].Verdict != "pass" || rep.Chapters[1].ChunksDone != 1 {
|
||
t.Errorf("ch2 passport = %+v (want pass/1 done)", rep.Chapters[1])
|
||
}
|
||
wantCommitted := 3 * fakeCallUSD
|
||
if math.Abs(rep.CommittedUSD-wantCommitted) > 1e-9 {
|
||
t.Errorf("committed = %v, want %v", rep.CommittedUSD, wantCommitted)
|
||
}
|
||
// Projected extrapolates the per-processed-chunk average over the whole book (here both
|
||
// chunks are processed, so it equals committed).
|
||
if math.Abs(rep.ProjectedBookUSD-wantCommitted) > 1e-9 {
|
||
t.Errorf("projected = %v, want %v", rep.ProjectedBookUSD, wantCommitted)
|
||
}
|
||
r1.Close()
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatalf("status made provider calls: %d -> %d", callsAfterRun1, rec.count())
|
||
}
|
||
|
||
// --- REDRIVE dry-run: identify the target, mutate nothing. ---
|
||
r2 := newRunner(t, bookPath)
|
||
sum, res, err := r2.Redrive(ctx, RedriveSelector{Chapter: -1, ChunkIdx: -1, Reason: string(FlagSoftRefusal), DryRun: true})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res != nil || sum.ResetRun {
|
||
t.Error("dry-run must not re-run the book")
|
||
}
|
||
if len(sum.Targets) != 1 || sum.Targets[0].Chapter != 1 || sum.Targets[0].ChunkIdx != 0 {
|
||
t.Fatalf("dry-run targets = %+v", sum.Targets)
|
||
}
|
||
// The flagged stage AND its skipped downstream are both selected for reset.
|
||
if strings.Join(sum.Targets[0].Stages, ",") != "draft,edit" {
|
||
t.Errorf("dry-run reset stages = %v, want draft,edit", sum.Targets[0].Stages)
|
||
}
|
||
if cs, _ := r2.Store.GetChunkStatus("test-book", 1, 0, "draft"); cs == nil || cs.Disposition != "flagged" {
|
||
t.Fatalf("dry-run mutated chunk_status: %+v", cs)
|
||
}
|
||
r2.Close()
|
||
if rec.count() != callsAfterRun1 {
|
||
t.Fatal("dry-run made provider calls")
|
||
}
|
||
|
||
// --- REDRIVE real: flip the provider to success, reset ch1, re-run. ---
|
||
mu.Lock()
|
||
refuse = false
|
||
mu.Unlock()
|
||
r3 := newRunner(t, bookPath)
|
||
defer r3.Close()
|
||
sum3, res3, err := r3.Redrive(ctx, RedriveSelector{Chapter: 1, ChunkIdx: 0, Reason: ""})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !sum3.ResetRun || res3 == nil {
|
||
t.Fatal("real redrive should reset and re-run")
|
||
}
|
||
if res3.Flagged != 0 {
|
||
t.Fatalf("redrive should have fixed the flag, still %d flagged", res3.Flagged)
|
||
}
|
||
// Exactly two FRESH calls (ch1 draft + ch1 edit); ch2 resumes at $0.
|
||
if got := rec.count() - callsAfterRun1; got != 2 {
|
||
t.Fatalf("redrive should make exactly 2 fresh calls (ch1 draft+edit), made %d", got)
|
||
}
|
||
if cs, _ := r3.Store.GetChunkStatus("test-book", 1, 0, "draft"); cs == nil || cs.Disposition != "ok" {
|
||
t.Fatalf("redrive did not fix ch1 draft: %+v", cs)
|
||
}
|
||
// MONEY (documented): the discarded refusal's spend STAYS committed; redrive bills the two
|
||
// new calls on top → committed reflects total lifetime spend (5 calls), ceilings stay honest.
|
||
committed, _, _ := r3.Store.SpentUSD("test-book")
|
||
if math.Abs(committed-5*fakeCallUSD) > 1e-9 {
|
||
t.Errorf("committed after redrive = %v, want %v (3 original + 2 redrive)", committed, 5*fakeCallUSD)
|
||
}
|
||
// A clean status now: everything done.
|
||
rep2, err := r3.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep2.Done != 2 || rep2.Flagged != 0 {
|
||
t.Errorf("post-redrive status: done=%d flagged=%d, want 2/0", rep2.Done, rep2.Flagged)
|
||
}
|
||
}
|
||
|
||
// TestStatusSurfacesGlossaryMissGate covers self-review finding #2: the post-check GATE flags a
|
||
// chunk at the CHUNK level (never a chunk_status row), so status must promote a gate-on chunk
|
||
// with a CONFIRMED post-check miss to flagged/glossary_miss — otherwise it reports done/pass
|
||
// while `tmctl translate` exits 2, contradicting the run.
|
||
func TestStatusSurfacesGlossaryMissGate(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||
if isEditBody(body) {
|
||
return "ОТРЕДАКТИРОВАННЫЙ.", "stop"
|
||
}
|
||
return "Некто пошёл в библиотеку.", "stop" // drops the approved name → CONFIRMED post-check miss
|
||
})
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true})
|
||
ctx := context.Background()
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
res, err := r.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.Flagged != 1 {
|
||
t.Fatalf("translate must flag the glossary_miss chunk: flagged=%d", res.Flagged)
|
||
}
|
||
rep, err := r.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.Flagged != 1 || rep.Done != 0 {
|
||
t.Fatalf("status must match translate (surface glossary_miss): done=%d flagged=%d, want 0/1", rep.Done, rep.Flagged)
|
||
}
|
||
if len(rep.Chapters) != 1 || rep.Chapters[0].WorstFlagReason != string(FlagGlossaryMiss) {
|
||
t.Fatalf("passport must show glossary_miss: %+v", rep.Chapters)
|
||
}
|
||
if rep.Chapters[0].Verdict == "pass" {
|
||
t.Error("a chapter with a glossary_miss must not be 'pass'")
|
||
}
|
||
}
|
||
|
||
// TestStatusDetectsConfigDrift covers self-review finding #3: after a wire-affecting config edit
|
||
// (a prompt_version bump) since the last run, status must flag ConfigDrift — the current config
|
||
// renders a different snapshot than the stored rows, which translate would --resnapshot (re-bill).
|
||
// Without it status shows a misleading "done/consistent".
|
||
func TestStatusDetectsConfigDrift(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||
ctx := context.Background()
|
||
|
||
r1 := newRunner(t, bookPath)
|
||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
r1.Close()
|
||
|
||
// No config change → no drift.
|
||
r2 := newRunner(t, bookPath)
|
||
rep, err := r2.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rep.ConfigDrift {
|
||
t.Fatalf("an unchanged config must not drift (current=%s)", rep.CurrentSnapshot)
|
||
}
|
||
r2.Close()
|
||
|
||
// Bump a wire-affecting field (draft prompt_version) → the current config renders a new
|
||
// snapshot → drift.
|
||
dir := filepath.Dir(bookPath)
|
||
body, err := os.ReadFile(filepath.Join(dir, "pipeline.yaml"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
changed := strings.Replace(string(body), "prompt_version: v-test", "prompt_version: v-test2", 1)
|
||
if changed == string(body) {
|
||
t.Fatal("setup: prompt_version token not found to change")
|
||
}
|
||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), changed)
|
||
|
||
r3 := newRunner(t, bookPath)
|
||
defer r3.Close()
|
||
rep3, err := r3.Status(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !rep3.ConfigDrift || rep3.CurrentSnapshot == "" {
|
||
t.Errorf("a prompt_version bump since the run must surface as ConfigDrift: %+v", rep3)
|
||
}
|
||
}
|
||
|
||
// TestRedriveNoTargets confirms a selector that matches nothing is a clean no-op (no re-run).
|
||
func TestRedriveNoTargets(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := newJSONProvider(rec, draftEdit)
|
||
defer srv.Close()
|
||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "ГЛАВАОДИН\fГЛАВАДВА"})
|
||
ctx := context.Background()
|
||
|
||
r := newRunner(t, bookPath)
|
||
defer r.Close()
|
||
if _, err := r.TranslateBook(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
calls := rec.count()
|
||
// Nothing is flagged → redrive finds no targets and never re-runs.
|
||
sum, res, err := r.Redrive(ctx, RedriveSelector{Chapter: -1, ChunkIdx: -1})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(sum.Targets) != 0 || sum.ResetRun || res != nil {
|
||
t.Fatalf("redrive with no flagged chunks must be a no-op: %+v", sum)
|
||
}
|
||
if rec.count() != calls {
|
||
t.Fatal("no-target redrive made provider calls")
|
||
}
|
||
}
|