textmachine/backend/internal/pipeline/repair_integration_test.go

543 lines
22 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package pipeline
import (
"context"
"path/filepath"
"strings"
"testing"
"os"
"textmachine/backend/internal/checks"
"textmachine/backend/internal/obs"
"textmachine/backend/internal/store"
)
// repair_integration_test.go: the repair sub-step driven end-to-end by the mock provider (pack-16). The
// properties that matter are the ones a paid run would otherwise discover: the repaired bytes are what
// SHIPS, a resume re-serves them for $0, a declined or guard-rejected repair leaves the text exactly as it
// was, and no repair ever moves a disposition.
// repairGateYAML is the gates block enabling the loop for one class in a fixture project.
func repairGateYAML(classes string) string {
return "\ngates:\n repair:\n enabled: true\n model: fake-model\n max_calls_per_unit: 2\n budget_usd: 1.0\n classes: [" + classes + "]\n"
}
// setupRepairProject builds a fixture whose SOURCE carries a 半个时辰 and whose editor renders it as
// «полчаса» — the defect class the pack-16 pair-data fix made detectable. It also lays out the pair's repair
// prompt by the real convention (`<prompts root>/<pair>/repair/<class>.md`) so the loading path under test
// is the shipping one, not a shortcut.
func setupRepairProject(t *testing.T, providerURL string) string {
t.Helper()
bookPath := setupProjectOpts(t, providerURL, projectOpts{
source: "他等了半个时辰。",
gatesYAML: repairGateYAML("dc1_fractional"),
})
dir := filepath.Dir(bookPath)
// Make the fixture a REAL zh→ru project: the book declares the pair and points at the in-repo langpack,
// so the checker spec (and with it the repair class's data contract) is compiled by the ordinary loader
// at open time — the same path production takes. Patching the runner's checkers after construction would
// bypass exactly the load-time validation these tests need to exercise.
raw, err := os.ReadFile(bookPath)
if err != nil {
t.Fatal(err)
}
packRoot, err := filepath.Abs("../../configs/langpacks")
if err != nil {
t.Fatal(err)
}
body := strings.Replace(string(raw), "source_lang: ja", "source_lang: zh\nlangpack_root: "+packRoot, 1)
writeFile(t, bookPath, body)
writeFile(t, filepath.Join(dir, "pairs", "zh-ru.yaml"), "pair: zh-ru\nprompts_root: ../prompts\n")
writeFile(t, filepath.Join(dir, "prompts", "zh-ru", "repair", "dc1_fractional.md"),
"Исправь единицу времени во фрагменте.\n---USER---\nИсходник: {{text}}\nФрагмент: {{draft}}")
return bookPath
}
// newRepairRunner opens the fixture runner and compiles the REAL zh-ru checker spec onto it: the fixture
// book declares no langpack, and the detector is only meaningful for a pair that ships checker data.
// newRepairRunner opens the fixture runner. The project declares its pair and langpack, so the checker spec
// arrives through the ordinary loader — no post-construction patching.
func newRepairRunner(t *testing.T, bookPath string) *Runner {
t.Helper()
return newRunner(t, bookPath)
}
// isRepairBody reports whether a mock request is the repair call (its user message carries the fragment
// label the repair prompt writes, which neither the draft nor the edit prompt does).
func isRepairBody(body string) bool { return strings.Contains(body, "Фрагмент:") }
func TestRepairAppliesAndShipsRepairedBytes(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
switch {
case isRepairBody(body):
return "Он прождал час и вошёл внутрь.", "stop"
case isEditBody(body):
return "Он прождал полчаса и вошёл внутрь.", "stop"
}
return "ЧЕРНОВИК ПЕРЕВОДА.", "stop"
})
defer srv.Close()
bookPath := setupRepairProject(t, srv.URL)
r := newRepairRunner(t, bookPath)
defer r.Close()
res, err := r.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
if res.Flagged != 0 {
t.Fatalf("a repair must never flag the unit, got flagged=%d", res.Flagged)
}
if len(res.Chunks) != 1 {
t.Fatalf("want one output unit, got %d", len(res.Chunks))
}
final := res.Chunks[0].FinalText
if strings.Contains(final, "полчаса") || !strings.Contains(final, "час") {
t.Fatalf("the SHIPPED text must carry the repair, got %q", final)
}
// The export contract reads final_hash → checkpoint, so the repaired bytes must live in a derived
// checkpoint of the repair namespace, not in the editor's own attempt.
cs, err := r.Store.GetChunkStatus("test-book", 1, 0, "edit")
if err != nil || cs == nil {
t.Fatalf("no edit chunk_status: %v", err)
}
if cs.Disposition != string(DispOK) {
t.Fatalf("disposition moved: %s/%s", cs.Disposition, cs.FlagReason)
}
if !strings.HasPrefix(cs.FinalHash, repairDerivedNS+":") {
t.Fatalf("final_hash must point at the repair export, got %q", cs.FinalHash)
}
cp, err := r.Store.GetCheckpoint(cs.FinalHash)
if err != nil || cp == nil {
t.Fatalf("derived repair checkpoint missing: %v", err)
}
if strings.Contains(cp.ResponseText, "полчаса") {
t.Fatalf("the derived checkpoint still holds the defect: %q", cp.ResponseText)
}
// The spend is attributable WITHOUT a migration: the repair call carries its own role.
spent, err := r.Store.RepairSpentUSD("test-book")
if err != nil {
t.Fatal(err)
}
if spent <= 0 {
t.Fatalf("repair spend must be queryable by role, got %.6f", spent)
}
// And the export projection ships exactly those bytes.
exp, err := r.Export(false)
if err != nil {
t.Fatal(err)
}
if len(exp.Chunks) != 1 || strings.Contains(exp.Chunks[0].FinalText, "полчаса") {
t.Fatalf("export must ship the repaired text, got %+v", exp.Chunks)
}
}
// A resumed run must re-serve the repaired bytes for $0 and make no fresh call: the repair is not re-run,
// it is read back from the derived checkpoint through chunk_status.
func TestRepairResumeIsFreeAndIdentical(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
switch {
case isRepairBody(body):
return "Он прождал час и вошёл внутрь.", "stop"
case isEditBody(body):
return "Он прождал полчаса и вошёл внутрь.", "stop"
}
return "ЧЕРНОВИК ПЕРЕВОДА.", "stop"
})
defer srv.Close()
bookPath := setupRepairProject(t, srv.URL)
r1 := newRepairRunner(t, bookPath)
res1, err := r1.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
r1.Close()
callsAfterFirst := rec.count()
r2 := newRepairRunner(t, bookPath)
defer r2.Close()
res2, err := r2.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
if rec.count() != callsAfterFirst {
t.Fatalf("a resumed run made %d fresh provider calls; a resolved unit must resume at $0", rec.count()-callsAfterFirst)
}
if res2.TotalUSD != 0 {
t.Fatalf("resume must cost $0, got %.6f", res2.TotalUSD)
}
if res1.Chunks[0].FinalText != res2.Chunks[0].FinalText {
t.Fatalf("resume shipped different bytes:\n first: %q\nsecond: %q", res1.Chunks[0].FinalText, res2.Chunks[0].FinalText)
}
}
// ⟦TM-NOCHANGE⟧ is a first-class answer: the flagged text is kept EXACTLY as the editor produced it, no
// derived checkpoint is written, and the unit stays ok. This is the protocol that keeps a precision-limited
// flagger from becoming a text-corrupting actuator.
func TestRepairDeclineKeepsOriginal(t *testing.T) {
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
switch {
case isRepairBody(body):
return repairNoChange, "stop"
case isEditBody(body):
return "Он прождал полчаса и вошёл внутрь.", "stop"
}
return "ЧЕРНОВИК ПЕРЕВОДА.", "stop"
})
defer srv.Close()
bookPath := setupRepairProject(t, srv.URL)
r := newRepairRunner(t, bookPath)
defer r.Close()
res, err := r.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(res.Chunks[0].FinalText, "полчаса") {
t.Fatalf("a declined repair must leave the text untouched, got %q", res.Chunks[0].FinalText)
}
cs, _ := r.Store.GetChunkStatus("test-book", 1, 0, "edit")
if cs == nil || strings.HasPrefix(cs.FinalHash, repairDerivedNS+":") {
t.Fatalf("a declined repair must not write a derived export: %+v", cs)
}
if cs.Disposition != string(DispOK) {
t.Fatalf("a declined repair must not move the disposition: %s", cs.Disposition)
}
}
// The guard that matters most: every shipped checker goes SILENT when the defective phrase is simply
// deleted, so a reply that guts the span would pass a naive "the class stopped firing" re-gate. It must be
// refused on the reply's own shape, and the original text must survive.
func TestRepairRejectsDeletingReply(t *testing.T) {
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
switch {
case isRepairBody(body):
return "Он вошёл.", "stop" // the duration is gone entirely — a deletion, not a repair
case isEditBody(body):
return "Он прождал полчаса и вошёл внутрь.", "stop"
}
return "ЧЕРНОВИК ПЕРЕВОДА.", "stop"
})
defer srv.Close()
bookPath := setupRepairProject(t, srv.URL)
r := newRepairRunner(t, bookPath)
defer r.Close()
res, err := r.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(res.Chunks[0].FinalText, "полчаса") {
t.Fatalf("a deleting reply must be rejected and the original kept, got %q", res.Chunks[0].FinalText)
}
cs, _ := r.Store.GetChunkStatus("test-book", 1, 0, "edit")
if cs == nil || strings.HasPrefix(cs.FinalHash, repairDerivedNS+":") {
t.Fatalf("a rejected repair must not write a derived export: %+v", cs)
}
}
// A reply that smuggles source-script glyphs into the target prose is refused: a few CJK runes inside one
// sentence sit far below the intrinsic echo threshold, so nothing downstream would catch them.
func TestRepairRejectsScriptLeakReply(t *testing.T) {
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
switch {
case isRepairBody(body):
return "Он прождал 半个时辰 и вошёл внутрь.", "stop"
case isEditBody(body):
return "Он прождал полчаса и вошёл внутрь.", "stop"
}
return "ЧЕРНОВИК ПЕРЕВОДА.", "stop"
})
defer srv.Close()
bookPath := setupRepairProject(t, srv.URL)
r := newRepairRunner(t, bookPath)
defer r.Close()
res, err := r.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(res.Chunks[0].FinalText, "полчаса") {
t.Fatalf("a script-leaking reply must be rejected, got %q", res.Chunks[0].FinalText)
}
}
// The repair model is the THIRD model axis: it must be pre-built like the stage models, or the wave would
// hit a loud "no pre-built client" error mid-run.
func TestRepairModelIsInReachableSet(t *testing.T) {
srv := newJSONProvider(&reqRec{}, func(string) (string, string) { return "x", "stop" })
defer srv.Close()
bookPath := setupRepairProject(t, srv.URL)
r := newRepairRunner(t, bookPath)
defer r.Close()
found := false
for _, m := range r.reachableModels() {
if m == r.Pipeline.Gates.Repair.Model {
found = true
}
}
if !found {
t.Fatalf("the repair model must be in the eager-built set, got %v", r.reachableModels())
}
}
// An unknown class name is a LOUD load failure (the runner owns the class vocabulary; config cannot import
// it without a cycle), and so is a missing class prompt.
func TestRepairLoadFailsLoudOnUnknownClassAndMissingPrompt(t *testing.T) {
srv := newJSONProvider(&reqRec{}, func(string) (string, string) { return "x", "stop" })
defer srv.Close()
t.Run("unknown class", func(t *testing.T) {
bookPath := setupProjectOpts(t, srv.URL, projectOpts{gatesYAML: repairGateYAML("dc1_typo")})
dir := filepath.Dir(bookPath)
writeFile(t, filepath.Join(dir, "pairs", "ja-ru.yaml"), "pair: ja-ru\nprompts_root: ../prompts\n")
if _, err := NewRunner(bookPath, obs.NewLogger()); err == nil {
t.Fatal("an unknown repair class must fail loud at load")
} else if !strings.Contains(err.Error(), "dc1_typo") {
t.Fatalf("the error must name the offending class, got: %v", err)
}
})
t.Run("missing prompt", func(t *testing.T) {
// latin_residue needs no pair data, so the MISSING PROMPT is the first thing that can fail here.
bookPath := setupProjectOpts(t, srv.URL, projectOpts{gatesYAML: repairGateYAML("latin_residue")})
dir := filepath.Dir(bookPath)
writeFile(t, filepath.Join(dir, "pairs", "ja-ru.yaml"), "pair: ja-ru\nprompts_root: ../prompts\n")
_, err := NewRunner(bookPath, obs.NewLogger())
if err == nil {
t.Fatal("a missing repair prompt must fail loud at load")
}
if !strings.Contains(err.Error(), "latin_residue.md") {
t.Fatalf("the error must name the expected path, got: %v", err)
}
})
}
// Enabling the gate must move the snapshot of the wave that owns the SHIPPING stage — and ONLY that wave,
// so the paid draft checkpoints stay valid and the draft resumes at $0.
func TestRepairFoldsOnlyIntoFinalStageWaveSnapshot(t *testing.T) {
srv := newJSONProvider(&reqRec{}, func(string) (string, string) { return "x", "stop" })
defer srv.Close()
r := newRepairRunner(t, setupRepairProject(t, srv.URL))
defer r.Close()
draftOn, _, err := r.snapshotIDForWave(waveDraft)
if err != nil {
t.Fatal(err)
}
editOn, _, err := r.snapshotIDForWave(waveEdit)
if err != nil {
t.Fatal(err)
}
// SAME book, SAME pack, SAME stages — only the gate flips, which isolates the fold from every other input.
r.Pipeline.Gates.Repair.Enabled = false
draftOff, _, err := r.snapshotIDForWave(waveDraft)
if err != nil {
t.Fatal(err)
}
editOff, _, err := r.snapshotIDForWave(waveEdit)
if err != nil {
t.Fatal(err)
}
if draftOff != draftOn {
t.Fatalf("flipping the repair gate moved the DRAFT-wave snapshot — the paid draft wave would be re-billed:\n off %s\n on %s", draftOff, draftOn)
}
if editOff == editOn {
t.Fatal("flipping the repair gate must move the EDIT-wave snapshot (it changes the resolved shipped bytes)")
}
}
// TestRepairReplyGuards pins the acceptance rules directly, without a provider: these are what stand between
// a plausible-looking reply and corrupted shipped prose, and every shipped checker is blind to the damage
// (they all go silent when the defective text is simply deleted).
func TestRepairReplyGuards(t *testing.T) {
const orig = "Он прождал полчаса и заплатил 500 монет."
c := compileTestCheckers(t)
ok := func(text string) stageAttempt {
return stageAttempt{text: text, cls: classification{Reason: reasonOK}}
}
cases := []struct {
name string
att stageAttempt
want repairVerdict
}{
{"clean repair", ok("Он прождал час и заплатил 500 монет."), repairAccepted},
{"sentinel decline", ok(repairNoChange), repairDeclined},
{"sentinel mixed with prose", ok("Тут всё верно " + repairNoChange), repairBadShape},
{"empty", ok(" "), repairBadShape},
{"classifier rejected", stageAttempt{text: "x", cls: classification{Reason: FlagSoftRefusal}}, repairBadVerdict},
{"deletes the span", ok("Он ушёл."), repairBadShape},
{"drops a number", ok("Он прождал час и заплатил монет."), repairBadContent},
{"invents a number", ok("Он прождал час и заплатил 500 монет за 7 дней."), repairBadContent},
{"leaks source script", ok("Он прождал 半时辰 и заплатил 500 монет."), repairBadContent},
{"re-paragraphs", ok("Он прождал час\nи заплатил 500 монет."), repairBadShape},
{"rewrites far beyond the span", ok(strings.Repeat("Он прождал час и заплатил 500 монет. ", 3)), repairBadShape},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if _, got := repairReplyVerdict(tc.att, orig, checks.RepairDC1Fractional, c); got != tc.want {
t.Fatalf("verdict = %q, want %q", got, tc.want)
}
})
}
}
// A number may CHANGE (that is what a unit/scale repair does) but may not vanish or appear: exactly one
// substitution is legitimate.
func TestRepairDigitGuardAllowsOneSubstitution(t *testing.T) {
c := compileTestCheckers(t)
att := stageAttempt{text: "Он ждал 6 часов.", cls: classification{Reason: reasonOK}}
if _, v := repairReplyVerdict(att, "Он ждал 3 часа.", checks.RepairDC1TimeUnits, c); v != repairAccepted {
t.Fatalf("a single numeric substitution must be accepted, got %q", v)
}
if _, v := repairReplyVerdict(att, "Он ждал 3 часа и 40 минут.", checks.RepairDC1TimeUnits, c); v == repairAccepted {
t.Fatal("dropping the second figure must NOT be accepted")
}
}
// The case the ratified POSITIVE re-gate exists for, and that a negative-only re-gate cannot catch: the reply
// changes the wrong number to a DIFFERENT wrong number. The detector goes silent, every content guard passes.
func TestRepairRejectsDifferentWrongValue(t *testing.T) {
c := compileTestCheckers(t)
att := stageAttempt{text: "Он затворился на пять часов.", cls: classification{Reason: reasonOK}}
if _, v := repairReplyVerdict(att, "Он затворился на три часа.", checks.RepairDC1TimeUnits, c); v == repairAccepted {
t.Fatal("«три часа» → «пять часов» is a different WRONG value and must be refused")
}
good := stageAttempt{text: "Он затворился на шесть часов.", cls: classification{Reason: reasonOK}}
if _, v := repairReplyVerdict(good, "Он затворился на три часа.", checks.RepairDC1TimeUnits, c); v != repairAccepted {
t.Fatalf("the CORRECT doubling must be accepted, got %q", v)
}
}
// The fractional class must CONVERT the duration, not delete it.
func TestRepairFractionalMustStateADuration(t *testing.T) {
c := compileTestCheckers(t)
deleted := stageAttempt{text: "Он прождал и вошёл внутрь.", cls: classification{Reason: reasonOK}}
if _, v := repairReplyVerdict(deleted, "Он прождал полчаса и вошёл внутрь.", checks.RepairDC1Fractional, c); v == repairAccepted {
t.Fatal("dropping the duration must be refused")
}
fixed := stageAttempt{text: "Он прождал час и вошёл внутрь.", cls: classification{Reason: reasonOK}}
if _, v := repairReplyVerdict(fixed, "Он прождал полчаса и вошёл внутрь.", checks.RepairDC1Fractional, c); v != repairAccepted {
t.Fatalf("the correct conversion must be accepted, got %q", v)
}
}
// An already-PAID repair must replay even when the book budget is exhausted, or a crash between the paid call
// and the chunk_status write would make the resumed run ship different bytes than the run that paid.
func TestRepairPaidCallReplaysUnderExhaustedBudget(t *testing.T) {
rec := &reqRec{}
srv := newJSONProvider(rec, func(body string) (string, string) {
switch {
case isRepairBody(body):
return "Он прождал час и вошёл внутрь.", "stop"
case isEditBody(body):
return "Он прождал полчаса и вошёл внутрь.", "stop"
}
return "ЧЕРНОВИК ПЕРЕВОДА.", "stop"
})
defer srv.Close()
bookPath := setupRepairProject(t, srv.URL)
r1 := newRepairRunner(t, bookPath)
res1, err := r1.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
r1.Close()
if strings.Contains(res1.Chunks[0].FinalText, "полчаса") {
t.Fatalf("premise broken: the first run must have repaired, got %q", res1.Chunks[0].FinalText)
}
r2 := newRepairRunner(t, bookPath)
defer r2.Close()
r2.Pipeline.Gates.Repair.BudgetUSD = 0.000001 // below what run 1 already spent
if err := r2.Store.UpsertChunkStatus(store.ChunkStatus{
BookID: "test-book", Chapter: 1, ChunkIdx: 0, Stage: "edit",
SnapshotID: "stale", Disposition: string(DispSkipped),
}); err != nil {
t.Fatal(err)
}
before := rec.count()
res2, err := r2.TranslateBook(context.Background())
if err != nil {
t.Fatal(err)
}
if strings.Contains(res2.Chunks[0].FinalText, "полчаса") {
t.Fatalf("an already-paid repair must replay under an exhausted budget; shipped %q", res2.Chunks[0].FinalText)
}
if rec.count() != before {
t.Fatalf("the replay must be $0: %d fresh calls", rec.count()-before)
}
}
// The loop's counters are DERIVED from durable artifacts and therefore survive a resume — the reason no
// counter column was added.
func TestRepairCountersDerivedAndSurviveResume(t *testing.T) {
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
switch {
case isRepairBody(body):
return "Он прождал час и вошёл внутрь.", "stop"
case isEditBody(body):
return "Он прождал полчаса и вошёл внутрь.", "stop"
}
return "ЧЕРНОВИК ПЕРЕВОДА.", "stop"
})
defer srv.Close()
bookPath := setupRepairProject(t, srv.URL)
r1 := newRepairRunner(t, bookPath)
if _, err := r1.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
q1, err := r1.QualityReport()
if err != nil {
t.Fatal(err)
}
r1.Close()
if q1.RepairCalls != 1 || q1.RepairApplied != 1 || q1.RepairDeclined != 0 || q1.RepairRejected != 0 {
t.Fatalf("counters after the paid run = calls %d/applied %d/declined %d/rejected %d, want 1/1/0/0",
q1.RepairCalls, q1.RepairApplied, q1.RepairDeclined, q1.RepairRejected)
}
r2 := newRepairRunner(t, bookPath)
defer r2.Close()
if _, err := r2.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
q2, err := r2.QualityReport()
if err != nil {
t.Fatal(err)
}
if q2.RepairCalls != q1.RepairCalls || q2.RepairApplied != q1.RepairApplied {
t.Fatalf("a resumed run lost the counters: %d/%d → %d/%d",
q1.RepairCalls, q1.RepairApplied, q2.RepairCalls, q2.RepairApplied)
}
}
// A DECLINE is the loop's own precision measurement and must be counted as such, not as a failure.
func TestRepairDeclineIsCountedSeparately(t *testing.T) {
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
switch {
case isRepairBody(body):
return repairNoChange, "stop"
case isEditBody(body):
return "Он прождал полчаса и вошёл внутрь.", "stop"
}
return "ЧЕРНОВИК ПЕРЕВОДА.", "stop"
})
defer srv.Close()
r := newRepairRunner(t, setupRepairProject(t, srv.URL))
defer r.Close()
if _, err := r.TranslateBook(context.Background()); err != nil {
t.Fatal(err)
}
q, err := r.QualityReport()
if err != nil {
t.Fatal(err)
}
if q.RepairCalls != 1 || q.RepairDeclined != 1 || q.RepairApplied != 0 || q.RepairRejected != 0 {
t.Fatalf("a decline must count as declined, got calls %d/applied %d/declined %d/rejected %d",
q.RepairCalls, q.RepairApplied, q.RepairDeclined, q.RepairRejected)
}
}