package pipeline import ( "context" "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "path/filepath" "sync" "sync/atomic" "testing" "textmachine/backend/internal/obs" ) // cutoververdict_test.go asks ONE question, and it is a question rather than a fix: when a run is cut // over a position that ALREADY HAD a verdict, what is left behind? // // The suspicion (backlog row 375, the ⛔ half) is that the mark a cut writes REPLACES that verdict — // `chunk_status` is keyed (book, chapter, chunk, stage) and the write is an upsert — and that it writes // an EMPTY final_hash, so the position loses the pointer to text whose checkpoint is still on disk. The // book would then be WORSE after an interrupted run than it was before it started, which is exactly what // the ratified invariant (D39.240) forbids: a stop may cost money, and may not leave rubbish. // // ⛔ THE STATE IS REACHED THROUGH PRODUCTION PATHS ONLY. Nothing here writes a chunk_status row by hand: // the first verdict is written by a real run, the re-attack is caused by a real prompt edit plus // --resnapshot, and the cut is a real cancellation over a real request the server is holding open. A // fixture that assembled the state directly would prove the upsert overwrites — which nobody doubts — // while saying nothing about whether a run can GET there. // // ⚠ ONE OF THE THREE ROUTES THE ROW NAMES IS ALREADY OUT, and it was eliminated by reading rather than // here: a REDRIVE cannot produce this state, because ResetChunkStages deletes the chunk_status row AND // its checkpoints in one transaction before anything re-attacks the position (store/chunkstatus.go), so // there is no verdict left to overwrite. What remains, and what this file drives, is the re-pin. // holdWhenArmed answers like the ordinary fake provider, except that while `armed` is set it ANNOUNCES // the request's arrival and then holds it until `release` is closed. One server serves both runs, so the // fixture never has to re-point a book at a second URL mid-way. // // ⛔ THE ARRIVAL SIGNAL IS WHAT MAKES THE TEST ABOUT ITS SUBJECT, and the first version of this file had // none. Cancelling as soon as the money gate reports a call «in flight» cancels somewhere between the // reservation and the request actually being WRITTEN — and a request that never went out is not a // delivered cut, so the engine released the reservation, wrote no mark, and the assertion below read // «the row is untouched» as if the suspicion had been refuted. It had not been tested. Announcing from // inside the handler makes delivery a fact: the bytes are in the server's hands before anything cancels. func holdWhenArmed(rec *reqRec, armed *atomic.Bool, arrived chan<- struct{}, release <-chan struct{}) *httptest.Server { var once sync.Once return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) rec.record(string(body)) if armed.Load() { once.Do(func() { close(arrived) }) // Deliberately NOT selecting on the request's own context: the subject is what the ENGINE // records when its call is cut, so the server must not answer early just because somebody // cancelled it (the same rule gatedProvider states next door). <-release } text, finish := draftEdit(string(body)) tb, _ := json.Marshal(text) fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":%q}], "usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`, tb, finish) })) } func TestACutOverAPositionThatAlreadyShippedTextKeepsIt(t *testing.T) { rec := &reqRec{} var armed atomic.Bool release := make(chan struct{}) arrived := make(chan struct{}) srv := holdWhenArmed(rec, &armed, arrived, release) // ⚠ THE ORDER OF THESE TWO DEFERS IS THE TEST, NOT ITS TIDINESS. Defers run LIFO, and // httptest.Server.Close() BLOCKS until every outstanding handler returns — so registering the server's // close last parks it on a handler waiting for a channel closed by the defer behind it, and the test // hangs until the package timeout with no failure to read. Measured here: 300 s and a kill. defer srv.Close() defer close(release) // A draft-only pipeline, one chunk: the draft IS the shipping row, so the position under test and the // row the export reads are the same row — no edit wave between the subject and the assertion. bookPath := setupProjectOpts(t, srv.URL, projectOpts{draftOnly: true, bookUSD: 100, rebillConsentUSD: 1}) // --- run 1: an ordinary run. It leaves a verdict and shippable text. --- r1 := newRunner(t, bookPath) if _, err := r1.TranslateBook(tracedCtx()); err != nil { r1.Close() t.Fatalf("the first run must finish: %v", err) } before, err := r1.Export(false) if err != nil { r1.Close() t.Fatal(err) } if len(before.Chunks) != 1 || before.Chunks[0].FinalText == "" { r1.Close() t.Fatalf("the first run shipped nothing — there is no text for the second run to endanger (%d position(s))", len(before.Chunks)) } shipped := before.Chunks[0].FinalText cs, err := r1.Store.GetChunkStatus(r1.Book.BookID, 1, 0, "draft") if err != nil || cs == nil { r1.Close() t.Fatalf("no verdict after the first run: %v", err) } oldHash, oldDisposition, oldCost := cs.FinalHash, cs.Disposition, cs.CostUSD if oldHash == "" { r1.Close() t.Fatal("the first run left no final_hash, so there is no pointer for a cut to erase") } r1.Close() t.Logf("after run 1: disposition=%q final_hash=%.12s cost_usd=%.6f text=%q", oldDisposition, oldHash, oldCost, shipped) // --- the prompt moves. Both guards that would have served the stored row now refuse it: the rendered // messages change (content_hash) and so does the snapshot, and repinnable answers false for a move // that is not bank-only. This is the re-pin route, driven by an ordinary file edit. --- writeFile(t, filepath.Join(filepath.Dir(bookPath), "prompts", "translator.md"), "Переводи с {{source_lang}} на {{target_lang}}. Второй вариант промта.\n---USER---\n{{text}}") // --- run 2: re-attacks the position, and is cut while its call is out. --- armed.Store(true) r2 := newRunner(t, bookPath) defer r2.Close() r2.Resnapshot = true ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})) go func() { // Cancel only once the PROVIDER HAS THE REQUEST. The money gate's counter says a reservation was // taken, which is a moment earlier and a different fact: a cancellation in that window kills a // request that never went out, the engine owes nothing and marks nothing, and this test would then // read «the row is untouched» as proof of a guard that never ran. Measured: the first version of // this fixture did exactly that. <-arrived cancel() }() _, err = r2.TranslateBook(ctx) if rec.count() < 2 { t.Fatalf("the second run made no provider call of its own (%d call(s) in total) — the position was served from the store and nothing was cut", rec.count()) } if err == nil { t.Fatal("a cancelled run must not report success") } // --- what is left behind: the verdict this run did not produce, exactly as it was --- after, err := r2.Store.GetChunkStatus(r2.Book.BookID, 1, 0, "draft") if err != nil || after == nil { t.Fatalf("the position lost its row entirely: %v", err) } t.Logf("after the cut: disposition=%q flag_reason=%q final_hash=%.12s cost_usd=%.6f", after.Disposition, after.FlagReason, after.FinalHash, after.CostUSD) if FlagReason(after.FlagReason) == FlagCancelled { t.Fatal("the cut REPLACED a verdict it did not produce: the position now reads `cancelled`, and the text the earlier run bought is unreachable") } if after.FinalHash != oldHash { t.Fatalf("the pointer to the earlier run's text moved (%.12s → %q) — the checkpoint is still on disk with nothing addressing it", oldHash, after.FinalHash) } if after.CostUSD != oldCost { t.Fatalf("the position's recorded spend was rewritten by the cut: $%.6f → $%.6f", oldCost, after.CostUSD) } // The text really is still reachable, end to end — not merely a hash that looks right. cp, err := r2.Store.GetCheckpoint(after.FinalHash) if err != nil { t.Fatal(err) } if cp == nil || cp.ResponseText == "" { t.Fatal("the row points at a checkpoint that is gone or empty") } afterExp, err := r2.Export(false) if err != nil { t.Fatal(err) } if len(afterExp.Chunks) != 1 { t.Fatalf("export projected %d positions, want 1", len(afterExp.Chunks)) } t.Logf("export after the cut: disposition=%q final_text=%q", afterExp.Chunks[0].Disposition, afterExp.Chunks[0].FinalText) if afterExp.Chunks[0].FinalText != shipped { t.Fatalf("the reader lost a finished chapter to an interrupted run: shipped %q before, %q after", shipped, afterExp.Chunks[0].FinalText) } } // TestACutOverAVirginPositionStillMarksIt is the CONTROL, and without it the test above is satisfied by // deleting the mark altogether. // // The mark exists for the position that has NO verdict: a run cut over its first attempt would otherwise // leave the money booked and the position reading «never started», which is the invisible hole // recordStoppedPosition was written for. The guard added for the row above must not have eaten it. func TestACutOverAVirginPositionStillMarksIt(t *testing.T) { rec := &reqRec{} var armed atomic.Bool armed.Store(true) // hold from the very first call: this position has never been translated release := make(chan struct{}) arrived := make(chan struct{}) srv := holdWhenArmed(rec, &armed, arrived, release) defer srv.Close() defer close(release) bookPath := setupProjectOpts(t, srv.URL, projectOpts{draftOnly: true, bookUSD: 100}) r := newRunner(t, bookPath) defer r.Close() ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})) go func() { <-arrived cancel() }() if _, err := r.TranslateBook(ctx); err == nil { t.Fatal("a cancelled run must not report success") } cs, err := r.Store.GetChunkStatus(r.Book.BookID, 1, 0, "draft") if err != nil { t.Fatal(err) } if cs == nil { t.Fatal("the stopped position has NO row at all: its money is on the ledger and the export will read it as never started — the invisible hole the mark exists to close") } t.Logf("virgin position after the cut: disposition=%q flag_reason=%q attempts=%d", cs.Disposition, cs.FlagReason, cs.Attempts) if FlagReason(cs.FlagReason) != FlagCancelled { t.Fatalf("a stopped virgin position must be marked `cancelled`, got %q", cs.FlagReason) } } // TestAStoppedPositionIsNotRecordedAsFailed is backlog row 389's second half, at the site a stop actually // travels through: a call a person CUT was healthy when they cut it, and the resume re-does it at the // same budget. `failed` is the read-model's word for a position that broke, and a reader who finds it // there goes looking for a defect that is not there. // // ⚠ THE CONTROL IS IN THE SAME TEST, and without it this pin is satisfied by never writing the word at // all: a socket that dies on its own IS a failure and must keep it. Both halves run over the same // fixture, so the difference asserted is the CAUSE and nothing else. func TestAStoppedPositionIsNotRecordedAsFailed(t *testing.T) { rec := &reqRec{} var armed atomic.Bool armed.Store(true) release := make(chan struct{}) arrived := make(chan struct{}) srv := holdWhenArmed(rec, &armed, arrived, release) defer srv.Close() defer close(release) bookPath := setupProjectOpts(t, srv.URL, projectOpts{draftOnly: true, bookUSD: 100}) r := newRunner(t, bookPath) defer r.Close() ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})) go func() { <-arrived cancel() }() if _, err := r.TranslateBook(ctx); err == nil { t.Fatal("a cancelled run must not report success") } job, err := r.Store.EnsureJob(r.Book.BookID, 1, "draft", "") if err != nil { t.Fatal(err) } if job == nil { t.Fatal("the position has no job row — this pin would then be asserting about nothing") } t.Logf("job status after the human's cut: %q", job.Status) if job.Status == "failed" { t.Fatal("the position a person stopped is recorded as a FAILURE; the resume re-does it and nothing broke") } } // TestABrokenSocketIsStillRecordedAsFailed is the control for the pin above: the word must still be // written where it is true. Same shape, different cause — the server hangs up mid-request instead of the // operator stopping the run. func TestABrokenSocketIsStillRecordedAsFailed(t *testing.T) { rec := &reqRec{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) rec.record(string(body)) // Hijack and close: the request was delivered, the answer never arrives, and the socket dies — // llm.CutByConnection rather than a cancellation. hj, ok := w.(http.Hijacker) if !ok { t.Error("the test server cannot hijack; this control cannot build its cause") return } conn, _, err := hj.Hijack() if err != nil { t.Errorf("hijack: %v", err) return } conn.Close() })) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{draftOnly: true, bookUSD: 100}) r := newRunner(t, bookPath) defer r.Close() if _, err := r.TranslateBook(tracedCtx()); err == nil { t.Fatal("a run against a provider that hangs up must fail") } job, err := r.Store.EnsureJob(r.Book.BookID, 1, "draft", "") if err != nil { t.Fatal(err) } if job == nil { t.Fatal("no job row") } t.Logf("job status after a broken socket: %q", job.Status) if job.Status != "failed" { t.Fatalf("a position whose provider died must keep the word for it, got %q — the pin above would then be green over a word nobody writes any more", job.Status) } }