textmachine/backend/internal/pipeline/cutoververdict_test.go

package pipeline

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"

	"textmachine/backend/internal/obs"
)

// cutoververdict_test.go asks ONE question, and it is a question rather than a fix: when a run is cut
// over a position that ALREADY HAD a verdict, what is left behind?
//
// The suspicion (backlog row 375, the ⛔ half) is that the mark a cut writes REPLACES that verdict —
// `chunk_status` is keyed (book, chapter, chunk, stage) and the write is an upsert — and that it writes
// an EMPTY final_hash, so the position loses the pointer to text whose checkpoint is still on disk. The
// book would then be WORSE after an interrupted run than it was before it started, which is exactly what
// the ratified invariant (D39.240) forbids: a stop may cost money, and may not leave rubbish.
//
// ⛔ THE STATE IS REACHED THROUGH PRODUCTION PATHS ONLY. Nothing here writes a chunk_status row by hand:
// the first verdict is written by a real run, the re-attack is caused by a real prompt edit plus
// --resnapshot, and the cut is a real cancellation over a real request the server is holding open. A
// fixture that assembled the state directly would prove the upsert overwrites — which nobody doubts —
// while saying nothing about whether a run can GET there.
//
// ⚠ ONE OF THE THREE ROUTES THE ROW NAMES IS ALREADY OUT, and it was eliminated by reading rather than
// here: a REDRIVE cannot produce this state, because ResetChunkStages deletes the chunk_status row AND
// its checkpoints in one transaction before anything re-attacks the position (store/chunkstatus.go), so
// there is no verdict left to overwrite. What remains, and what this file drives, is the re-pin.

// holdWhenArmed answers like the ordinary fake provider, except that while `armed` is set it ANNOUNCES
// the request's arrival and then holds it until `release` is closed. One server serves both runs, so the
// fixture never has to re-point a book at a second URL mid-way.
//
// ⛔ THE ARRIVAL SIGNAL IS WHAT MAKES THE TEST ABOUT ITS SUBJECT, and the first version of this file had
// none. Cancelling as soon as the money gate reports a call «in flight» cancels somewhere between the
// reservation and the request actually being WRITTEN — and a request that never went out is not a
// delivered cut, so the engine released the reservation, wrote no mark, and the assertion below read
// «the row is untouched» as if the suspicion had been refuted. It had not been tested. Announcing from
// inside the handler makes delivery a fact: the bytes are in the server's hands before anything cancels.
func holdWhenArmed(rec *reqRec, armed *atomic.Bool, arrived chan<- struct{}, release <-chan struct{}) *httptest.Server {
	var once sync.Once
	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		body, _ := io.ReadAll(r.Body)
		rec.record(string(body))
		if armed.Load() {
			once.Do(func() { close(arrived) })
			// Deliberately NOT selecting on the request's own context: the subject is what the ENGINE
			// records when its call is cut, so the server must not answer early just because somebody
			// cancelled it (the same rule gatedProvider states next door).
			<-release
		}
		text, finish := draftEdit(string(body))
		tb, _ := json.Marshal(text)
		fmt.Fprintf(w, `{"id":"fake","model":"fake-model","choices":[{"message":{"content":%s},"finish_reason":%q}],
			"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`,
			tb, finish)
	}))
}

func TestACutOverAPositionThatAlreadyShippedTextKeepsIt(t *testing.T) {
	rec := &reqRec{}
	var armed atomic.Bool
	release := make(chan struct{})
	arrived := make(chan struct{})
	srv := holdWhenArmed(rec, &armed, arrived, release)
	// ⚠ THE ORDER OF THESE TWO DEFERS IS THE TEST, NOT ITS TIDINESS. Defers run LIFO, and
	// httptest.Server.Close() BLOCKS until every outstanding handler returns — so registering the server's
	// close last parks it on a handler waiting for a channel closed by the defer behind it, and the test
	// hangs until the package timeout with no failure to read. Measured here: 300 s and a kill.
	defer srv.Close()
	defer close(release)

	// A draft-only pipeline, one chunk: the draft IS the shipping row, so the position under test and the
	// row the export reads are the same row — no edit wave between the subject and the assertion.
	bookPath := setupProjectOpts(t, srv.URL, projectOpts{draftOnly: true, bookUSD: 100, rebillConsentUSD: 1})

	// --- run 1: an ordinary run. It leaves a verdict and shippable text. ---
	r1 := newRunner(t, bookPath)
	if _, err := r1.TranslateBook(tracedCtx()); err != nil {
		r1.Close()
		t.Fatalf("the first run must finish: %v", err)
	}
	before, err := r1.Export(false)
	if err != nil {
		r1.Close()
		t.Fatal(err)
	}
	if len(before.Chunks) != 1 || before.Chunks[0].FinalText == "" {
		r1.Close()
		t.Fatalf("the first run shipped nothing — there is no text for the second run to endanger (%d position(s))", len(before.Chunks))
	}
	shipped := before.Chunks[0].FinalText
	cs, err := r1.Store.GetChunkStatus(r1.Book.BookID, 1, 0, "draft")
	if err != nil || cs == nil {
		r1.Close()
		t.Fatalf("no verdict after the first run: %v", err)
	}
	oldHash, oldDisposition, oldCost := cs.FinalHash, cs.Disposition, cs.CostUSD
	if oldHash == "" {
		r1.Close()
		t.Fatal("the first run left no final_hash, so there is no pointer for a cut to erase")
	}
	r1.Close()
	t.Logf("after run 1: disposition=%q final_hash=%.12s cost_usd=%.6f text=%q", oldDisposition, oldHash, oldCost, shipped)

	// --- the prompt moves. Both guards that would have served the stored row now refuse it: the rendered
	// messages change (content_hash) and so does the snapshot, and repinnable answers false for a move
	// that is not bank-only. This is the re-pin route, driven by an ordinary file edit. ---
	writeFile(t, filepath.Join(filepath.Dir(bookPath), "prompts", "translator.md"),
		"Переводи с {{source_lang}} на {{target_lang}}. Второй вариант промта.\n---USER---\n{{text}}")

	// --- run 2: re-attacks the position, and is cut while its call is out. ---
	armed.Store(true)
	r2 := newRunner(t, bookPath)
	defer r2.Close()
	r2.Resnapshot = true

	ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}))
	go func() {
		// Cancel only once the PROVIDER HAS THE REQUEST. The money gate's counter says a reservation was
		// taken, which is a moment earlier and a different fact: a cancellation in that window kills a
		// request that never went out, the engine owes nothing and marks nothing, and this test would then
		// read «the row is untouched» as proof of a guard that never ran. Measured: the first version of
		// this fixture did exactly that.
		<-arrived
		cancel()
	}()
	_, err = r2.TranslateBook(ctx)
	if rec.count() < 2 {
		t.Fatalf("the second run made no provider call of its own (%d call(s) in total) — the position was served from the store and nothing was cut", rec.count())
	}
	if err == nil {
		t.Fatal("a cancelled run must not report success")
	}

	// --- what is left behind: the verdict this run did not produce, exactly as it was ---
	after, err := r2.Store.GetChunkStatus(r2.Book.BookID, 1, 0, "draft")
	if err != nil || after == nil {
		t.Fatalf("the position lost its row entirely: %v", err)
	}
	t.Logf("after the cut: disposition=%q flag_reason=%q final_hash=%.12s cost_usd=%.6f",
		after.Disposition, after.FlagReason, after.FinalHash, after.CostUSD)

	if FlagReason(after.FlagReason) == FlagCancelled {
		t.Fatal("the cut REPLACED a verdict it did not produce: the position now reads `cancelled`, and the text the earlier run bought is unreachable")
	}
	if after.FinalHash != oldHash {
		t.Fatalf("the pointer to the earlier run's text moved (%.12s → %q) — the checkpoint is still on disk with nothing addressing it", oldHash, after.FinalHash)
	}
	if after.CostUSD != oldCost {
		t.Fatalf("the position's recorded spend was rewritten by the cut: $%.6f → $%.6f", oldCost, after.CostUSD)
	}
	// The text really is still reachable, end to end — not merely a hash that looks right.
	cp, err := r2.Store.GetCheckpoint(after.FinalHash)
	if err != nil {
		t.Fatal(err)
	}
	if cp == nil || cp.ResponseText == "" {
		t.Fatal("the row points at a checkpoint that is gone or empty")
	}
	afterExp, err := r2.Export(false)
	if err != nil {
		t.Fatal(err)
	}
	if len(afterExp.Chunks) != 1 {
		t.Fatalf("export projected %d positions, want 1", len(afterExp.Chunks))
	}
	t.Logf("export after the cut: disposition=%q final_text=%q", afterExp.Chunks[0].Disposition, afterExp.Chunks[0].FinalText)
	if afterExp.Chunks[0].FinalText != shipped {
		t.Fatalf("the reader lost a finished chapter to an interrupted run: shipped %q before, %q after",
			shipped, afterExp.Chunks[0].FinalText)
	}
}

// TestACutOverAVirginPositionStillMarksIt is the CONTROL, and without it the test above is satisfied by
// deleting the mark altogether.
//
// The mark exists for the position that has NO verdict: a run cut over its first attempt would otherwise
// leave the money booked and the position reading «never started», which is the invisible hole
// recordStoppedPosition was written for. The guard added for the row above must not have eaten it.
func TestACutOverAVirginPositionStillMarksIt(t *testing.T) {
	rec := &reqRec{}
	var armed atomic.Bool
	armed.Store(true) // hold from the very first call: this position has never been translated
	release := make(chan struct{})
	arrived := make(chan struct{})
	srv := holdWhenArmed(rec, &armed, arrived, release)
	defer srv.Close()
	defer close(release)

	bookPath := setupProjectOpts(t, srv.URL, projectOpts{draftOnly: true, bookUSD: 100})
	r := newRunner(t, bookPath)
	defer r.Close()

	ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}))
	go func() {
		<-arrived
		cancel()
	}()
	if _, err := r.TranslateBook(ctx); err == nil {
		t.Fatal("a cancelled run must not report success")
	}
	cs, err := r.Store.GetChunkStatus(r.Book.BookID, 1, 0, "draft")
	if err != nil {
		t.Fatal(err)
	}
	if cs == nil {
		t.Fatal("the stopped position has NO row at all: its money is on the ledger and the export will read it as never started — the invisible hole the mark exists to close")
	}
	t.Logf("virgin position after the cut: disposition=%q flag_reason=%q attempts=%d", cs.Disposition, cs.FlagReason, cs.Attempts)
	if FlagReason(cs.FlagReason) != FlagCancelled {
		t.Fatalf("a stopped virgin position must be marked `cancelled`, got %q", cs.FlagReason)
	}
}

// TestAStoppedPositionIsNotRecordedAsFailed is backlog row 389's second half, at the site a stop actually
// travels through: a call a person CUT was healthy when they cut it, and the resume re-does it at the
// same budget. `failed` is the read-model's word for a position that broke, and a reader who finds it
// there goes looking for a defect that is not there.
//
// ⚠ THE CONTROL IS IN THE SAME TEST, and without it this pin is satisfied by never writing the word at
// all: a socket that dies on its own IS a failure and must keep it. Both halves run over the same
// fixture, so the difference asserted is the CAUSE and nothing else.
func TestAStoppedPositionIsNotRecordedAsFailed(t *testing.T) {
	rec := &reqRec{}
	var armed atomic.Bool
	armed.Store(true)
	release := make(chan struct{})
	arrived := make(chan struct{})
	srv := holdWhenArmed(rec, &armed, arrived, release)
	defer srv.Close()
	defer close(release)

	bookPath := setupProjectOpts(t, srv.URL, projectOpts{draftOnly: true, bookUSD: 100})
	r := newRunner(t, bookPath)
	defer r.Close()

	ctx, cancel := context.WithCancel(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}))
	go func() {
		<-arrived
		cancel()
	}()
	if _, err := r.TranslateBook(ctx); err == nil {
		t.Fatal("a cancelled run must not report success")
	}
	job, err := r.Store.EnsureJob(r.Book.BookID, 1, "draft", "")
	if err != nil {
		t.Fatal(err)
	}
	if job == nil {
		t.Fatal("the position has no job row — this pin would then be asserting about nothing")
	}
	t.Logf("job status after the human's cut: %q", job.Status)
	if job.Status == "failed" {
		t.Fatal("the position a person stopped is recorded as a FAILURE; the resume re-does it and nothing broke")
	}
}

// TestABrokenSocketIsStillRecordedAsFailed is the control for the pin above: the word must still be
// written where it is true. Same shape, different cause — the server hangs up mid-request instead of the
// operator stopping the run.
func TestABrokenSocketIsStillRecordedAsFailed(t *testing.T) {
	rec := &reqRec{}
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		body, _ := io.ReadAll(r.Body)
		rec.record(string(body))
		// Hijack and close: the request was delivered, the answer never arrives, and the socket dies —
		// llm.CutByConnection rather than a cancellation.
		hj, ok := w.(http.Hijacker)
		if !ok {
			t.Error("the test server cannot hijack; this control cannot build its cause")
			return
		}
		conn, _, err := hj.Hijack()
		if err != nil {
			t.Errorf("hijack: %v", err)
			return
		}
		conn.Close()
	}))
	defer srv.Close()

	bookPath := setupProjectOpts(t, srv.URL, projectOpts{draftOnly: true, bookUSD: 100})
	r := newRunner(t, bookPath)
	defer r.Close()
	if _, err := r.TranslateBook(tracedCtx()); err == nil {
		t.Fatal("a run against a provider that hangs up must fail")
	}
	job, err := r.Store.EnsureJob(r.Book.BookID, 1, "draft", "")
	if err != nil {
		t.Fatal(err)
	}
	if job == nil {
		t.Fatal("no job row")
	}
	t.Logf("job status after a broken socket: %q", job.Status)
	if job.Status != "failed" {
		t.Fatalf("a position whose provider died must keep the word for it, got %q — the pin above would then be green over a word nobody writes any more", job.Status)
	}
}