textmachine/backend/internal/pipeline/echorecovered_test.go

package pipeline

import (
	"context"
	"errors"
	"strings"
	"testing"

	"textmachine/backend/internal/obs"
	"textmachine/backend/internal/store"
)

// echorecovered_test.go: «no longer the verdict» is not «recovered».
//
// The echo metric counts a draft whose FIRST attempt echoed even when the row's own verdict is something
// else — that split is what keeps a recovered echo visible (minirun_fixes_test.go). The RECOVERY half of
// the split used to be read off the same column: `first_flag_reason == cjk_artifact` was counted as an
// echo somebody fixed, without asking what the row says NOW. A re-roll that came back with a DIFFERENT
// failure lands exactly there — the unit ships nothing and the report says the echo was recovered.
//
// It is live rather than theoretical: `regenerate_echo_before_escalate: 1` stands in all four shipping
// pipelines (backend/configs/pipeline-c1.yaml, -c2, -arm-glm, -arm-mistral), so the re-roll this fixture
// drives is the one production runs.

// TestASupersededEchoIsNotCountedAsRecovered drives the whole path rather than writing the row by hand,
// because the claim has two halves: the row shape is REACHABLE on a shipping configuration, and the
// counter reads it correctly. A hand-written row would prove only the second.
func TestASupersededEchoIsNotCountedAsRecovered(t *testing.T) {
	rec := &reqRec{}
	calls := 0
	srv := newJSONProvider(rec, func(body string) (string, string) {
		if isEditBody(body) {
			return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop"
		}
		calls++
		if calls == 1 {
			// The echo: the source handed back, dense enough for the classifier to call it an artifact.
			return strings.Repeat(suzukiSource, 3), "stop"
		}
		// The re-roll answers in the target language and is CUT at the budget: a different failure, and
		// with the regeneration budget now spent it is the row's verdict.
		return "ЧЕРНОВИК ОБОРВАН", "length"
	})
	defer srv.Close()
	bookPath := setupProjectOpts(t, srv.URL, projectOpts{
		source: suzukiSource, glossarySeed: suzukiSeed,
		// Both budgets as the shipping pipelines carry them: one content regeneration, one echo re-roll.
		// They share the `regens` counter, which is why the length cut below cannot be re-attacked.
		regenerate: 1, regenerateEcho: 1,
	})

	r := newRunner(t, bookPath)
	defer r.Close()
	if _, err := r.TranslateBook(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})); err != nil {
		t.Fatal(err)
	}
	cs, err := r.Store.GetChunkStatus("test-book", 1, 0, "draft")
	if err != nil || cs == nil {
		t.Fatalf("draft row: %v %v", cs, err)
	}
	// The premise, printed: the row shape this test is about — an echo that is no longer the verdict, on a
	// unit that ships NOTHING. Without it the assertions below could pass on a row nobody is arguing about.
	t.Logf("the row the re-roll left: disposition=%q flag_reason=%q first_flag_reason=%q attempts=%d final_hash=%q",
		cs.Disposition, cs.FlagReason, cs.FirstFlagReason, cs.Attempts, cs.FinalHash)
	if cs.FirstFlagReason != string(FlagCJKArtifact) || cs.FlagReason != string(FlagLength) {
		t.Fatalf("premise broken: this fixture must leave first_flag_reason=cjk_artifact with the verdict moved to length, got %q/%q — the counter below would then be asked about a different row",
			cs.FirstFlagReason, cs.FlagReason)
	}
	if cs.Disposition != string(DispFlagged) {
		t.Fatalf("premise broken: the unit must still be flagged, got %q", cs.Disposition)
	}

	q, qerr := r.QualityReport()
	if qerr != nil {
		t.Fatal(qerr)
	}
	t.Logf("echo metric: chunks=%d recovered=%d rate=%.3f", q.EchoDraftChunks, q.EchoDraftRecovered, q.EchoDraftRate)
	if q.EchoDraftChunks != 1 {
		t.Errorf("echo_draft_chunks = %d, want 1 — the translator echoed, and that is what this metric watches", q.EchoDraftChunks)
	}
	if q.EchoDraftRecovered != 0 {
		t.Errorf("echo_draft_recovered = %d, want 0: nothing recovered this unit — the re-roll failed differently and the unit ships no text. A recovery count that reads only `first_flag_reason` reports our success at papering over an echo where there was none",
			q.EchoDraftRecovered)
	}
}

// TestAnEchoSomebodyPaidForStaysInTheMetricWhenTheMoneyRanOut is the OTHER half of the same column, and
// the one the money mark put at risk: a draft that ECHOED at attempt 0 and whose echo re-roll a ceiling
// refused.
//
// ⛔ THE ROW'S OWN REASON THEN SAYS WHY THE PURCHASE DID NOT HAPPEN, not what came back — so unless the
// mark carries the superseded first failure, the unit stays in the metric's DENOMINATOR (it has a draft
// row) and leaves its NUMERATOR, and the echo rate FALLS on the very run that paid for an echo. That is
// the shape of the 25.07 mini-run («echo_draft=0.0% of 20»), one column further along, and it is invisible
// to every other surface: the money is right, the mark is right, the rate is wrong.
func TestAnEchoSomebodyPaidForStaysInTheMetricWhenTheMoneyRanOut(t *testing.T) {
	rec := &reqRec{}
	calls := 0
	srv := newJSONProvider(rec, func(body string) (string, string) {
		calls++
		// Every call echoes: the re-roll this fixture is about is never bought, so only the first one runs.
		return strings.Repeat(suzukiSource, 3), "stop"
	})
	defer srv.Close()
	bookPath := setupProjectOpts(t, srv.URL, projectOpts{
		source: suzukiSource, glossarySeed: suzukiSeed,
		// The echo re-roll budget the four shipping pipelines carry. Draft-only so the unit's money is
		// one call plus the refused re-roll and nothing else.
		regenerate: 1, regenerateEcho: 1, draftOnly: true,
	})
	ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})

	r := newRunner(t, bookPath)
	defer r.Close()
	manifest, merr := r.bookChunks()
	if merr != nil {
		t.Fatal(merr)
	}
	if len(manifest) != 1 {
		t.Fatalf("fixture must be ONE unit, got %d", len(manifest))
	}
	ceiling, _, _ := ceilingThatRefusesTheReattack(t, r, manifest[0], 0)
	r.CeilingUSD = ceiling

	_, err := r.TranslateBook(ctx)
	var halt *CeilingHalt
	if !errors.As(err, &halt) {
		t.Fatalf("the re-roll must be refused by the ceiling, got %v", err)
	}
	if calls != 1 {
		t.Fatalf("the provider was called %d time(s), want 1: the echo was bought and the re-roll was not", calls)
	}
	cs, cerr := r.Store.GetChunkStatus("test-book", manifest[0].Chapter, manifest[0].ChunkIdx, "draft")
	if cerr != nil || cs == nil {
		t.Fatalf("the stopped position has no row: %v %v", cs, cerr)
	}
	t.Logf("the mark the stop left: disposition=%q flag_reason=%q first_flag_reason=%q cost_usd=%.6f",
		cs.Disposition, cs.FlagReason, cs.FirstFlagReason, cs.CostUSD)
	if FlagReason(cs.FlagReason) != FlagRetryUnaffordable {
		t.Fatalf("premise broken: the row must be the money mark, got %q", cs.FlagReason)
	}
	if cs.FirstFlagReason != string(FlagCJKArtifact) {
		t.Fatalf("the mark dropped what the PAID attempt came back as (first_flag_reason=%q): the echo this run bought is now invisible to the metric that exists to find it", cs.FirstFlagReason)
	}

	q, qerr := r.QualityReport()
	if qerr != nil {
		t.Fatal(qerr)
	}
	t.Logf("echo metric: rows=%d chunks=%d recovered=%d rate=%.3f", 1, q.EchoDraftChunks, q.EchoDraftRecovered, q.EchoDraftRate)
	if q.EchoDraftChunks != 1 || q.EchoDraftRate == 0 {
		t.Errorf("echo_draft_chunks=%d rate=%.3f, want 1 and non-zero — the translator echoed and the book paid for it", q.EchoDraftChunks, q.EchoDraftRate)
	}
	// And the other half of the same column, which this pack made honest: nothing RECOVERED this unit —
	// the re-roll was never bought.
	if q.EchoDraftRecovered != 0 {
		t.Errorf("echo_draft_recovered=%d, want 0: the re-roll this echo needed was never bought", q.EchoDraftRecovered)
	}
}

// TestTheEDITORsOwnEchoStaysCountableWhenTheMoneyRanOut is the same column on the stage that was missing
// it, and the pack is what made the gap cost something: the mark gives the edit position a row, `editRows`
// counts it unconditionally, and the numerator used to see an editor echo ONLY while it was still the
// row's verdict — so the edit echo RATE FELL on a run that had just paid for an editor echo.
//
// The blindness itself is older (an echo a re-roll RECOVERED was invisible too). Both halves are closed by
// the same second arm, and this fixture drives the half the pack created: the editor echoes, the echo
// re-roll is refused by a ceiling, and the run stops.
func TestTheEDITORsOwnEchoStaysCountableWhenTheMoneyRanOut(t *testing.T) {
	rec := &reqRec{}
	srv := newJSONProvider(rec, func(body string) (string, string) {
		if isEditBody(body) {
			// The EDITOR hands the source back — an echo of its own making, dense enough to be called one.
			return strings.Repeat(suzukiSource, 3), "stop"
		}
		return "ЧЕРНОВИК ПЕРЕВОДА", "stop"
	})
	defer srv.Close()
	bookPath := setupProjectOpts(t, srv.URL, projectOpts{
		source: suzukiSource, glossarySeed: suzukiSeed,
		regenerate: 1, regenerateEcho: 1,
	})
	ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})

	r := newRunner(t, bookPath)
	defer r.Close()
	manifest, merr := r.bookChunks()
	if merr != nil {
		t.Fatal(merr)
	}
	// The window is computed for the EDIT stage, which sizes and renders from the DRAFT the provider above
	// returns — the same rule runStage follows, asked through the same helper.
	ceiling, _, _ := ceilingThatRefusesTheReattackAt(t, r, 1, manifest[0], "ЧЕРНОВИК ПЕРЕВОДА", 1)
	r.CeilingUSD = ceiling

	_, err := r.TranslateBook(ctx)
	var halt *CeilingHalt
	if !errors.As(err, &halt) {
		t.Fatalf("the editor's echo re-roll must be refused by the ceiling, got %v", err)
	}
	editStage := r.Pipeline.Stages[len(r.Pipeline.Stages)-1].Name
	cs, cerr := r.Store.GetChunkStatus("test-book", manifest[0].Chapter, manifest[0].ChunkIdx, editStage)
	if cerr != nil || cs == nil {
		t.Fatalf("the stopped EDIT position has no row: %v %v", cs, cerr)
	}
	t.Logf("the edit row the stop left: disposition=%q flag_reason=%q first_flag_reason=%q cost_usd=%.6f",
		cs.Disposition, cs.FlagReason, cs.FirstFlagReason, cs.CostUSD)
	if FlagReason(cs.FlagReason) != FlagRetryUnaffordable || cs.FirstFlagReason != string(FlagCJKArtifact) {
		t.Fatalf("premise broken: this fixture must leave the EDIT row as the money mark over an echo, got %q/%q",
			cs.FlagReason, cs.FirstFlagReason)
	}
	if cs.CostUSD <= 0 {
		t.Fatalf("premise broken: the editor's echo must have been PAID for, got cost_usd=%.6f", cs.CostUSD)
	}

	q, qerr := r.QualityReport()
	if qerr != nil {
		t.Fatal(qerr)
	}
	t.Logf("echo metric: edit units=%d rate=%.3f · draft chunks=%d", q.EchoEditUnits, q.EchoEditRate, q.EchoDraftChunks)
	if q.EchoEditUnits != 1 || q.EchoEditRate == 0 {
		t.Errorf("echo_edit_units=%d rate=%.3f, want 1 and non-zero: the EDITOR echoed and the book paid $%.6f for it — a row in the denominator with nothing in the numerator makes the rate fall on the run that bought the echo",
			q.EchoEditUnits, q.EchoEditRate, cs.CostUSD)
	}
	// ⛔ AND THE OTHER SIDE OF THE SAME GUARD, which nothing asserted anywhere until a reader mutated it:
	// this echo was NOT cured — the ceiling refused the re-roll and the row carries no hash — so the
	// recovered subset must stay EMPTY. The draft arm has both sides of its guard (three tests assert its
	// zero); the edit arm had only the positive one, and dropping `shippedText` from it left the package
	// green while an operator read «echo edit=1 (100.0%), 1 recovered» over a book nobody cured anything in.
	if q.EchoEditRecovered != 0 {
		t.Errorf("echo_edit_recovered=%d, want 0: the money ran out BEFORE the re-roll, the row carries no final hash, and calling that a recovery inverts the very clause it was built for",
			q.EchoEditRecovered)
	}
	// The control: the translator did NOT echo in this fixture, so the draft counter must stay at zero —
	// otherwise the assertion above could be passing on a counter that says «echo» about everything.
	if q.EchoDraftChunks != 0 {
		t.Errorf("echo_draft_chunks=%d, want 0: the translator answered in the target language here", q.EchoDraftChunks)
	}
}

// TestARecoveredEchoIsReportedAsRecoveredOnBothStages is the GUARD the widened numerators needed, and the
// reason it exists is the same on both waves: a counter that says «echo» about a book which shipped clean
// prose reads as fresh breakage unless the recovered share is spelled out beside it.
//
// ⛔ AND THE RECOVERY QUESTION IS «DID THE UNIT SHIP», NOT «IS THE ROW OK». Keyed on the `ok` verdict alone
// it lost a real cure: the escalation hop's answer can come back with a COSMETIC leak, which the sanitizer
// STRIPS and ships (final_hash points at the derived export, and the executor calls that outcome recovered
// in its own words) — so an echo that was genuinely fixed reported as none, on the three shipping pipelines
// where the sanitizer gate is on. The rows below are written straight into the store because this is a
// READ-MODEL claim; that both shapes are reachable was measured by an adversarial pass (a stripped
// recovery shipping 275 bytes with `first_flag_reason=cjk_artifact`).
func TestARecoveredEchoIsReportedAsRecoveredOnBothStages(t *testing.T) {
	rec := &reqRec{}
	srv := newJSONProvider(rec, draftEdit)
	defer srv.Close()
	bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: suzukiSource, glossarySeed: suzukiSeed})
	ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
	r := newRunner(t, bookPath)
	defer r.Close()
	if _, err := r.TranslateBook(ctx); err != nil {
		t.Fatal(err)
	}
	manifest, merr := r.bookChunks()
	if merr != nil {
		t.Fatal(merr)
	}
	ch := manifest[0]
	draftStage, editStage := r.Pipeline.Stages[0].Name, r.Pipeline.Stages[len(r.Pipeline.Stages)-1].Name

	// Both rows: an echo at the first attempt, a verdict that is no longer it, and TEXT on disk — one
	// through an `ok` verdict, one through a cosmetic strip whose cleaned remainder IS the export.
	for _, row := range []store.ChunkStatus{
		{BookID: "test-book", Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: draftStage,
			Disposition: string(DispOK), FirstFlagReason: string(FlagCJKArtifact), FinalHash: "draft-hash", Attempts: 2},
		{BookID: "test-book", Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: editStage,
			Disposition: string(DispFlagged), FlagReason: string(FlagSanitizerStripped),
			FirstFlagReason: string(FlagCJKArtifact), FinalHash: "edit-hash", Attempts: 2},
	} {
		if err := r.Store.UpsertChunkStatus(row); err != nil {
			t.Fatal(err)
		}
	}
	q, qerr := r.QualityReport()
	if qerr != nil {
		t.Fatal(qerr)
	}
	t.Logf("draft: echo=%d recovered=%d · edit: echo=%d recovered=%d",
		q.EchoDraftChunks, q.EchoDraftRecovered, q.EchoEditUnits, q.EchoEditRecovered)
	if q.EchoDraftChunks != 1 || q.EchoDraftRecovered != 1 {
		t.Errorf("draft echo=%d recovered=%d, want 1 and 1: the translator echoed and the unit shipped", q.EchoDraftChunks, q.EchoDraftRecovered)
	}
	if q.EchoEditUnits != 1 || q.EchoEditRecovered != 1 {
		t.Errorf("edit echo=%d recovered=%d, want 1 and 1: the editor echoed, the strip shipped the cleaned text, and a headline without the recovered share reads as fresh breakage on a book that delivered",
			q.EchoEditUnits, q.EchoEditRecovered)
	}
}

// TestACuredEchoOnAMidEditStageIsStillCuredWhenTheUnitDiesLater pins the reading BOTH recovery counters
// deliberately take — the ROW, not the unit — on the only shape where the two come apart on the edit side.
// `waveStages` puts every non-translator stage in the edit wave, so a wave with a second stage has a first
// stage whose row is not the unit's last word. ⚠ The reachable shape is a second `editor` stage, which is
// what this fixture builds and what the engine actually runs: the one config in the repo declaring two
// (`select`+`edit` in c2) is REFUSED by CheckRunnable, so citing it as evidence would repeat a counterexample
// the repo has already withdrawn. The sibling pin above holds `secondEditStage`
// false, so it reports for the class and measures the one-stage slice; this is the slice it is silent about,
// and the pack's own doc claimed the opposite until a reader ran the shape.
//
// ⛔ WHY THIS IS THE READING AND NOT A DEFECT: the metric asks what the MODEL did and what curing it cost
// (D39.18 — it watches the model, not our success at papering over it). A later stage flagging the unit for
// its own reasons does not un-cure the echo, and the unit's absence from the book is what the hole counters
// report. The day somebody wants «recoveries that reached the reader», that is a DIFFERENT counter, and this
// test is where the difference is written down.
func TestACuredEchoOnAMidEditStageIsStillCuredWhenTheUnitDiesLater(t *testing.T) {
	rec := &reqRec{}
	srv := newJSONProvider(rec, draftEdit)
	defer srv.Close()
	bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: suzukiSource, glossarySeed: suzukiSeed, secondEditStage: true})
	ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
	r := newRunner(t, bookPath)
	defer r.Close()
	if _, err := r.TranslateBook(ctx); err != nil {
		t.Fatal(err)
	}
	// The premise, asserted rather than assumed: three stages, and the echoed one is NOT the last.
	if len(r.Pipeline.Stages) != 3 {
		t.Fatalf("the fixture must declare a second edit-wave stage, got %d stages", len(r.Pipeline.Stages))
	}
	midEdit, finalStage := r.Pipeline.Stages[1].Name, r.Pipeline.Stages[2].Name
	if midEdit == finalStage {
		t.Fatalf("mid stage %q is the final stage — this test is about a row that is not the unit's last word", midEdit)
	}
	manifest, merr := r.bookChunks()
	if merr != nil {
		t.Fatal(merr)
	}
	ch := manifest[0]
	for _, row := range []store.ChunkStatus{
		// The editor echoed on the mid stage and a re-roll cured it: the row's verdict is no longer the
		// echo and its output became a checkpoint the next stage read.
		{BookID: "test-book", Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: midEdit,
			Disposition: string(DispOK), FirstFlagReason: string(FlagCJKArtifact), FinalHash: "mid-hash", Attempts: 2},
		// …and then the LAST stage refused, so the unit ships nothing for a reason that has nothing to do
		// with the echo.
		{BookID: "test-book", Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: finalStage,
			Disposition: string(DispFlagged), FlagReason: string(FlagHardRefusal), FinalHash: "", Attempts: 1},
	} {
		if err := r.Store.UpsertChunkStatus(row); err != nil {
			t.Fatal(err)
		}
	}
	q, qerr := r.QualityReport()
	if qerr != nil {
		t.Fatal(qerr)
	}
	t.Logf("edit echo=%d recovered=%d rate=%.2f · text_units=%d processed_units=%d",
		q.EchoEditUnits, q.EchoEditRecovered, q.EchoEditRate, q.TextUnits, q.ProcessedUnits)
	// The fixture's own premise: the unit shipped NOTHING. Without this the assertion below would also pass
	// on a report where the unit was fine, and the test would be about the ordinary one-stage shape again.
	if q.TextUnits != 0 {
		t.Fatalf("text_units=%d, want 0 — the unit must ship nothing, or this test is not about the shape it names", q.TextUnits)
	}
	// The denominator proves BOTH edit-wave rows are in the wave: 1 of 2, not 1 of 1.
	if q.EchoEditRate != 0.5 {
		t.Fatalf("edit echo rate=%.2f, want 0.50 over the two edit-wave rows — the wave must hold both stages", q.EchoEditRate)
	}
	if q.EchoEditUnits != 1 || q.EchoEditRecovered != 1 {
		t.Fatalf("edit echo=%d recovered=%d, want 1 and 1: the echo WAS cured on the stage that made it, and a later stage's refusal is the hole counters' business, not the echo metric's",
			q.EchoEditUnits, q.EchoEditRecovered)
	}
}