package pipeline import ( "context" "errors" "strings" "testing" "textmachine/backend/internal/obs" "textmachine/backend/internal/store" ) // echorecovered_test.go: «no longer the verdict» is not «recovered». // // The echo metric counts a draft whose FIRST attempt echoed even when the row's own verdict is something // else — that split is what keeps a recovered echo visible (minirun_fixes_test.go). The RECOVERY half of // the split used to be read off the same column: `first_flag_reason == cjk_artifact` was counted as an // echo somebody fixed, without asking what the row says NOW. A re-roll that came back with a DIFFERENT // failure lands exactly there — the unit ships nothing and the report says the echo was recovered. // // It is live rather than theoretical: `regenerate_echo_before_escalate: 1` stands in all four shipping // pipelines (backend/configs/pipeline-c1.yaml, -c2, -arm-glm, -arm-mistral), so the re-roll this fixture // drives is the one production runs. // TestASupersededEchoIsNotCountedAsRecovered drives the whole path rather than writing the row by hand, // because the claim has two halves: the row shape is REACHABLE on a shipping configuration, and the // counter reads it correctly. A hand-written row would prove only the second. func TestASupersededEchoIsNotCountedAsRecovered(t *testing.T) { rec := &reqRec{} calls := 0 srv := newJSONProvider(rec, func(body string) (string, string) { if isEditBody(body) { return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД", "stop" } calls++ if calls == 1 { // The echo: the source handed back, dense enough for the classifier to call it an artifact. return strings.Repeat(suzukiSource, 3), "stop" } // The re-roll answers in the target language and is CUT at the budget: a different failure, and // with the regeneration budget now spent it is the row's verdict. return "ЧЕРНОВИК ОБОРВАН", "length" }) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{ source: suzukiSource, glossarySeed: suzukiSeed, // Both budgets as the shipping pipelines carry them: one content regeneration, one echo re-roll. // They share the `regens` counter, which is why the length cut below cannot be re-attacked. regenerate: 1, regenerateEcho: 1, }) r := newRunner(t, bookPath) defer r.Close() if _, err := r.TranslateBook(obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})); err != nil { t.Fatal(err) } cs, err := r.Store.GetChunkStatus("test-book", 1, 0, "draft") if err != nil || cs == nil { t.Fatalf("draft row: %v %v", cs, err) } // The premise, printed: the row shape this test is about — an echo that is no longer the verdict, on a // unit that ships NOTHING. Without it the assertions below could pass on a row nobody is arguing about. t.Logf("the row the re-roll left: disposition=%q flag_reason=%q first_flag_reason=%q attempts=%d final_hash=%q", cs.Disposition, cs.FlagReason, cs.FirstFlagReason, cs.Attempts, cs.FinalHash) if cs.FirstFlagReason != string(FlagCJKArtifact) || cs.FlagReason != string(FlagLength) { t.Fatalf("premise broken: this fixture must leave first_flag_reason=cjk_artifact with the verdict moved to length, got %q/%q — the counter below would then be asked about a different row", cs.FirstFlagReason, cs.FlagReason) } if cs.Disposition != string(DispFlagged) { t.Fatalf("premise broken: the unit must still be flagged, got %q", cs.Disposition) } q, qerr := r.QualityReport() if qerr != nil { t.Fatal(qerr) } t.Logf("echo metric: chunks=%d recovered=%d rate=%.3f", q.EchoDraftChunks, q.EchoDraftRecovered, q.EchoDraftRate) if q.EchoDraftChunks != 1 { t.Errorf("echo_draft_chunks = %d, want 1 — the translator echoed, and that is what this metric watches", q.EchoDraftChunks) } if q.EchoDraftRecovered != 0 { t.Errorf("echo_draft_recovered = %d, want 0: nothing recovered this unit — the re-roll failed differently and the unit ships no text. A recovery count that reads only `first_flag_reason` reports our success at papering over an echo where there was none", q.EchoDraftRecovered) } } // TestAnEchoSomebodyPaidForStaysInTheMetricWhenTheMoneyRanOut is the OTHER half of the same column, and // the one the money mark put at risk: a draft that ECHOED at attempt 0 and whose echo re-roll a ceiling // refused. // // ⛔ THE ROW'S OWN REASON THEN SAYS WHY THE PURCHASE DID NOT HAPPEN, not what came back — so unless the // mark carries the superseded first failure, the unit stays in the metric's DENOMINATOR (it has a draft // row) and leaves its NUMERATOR, and the echo rate FALLS on the very run that paid for an echo. That is // the shape of the 25.07 mini-run («echo_draft=0.0% of 20»), one column further along, and it is invisible // to every other surface: the money is right, the mark is right, the rate is wrong. func TestAnEchoSomebodyPaidForStaysInTheMetricWhenTheMoneyRanOut(t *testing.T) { rec := &reqRec{} calls := 0 srv := newJSONProvider(rec, func(body string) (string, string) { calls++ // Every call echoes: the re-roll this fixture is about is never bought, so only the first one runs. return strings.Repeat(suzukiSource, 3), "stop" }) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{ source: suzukiSource, glossarySeed: suzukiSeed, // The echo re-roll budget the four shipping pipelines carry. Draft-only so the unit's money is // one call plus the refused re-roll and nothing else. regenerate: 1, regenerateEcho: 1, draftOnly: true, }) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) defer r.Close() manifest, merr := r.bookChunks() if merr != nil { t.Fatal(merr) } if len(manifest) != 1 { t.Fatalf("fixture must be ONE unit, got %d", len(manifest)) } ceiling, _, _ := ceilingThatRefusesTheReattack(t, r, manifest[0], 0) r.CeilingUSD = ceiling _, err := r.TranslateBook(ctx) var halt *CeilingHalt if !errors.As(err, &halt) { t.Fatalf("the re-roll must be refused by the ceiling, got %v", err) } if calls != 1 { t.Fatalf("the provider was called %d time(s), want 1: the echo was bought and the re-roll was not", calls) } cs, cerr := r.Store.GetChunkStatus("test-book", manifest[0].Chapter, manifest[0].ChunkIdx, "draft") if cerr != nil || cs == nil { t.Fatalf("the stopped position has no row: %v %v", cs, cerr) } t.Logf("the mark the stop left: disposition=%q flag_reason=%q first_flag_reason=%q cost_usd=%.6f", cs.Disposition, cs.FlagReason, cs.FirstFlagReason, cs.CostUSD) if FlagReason(cs.FlagReason) != FlagRetryUnaffordable { t.Fatalf("premise broken: the row must be the money mark, got %q", cs.FlagReason) } if cs.FirstFlagReason != string(FlagCJKArtifact) { t.Fatalf("the mark dropped what the PAID attempt came back as (first_flag_reason=%q): the echo this run bought is now invisible to the metric that exists to find it", cs.FirstFlagReason) } q, qerr := r.QualityReport() if qerr != nil { t.Fatal(qerr) } t.Logf("echo metric: rows=%d chunks=%d recovered=%d rate=%.3f", 1, q.EchoDraftChunks, q.EchoDraftRecovered, q.EchoDraftRate) if q.EchoDraftChunks != 1 || q.EchoDraftRate == 0 { t.Errorf("echo_draft_chunks=%d rate=%.3f, want 1 and non-zero — the translator echoed and the book paid for it", q.EchoDraftChunks, q.EchoDraftRate) } // And the other half of the same column, which this pack made honest: nothing RECOVERED this unit — // the re-roll was never bought. if q.EchoDraftRecovered != 0 { t.Errorf("echo_draft_recovered=%d, want 0: the re-roll this echo needed was never bought", q.EchoDraftRecovered) } } // TestTheEDITORsOwnEchoStaysCountableWhenTheMoneyRanOut is the same column on the stage that was missing // it, and the pack is what made the gap cost something: the mark gives the edit position a row, `editRows` // counts it unconditionally, and the numerator used to see an editor echo ONLY while it was still the // row's verdict — so the edit echo RATE FELL on a run that had just paid for an editor echo. // // The blindness itself is older (an echo a re-roll RECOVERED was invisible too). Both halves are closed by // the same second arm, and this fixture drives the half the pack created: the editor echoes, the echo // re-roll is refused by a ceiling, and the run stops. func TestTheEDITORsOwnEchoStaysCountableWhenTheMoneyRanOut(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, func(body string) (string, string) { if isEditBody(body) { // The EDITOR hands the source back — an echo of its own making, dense enough to be called one. return strings.Repeat(suzukiSource, 3), "stop" } return "ЧЕРНОВИК ПЕРЕВОДА", "stop" }) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{ source: suzukiSource, glossarySeed: suzukiSeed, regenerate: 1, regenerateEcho: 1, }) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) defer r.Close() manifest, merr := r.bookChunks() if merr != nil { t.Fatal(merr) } // The window is computed for the EDIT stage, which sizes and renders from the DRAFT the provider above // returns — the same rule runStage follows, asked through the same helper. ceiling, _, _ := ceilingThatRefusesTheReattackAt(t, r, 1, manifest[0], "ЧЕРНОВИК ПЕРЕВОДА", 1) r.CeilingUSD = ceiling _, err := r.TranslateBook(ctx) var halt *CeilingHalt if !errors.As(err, &halt) { t.Fatalf("the editor's echo re-roll must be refused by the ceiling, got %v", err) } editStage := r.Pipeline.Stages[len(r.Pipeline.Stages)-1].Name cs, cerr := r.Store.GetChunkStatus("test-book", manifest[0].Chapter, manifest[0].ChunkIdx, editStage) if cerr != nil || cs == nil { t.Fatalf("the stopped EDIT position has no row: %v %v", cs, cerr) } t.Logf("the edit row the stop left: disposition=%q flag_reason=%q first_flag_reason=%q cost_usd=%.6f", cs.Disposition, cs.FlagReason, cs.FirstFlagReason, cs.CostUSD) if FlagReason(cs.FlagReason) != FlagRetryUnaffordable || cs.FirstFlagReason != string(FlagCJKArtifact) { t.Fatalf("premise broken: this fixture must leave the EDIT row as the money mark over an echo, got %q/%q", cs.FlagReason, cs.FirstFlagReason) } if cs.CostUSD <= 0 { t.Fatalf("premise broken: the editor's echo must have been PAID for, got cost_usd=%.6f", cs.CostUSD) } q, qerr := r.QualityReport() if qerr != nil { t.Fatal(qerr) } t.Logf("echo metric: edit units=%d rate=%.3f · draft chunks=%d", q.EchoEditUnits, q.EchoEditRate, q.EchoDraftChunks) if q.EchoEditUnits != 1 || q.EchoEditRate == 0 { t.Errorf("echo_edit_units=%d rate=%.3f, want 1 and non-zero: the EDITOR echoed and the book paid $%.6f for it — a row in the denominator with nothing in the numerator makes the rate fall on the run that bought the echo", q.EchoEditUnits, q.EchoEditRate, cs.CostUSD) } // ⛔ AND THE OTHER SIDE OF THE SAME GUARD, which nothing asserted anywhere until a reader mutated it: // this echo was NOT cured — the ceiling refused the re-roll and the row carries no hash — so the // recovered subset must stay EMPTY. The draft arm has both sides of its guard (two tests catch the // removal of its gate; a third asserts a zero on a row whose verdict IS the echo, which never reaches // the gate); the edit arm had only the positive one, and dropping `shippedText` from it left the package // green while an operator read «echo edit=1 (100.0%), 1 recovered» over a book nobody cured anything in. if q.EchoEditRecovered != 0 { t.Errorf("echo_edit_recovered=%d, want 0: the money ran out BEFORE the re-roll, the row carries no final hash, and calling that a recovery inverts the very clause it was built for", q.EchoEditRecovered) } // The control: the translator did NOT echo in this fixture, so the draft counter must stay at zero — // otherwise the assertion above could be passing on a counter that says «echo» about everything. if q.EchoDraftChunks != 0 { t.Errorf("echo_draft_chunks=%d, want 0: the translator answered in the target language here", q.EchoDraftChunks) } } // TestARecoveredEchoIsReportedAsRecoveredOnBothStages is the GUARD the widened numerators needed, and the // reason it exists is the same on both waves: a counter that says «echo» about a book which shipped clean // prose reads as fresh breakage unless the recovered share is spelled out beside it. // // ⛔ AND THE RECOVERY QUESTION IS «DID THE ROW PRODUCE TEXT», NOT «IS THE ROW OK». Keyed on the `ok` verdict alone // it lost a real cure: the escalation hop's answer can come back with a COSMETIC leak, which the sanitizer // STRIPS and ships (final_hash points at the derived export, and the executor calls that outcome recovered // in its own words) — so an echo that was genuinely fixed reported as none, on the three shipping pipelines // where the sanitizer gate is on. The rows below are written straight into the store because this is a // READ-MODEL claim; that both shapes are reachable was measured by an adversarial pass (a stripped // recovery shipping 275 bytes with `first_flag_reason=cjk_artifact`). func TestARecoveredEchoIsReportedAsRecoveredOnBothStages(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: suzukiSource, glossarySeed: suzukiSeed}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) defer r.Close() if _, err := r.TranslateBook(ctx); err != nil { t.Fatal(err) } manifest, merr := r.bookChunks() if merr != nil { t.Fatal(merr) } ch := manifest[0] draftStage, editStage := r.Pipeline.Stages[0].Name, r.Pipeline.Stages[len(r.Pipeline.Stages)-1].Name // Both rows: an echo at the first attempt, a verdict that is no longer it, and TEXT on disk — one // through an `ok` verdict, one through a cosmetic strip whose cleaned remainder IS the export. for _, row := range []store.ChunkStatus{ {BookID: "test-book", Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: draftStage, Disposition: string(DispOK), FirstFlagReason: string(FlagCJKArtifact), FinalHash: "draft-hash", Attempts: 2}, {BookID: "test-book", Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: editStage, Disposition: string(DispFlagged), FlagReason: string(FlagSanitizerStripped), FirstFlagReason: string(FlagCJKArtifact), FinalHash: "edit-hash", Attempts: 2}, } { if err := r.Store.UpsertChunkStatus(row); err != nil { t.Fatal(err) } } q, qerr := r.QualityReport() if qerr != nil { t.Fatal(qerr) } t.Logf("draft: echo=%d recovered=%d · edit: echo=%d recovered=%d", q.EchoDraftChunks, q.EchoDraftRecovered, q.EchoEditUnits, q.EchoEditRecovered) if q.EchoDraftChunks != 1 || q.EchoDraftRecovered != 1 { t.Errorf("draft echo=%d recovered=%d, want 1 and 1: the translator echoed and the unit shipped", q.EchoDraftChunks, q.EchoDraftRecovered) } if q.EchoEditUnits != 1 || q.EchoEditRecovered != 1 { t.Errorf("edit echo=%d recovered=%d, want 1 and 1: the editor echoed, the strip shipped the cleaned text, and a headline without the recovered share reads as fresh breakage on a book that delivered", q.EchoEditUnits, q.EchoEditRecovered) } } // TestACuredEchoOnAMidEditStageIsStillCuredWhenTheUnitDiesLater pins the reading BOTH recovery counters // deliberately take — the ROW, not the unit — on the only shape where the two come apart on the edit side. // `waveStages` puts every non-translator stage in the edit wave, so a wave with a second stage has a first // stage whose row is not the unit's last word. ⚠ The reachable shape is a second `editor` stage, which is // what this fixture builds and what the engine actually runs: the one config in the repo declaring two // (`select`+`edit` in c2) is REFUSED by CheckRunnable, so citing it as evidence would repeat a counterexample // the repo has already withdrawn. The sibling pin above holds `secondEditStage` // false, so it reports for the class and measures the one-stage slice; this is the slice it is silent about, // and the pack's own doc claimed the opposite until a reader ran the shape. // // ⛔ WHY THIS IS THE READING THE CODE TAKES: the metric asks what the MODEL did and what curing it cost. A // later stage flagging the unit for its own reasons does not un-cure the echo, and the unit's absence from // the book is what the hole counters report. ⚠ A READING, NOT A RATIFIED RULE: D39.18 kept the «echo // anywhere» semantics and left «echo counted over shipped text only» open to the owner, so this test pins // what the code does today, not a decision. The day somebody wants «recoveries that reached the reader», // that is a DIFFERENT counter, and this test is where the difference is written down. func TestACuredEchoOnAMidEditStageIsStillCuredWhenTheUnitDiesLater(t *testing.T) { rec := &reqRec{} srv := newJSONProvider(rec, draftEdit) defer srv.Close() bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: suzukiSource, glossarySeed: suzukiSeed, secondEditStage: true}) ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) r := newRunner(t, bookPath) defer r.Close() if _, err := r.TranslateBook(ctx); err != nil { t.Fatal(err) } // The premise, asserted rather than assumed: three stages, and the echoed one is NOT the last. if len(r.Pipeline.Stages) != 3 { t.Fatalf("the fixture must declare a second edit-wave stage, got %d stages", len(r.Pipeline.Stages)) } midEdit, finalStage := r.Pipeline.Stages[1].Name, r.Pipeline.Stages[2].Name if midEdit == finalStage { t.Fatalf("mid stage %q is the final stage — this test is about a row that is not the unit's last word", midEdit) } manifest, merr := r.bookChunks() if merr != nil { t.Fatal(merr) } ch := manifest[0] for _, row := range []store.ChunkStatus{ // The editor echoed on the mid stage and a re-roll cured it: the row's verdict is no longer the // echo and its output became a checkpoint the next stage read. {BookID: "test-book", Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: midEdit, Disposition: string(DispOK), FirstFlagReason: string(FlagCJKArtifact), FinalHash: "mid-hash", Attempts: 2}, // …and then the LAST stage refused, so the unit ships nothing for a reason that has nothing to do // with the echo. {BookID: "test-book", Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: finalStage, Disposition: string(DispFlagged), FlagReason: string(FlagHardRefusal), FinalHash: "", Attempts: 1}, } { if err := r.Store.UpsertChunkStatus(row); err != nil { t.Fatal(err) } } q, qerr := r.QualityReport() if qerr != nil { t.Fatal(qerr) } t.Logf("edit echo=%d recovered=%d rate=%.2f · text_units=%d processed_units=%d", q.EchoEditUnits, q.EchoEditRecovered, q.EchoEditRate, q.TextUnits, q.ProcessedUnits) // The fixture's own premise: the unit shipped NOTHING. Without this the assertion below would also pass // on a report where the unit was fine, and the test would be about the ordinary one-stage shape again. if q.TextUnits != 0 { t.Fatalf("text_units=%d, want 0 — the unit must ship nothing, or this test is not about the shape it names", q.TextUnits) } // The denominator proves BOTH edit-wave rows are in the wave: 1 of 2, not 1 of 1. if q.EchoEditRate != 0.5 { t.Fatalf("edit echo rate=%.2f, want 0.50 over the two edit-wave rows — the wave must hold both stages", q.EchoEditRate) } if q.EchoEditUnits != 1 || q.EchoEditRecovered != 1 { t.Fatalf("edit echo=%d recovered=%d, want 1 and 1: the echo WAS cured on the stage that made it, and a later stage's refusal is the hole counters' business, not the echo metric's", q.EchoEditUnits, q.EchoEditRecovered) } }