package pipeline import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "sort" "strconv" "strings" "testing" "time" "textmachine/backend/internal/obs" ) // golden_test.go — the refactor determinism guard (бэкенд-пакет №4, железное // ограничение №1): on the static fixture book (testdata/golden/) it pins BIT-FOR-BIT // the snapshotID + payload, every stage's request_hash and wire body, the chunk_status // / retrieval_state rows and the final rendered texts — for a fresh run AND a resume // run. Any refactoring step that changes a single byte of the wire or of a resolved // verdict fails this test loudly: a changed request_hash = a missed checkpoint = // --resnapshot = the whole book re-billed (D15), which no code-health win can pay for. // // The fixture deliberately walks the money-relevant paths in one book: a 2-chunk // chapter (sticky window + injection), a spoiler-blocked term (since_ch: 2), a nested // glossary key (紋章 inside 竜の紋章) exercising the collision/longest-match path, a // CJK-echo draft cured by the single-hop escalation re-gate (ch2), and a hard refusal // where the fallback also refuses → flag + skipped edit + exit 2 (ch3). // // D28.2 fixture expansion (D30.9 diff-class, this package): three ratified behaviours now // PINNED by golden, not only by the dedicated unit tests — // • FLOOR: fake-model floors max_tokens to 4000, fake-fallback to 6000 (per-model // capabilities.min_max_tokens) → the wire max_tokens and the resolved capability move, // and the escalation HOP takes ITS OWN model's floor (ch2 hop = 6000, primary = 4000); // • UNION: ch5 fires an OBLIQUE-ONLY decl term (老人→старик) whose FINAL text carries the // NOMINATIVE «старик» → postcheck_miss=0 only because the base-dst∪decl union checks the // base (revert the union → this chunk false-flags a miss); // • SANITIZER (SUBSTANTIVE): ch6 edit leaks a service preamble → the output-sanitizer gate (ON // in the fixture pipeline) flags it FlagSanitizerDefect and the edit is DROPPED (final_text ""). // D38 infra-pack cells (pin the cosmetic strip-and-export path, D35.4a — final_hash points at a // derived $0 sanitized checkpoint so the export contract yields the cleaned text; resume re-serves // it identically at $0): // • SANITIZER (COSMETIC markdown): ch7 edit leaks a leading «### Глава 7» → flagged // sanitizer_stripped, final_text = «Глава 7\n\n…» (the «### » stripped), a derived checkpoint holds it; // • SANITIZER (COSMETIC CJK): ch8 edit leaks a sparse «特产» → flagged sanitizer_stripped, final_text // = the run removed and the seam tidied. // A cell with AMBIGUOUS>0 (an auto/draft candidate forcing a post-check) remains an OPTIONAL // future extension, deliberately NOT added here. // // Update (ONLY on a deliberate, ratified behaviour change — never to "fix" a red // refactor): TM_UPDATE_GOLDEN=1 go test ./internal/pipeline/ -run TestGolden const goldenFile = "testdata/golden/capture.golden" // goldenEchoMarker / goldenRefusalMarker / goldenUnionMarker / goldenSanitizerMarker are // unique substrings of chapters 2/3/5/6 of the fixture source; the mock keys its // echo / refusal / union / sanitizer behaviour off them. const ( goldenEchoMarker = "響動計画" goldenRefusalMarker = "拒絶計画" goldenUnionMarker = "老人の秘密" // ch5 — oblique-only decl union case goldenSanitizerMarker = "序文漏洩" // ch6 — sanitizer preamble-leak case (SUBSTANTIVE → dropped) goldenMarkdownMarker = "見出漏洩" // ch7 — cosmetic markdown «### Глава» leak (STRIPPED + exported) goldenCJKMarker = "漢字漏洩" // ch8 — cosmetic CJK-leak «特产» (STRIPPED + exported) ) // goldenRespond is the deterministic mock provider brain. The response text is a pure // function of the request body (no time, no randomness): default drafts/edits carry a // body-hash suffix so every chunk×stage output is distinct, plus glossary dst forms so // the post-check exercises both hit and miss. func goldenRespond(body string) (text, finish string) { var req struct { Model string `json:"model"` } _ = json.Unmarshal([]byte(body), &req) sum := sha256.Sum256([]byte(body)) tag := hex.EncodeToString(sum[:])[:12] switch { case strings.Contains(body, goldenRefusalMarker): // Both the primary and the fallback refuse → the chunk stays flagged (1 hop). return "Не могу помочь с этим фрагментом.", "refusal" case strings.Contains(body, goldenSanitizerMarker) && !isEditBody(body): // ch6 draft ALSO leaks a preamble AND carries a keying token («ПРЕАМБУЛА-СЦЕНА»). The // sanitizer runs ONLY on the FINAL (edit) output (адверсариальное ревью: не санитайзить // промежуточный черновик), so with isFinal this draft stays OK and the edit is the one // flagged. This PINS isFinal: revert «isFinal &&» in classifyOutput and the DRAFT flags // instead (edit skipped) → golden diff. return "Вот перевод фрагмента: ПРЕАМБУЛА-СЦЕНА Судзуки открыл дверь.", "stop" case strings.Contains(body, goldenUnionMarker) && !isEditBody(body): // ch5 draft carries a keying token («СТАРИК-СЦЕНА») the edit branch below keys on. return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". СТАРИК-СЦЕНА Судзуки увидел старика.", "stop" case strings.Contains(body, goldenMarkdownMarker) && !isEditBody(body): // ch7 draft carries a keying token («МАРКДАУН-СЦЕНА») the cosmetic edit branch keys on. return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". МАРКДАУН-СЦЕНА Судзуки открыл седьмую дверь.", "stop" case strings.Contains(body, goldenCJKMarker) && !isEditBody(body): // ch8 draft carries a keying token («ИЕРОГЛИФ-СЦЕНА») the cosmetic edit branch keys on. return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". ИЕРОГЛИФ-СЦЕНА Судзуки нашёл камень.", "stop" case strings.Contains(body, goldenEchoMarker) && !isEditBody(body): if req.Model == "fake-fallback" { // The escalation hop returns a clean translation → the re-gate passes it. return "ЭСКАЛАЦИОННЫЙ ПЕРЕВОД " + tag + ". Судзуки вынес Драконью печать из Академии магии.", "stop" } // The primary draft echoes CJK instead of translating → cjk_artifact. return "夜明け前、鈴木は再び書庫に戻り、竜の紋章を布に包んで持ち出した。", "stop" case isEditBody(body) && strings.Contains(body, "ПРЕАМБУЛА-СЦЕНА"): // ch6 edit leaks a service preamble → the output-sanitizer flags the FINAL stage // (FlagSanitizerDefect). Colon after «фрагмента» is the high-precision signal. return "Вот перевод фрагмента: Судзуки открыл дверь книгохранилища.", "stop" case isEditBody(body) && strings.Contains(body, "СТАРИК-СЦЕНА"): // ch5 edit — the FINAL text carries the NOMINATIVE «старик» (not an oblique decl // form of 老人→старик), exercising the base-dst∪decl union → postcheck_miss=0. return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки увидел, как в тени ждал старик.", "stop" case isEditBody(body) && strings.Contains(body, "МАРКДАУН-СЦЕНА"): // ch7 edit leaks a leading markdown «### Глава» header — a COSMETIC class: the sanitizer // STRIPS it and exports the remainder flagged sanitizer_stripped (D35.4a). This PINS the // strip-and-export path: final_text = «Глава 7\n\n…» (no «### »), disposition=flagged. return "### Глава 7\n\nСудзуки открыл седьмую дверь книгохранилища.", "stop" case isEditBody(body) && strings.Contains(body, "ИЕРОГЛИФ-СЦЕНА"): // ch8 edit leaks a sparse CJK run «特产» in otherwise-Russian prose (below the 15% echo // threshold classify() catches) — a COSMETIC class: stripped + exported flagged. PINS the // CJK-leak strip: «Судзуки нашёл 特产 древний камень.» → «Судзуки нашёл древний камень.» return "Судзуки нашёл 特产 древний камень на каменном алтаре.", "stop" case isEditBody(body): return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки шёл по коридорам Академии магии.", "stop" default: return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". Судзуки шёл по коридорам Академии магии.", "stop" } } // setupGoldenProject copies the static fixture into a temp dir and writes models.yaml // (the only runtime-generated file: it carries the live mock URL and a fresh // prices_checked date — neither enters the snapshot, request hashes or wire bodies). func setupGoldenProject(t *testing.T, providerURL string) string { t.Helper() dir := t.TempDir() src := filepath.Join("testdata", "golden") for _, f := range []string{"book.yaml", "pipeline.yaml", "source.txt", "glossary-seed.yaml", filepath.Join("prompts", "translator.md"), filepath.Join("prompts", "editor.md")} { data, err := os.ReadFile(filepath.Join(src, f)) if err != nil { t.Fatal(err) } writeFile(t, filepath.Join(dir, f), string(data)) } writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(` prices_checked: %q default_model: fake-model providers: fake: kind: openai base_url: %q timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 } models: fake-model: provider: fake price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 } capabilities: { min_max_tokens: 4000 } # D28.2: floor the PRIMARY model — pins floored max_tokens on the wire fake-fallback: provider: fake price: { input_per_m: 3.0, cached_per_m: 0.3, cache_write_per_m: 0, output_per_m: 6.0 } capabilities: { min_max_tokens: 6000 } # D28.2: a DISTINCT hop floor — pins that the hop re-floors by ITS own model `, time.Now().UTC().Format("2006-01-02"), providerURL)) return filepath.Join(dir, "book.yaml") } // captureGolden renders the full deterministic state of a run as one canonical text // blob. Everything time/host-dependent (latency, timestamps, temp paths, provider // URL) is deliberately excluded; everything byte/verdict-relevant is included. func captureGolden(t *testing.T, label string, r *Runner, res *BookResult, wireBodies []string) string { t.Helper() var b strings.Builder w := func(format string, args ...any) { fmt.Fprintf(&b, format+"\n", args...) } fl := func(f float64) string { return strconv.FormatFloat(f, 'f', -1, 64) } w("==== run %s ====", label) snapID, payload, err := r.snapshotID() if err != nil { t.Fatal(err) } w("snapshot_id: %s", snapID) w("snapshot_payload: %s", payload) w("brief_hash: %s", r.Book.BriefHash()) w("memory_version: %s", r.memoryVersion()) w("-- book result --") w("chunks=%d flagged=%d exit=%d total_usd=%s", len(res.Chunks), res.Flagged, res.ExitCode(), fl(res.TotalUSD)) for _, ch := range res.Chunks { w("chunk ch%d/%d disposition=%s flag=%q final_text=%q cost=%s", ch.Chapter, ch.ChunkIdx, ch.Disposition, ch.FlagReason, ch.FinalText, fl(ch.CostUSD)) for _, st := range ch.Stages { w(" stage=%s role=%s model=%s resume=%t disp=%s flag=%q attempts=%d escalated=%t esc_model=%q finish=%q cum_usd=%s detail=%q", st.Stage, st.Role, st.Model, st.FromResume, st.Disposition, st.FlagReason, st.Attempts, st.Escalated, st.EscalationModel, st.FinishReason, fl(st.CumCostUSD), st.Detail) w(" stage_text=%q recovered=%q", st.Text, st.RecoveredText) } } w("-- chunk_status --") css, err := r.Store.ChunkStatusesForBook(r.Book.BookID) if err != nil { t.Fatal(err) } sort.Slice(css, func(i, j int) bool { a, c := css[i], css[j] if a.Chapter != c.Chapter { return a.Chapter < c.Chapter } if a.ChunkIdx != c.ChunkIdx { return a.ChunkIdx < c.ChunkIdx } return a.Stage < c.Stage }) for _, cs := range css { w("ch%d/%d %s snap_match=%t content_hash=%s disp=%s flag=%q attempts=%d final_hash=%s cost=%s escalated=%t esc_model=%q detail=%q", cs.Chapter, cs.ChunkIdx, cs.Stage, cs.SnapshotID == snapID, cs.ContentHash, cs.Disposition, cs.FlagReason, cs.Attempts, cs.FinalHash, fl(cs.CostUSD), cs.Escalated, cs.EscalationModel, cs.Detail) } w("-- request_log (insertion order) --") rls, err := r.Store.RequestLogRows(r.Book.BookID) if err != nil { t.Fatal(err) } for _, rl := range rls { w("ch%d/%d %s role=%s req=%s actual=%s hash=%s tm_hit=%d ok=%d finish=%q degraded=%q cost=%s tokens=%d/%d/%d/%d/%d err=%q", rl.Chapter, rl.ChunkIdx, rl.Stage, rl.Role, rl.ModelRequested, rl.ModelActual, rl.RequestHash, rl.TMHit, rl.OK, rl.FinishReason, rl.Degraded, fl(rl.CostUSD), rl.PromptTokens, rl.CachedTokens, rl.CacheCreationTokens, rl.CompletionTokens, rl.ReasoningTokens, rl.Err) } w("-- retrieval_state --") rss, err := r.Store.RetrievalStatesForBook(r.Book.BookID) if err != nil { t.Fatal(err) } sort.Slice(rss, func(i, j int) bool { if rss[i].Chapter != rss[j].Chapter { return rss[i].Chapter < rss[j].Chapter } return rss[i].ChunkIdx < rss[j].ChunkIdx }) for _, rs := range rss { w("ch%d/%d snap_match=%t exact=%d sticky=%d ambiguous=%d spoiler=%d evicted=%d postcheck_miss=%d style_flags=%d", rs.Chapter, rs.ChunkIdx, rs.SnapshotID == snapID, rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted, rs.NPostcheckMiss, rs.NStyleFlags) w(" injected_ids=%s postcheck_detail=%s style_detail=%s", rs.InjectedIDs, rs.PostcheckDetail, rs.StyleDetail) } w("-- wire bodies (call order) --") for i, body := range wireBodies { w("[%d] %s", i, body) } return b.String() } // TestGoldenDeterminism is the refactor guard: fresh run + resume run against the // static fixture must match testdata/golden/capture.golden byte-for-byte. func TestGoldenDeterminism(t *testing.T) { rec := &reqRec{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) rec.record(string(body)) text, finish := goldenRespond(string(body)) tb, _ := json.Marshal(text) var req struct { Model string `json:"model"` } _ = json.Unmarshal(body, &req) fmt.Fprintf(w, `{"id":"fake","model":%q,"choices":[{"message":{"content":%s},"finish_reason":%q}], "usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`, req.Model, tb, finish) })) defer srv.Close() bookPath := setupGoldenProject(t, srv.URL) ctx := obs.WithReqInfo(t.Context(), obs.ReqInfo{TraceID: "golden-trace"}) // Run 1: fresh book, every call is a fresh reserve→call→settle. r1 := newRunner(t, bookPath) res1, err := r1.TranslateBook(ctx) if err != nil { t.Fatal(err) } capture := captureGolden(t, "1 (fresh)", r1, res1, rec.all()) freshCalls := rec.count() r1.Close() // Run 2: resume from checkpoints/chunk_status — must make ZERO provider calls // and reproduce the identical texts/verdicts (the wire-bodies section stays as // captured in run 1; a resume that re-called the provider would append to it). r2 := newRunner(t, bookPath) defer r2.Close() res2, err := r2.TranslateBook(ctx) if err != nil { t.Fatal(err) } if rec.count() != freshCalls { t.Fatalf("resume made %d extra provider calls", rec.count()-freshCalls) } capture += captureGolden(t, "2 (resume)", r2, res2, nil) if os.Getenv("TM_UPDATE_GOLDEN") == "1" { if err := os.WriteFile(goldenFile, []byte(capture), 0o644); err != nil { t.Fatal(err) } t.Logf("golden updated: %s (%d bytes, %d provider calls)", goldenFile, len(capture), freshCalls) return } want, err := os.ReadFile(goldenFile) if err != nil { t.Fatalf("golden file missing (generate once with TM_UPDATE_GOLDEN=1): %v", err) } if string(want) != capture { t.Fatalf("golden mismatch: the run's bytes diverged from %s.\nЭто значит, что рефакторинг изменил wire-байты, хеши или вердикты — "+ "любое такое изменение = --resnapshot = переоплата книги (D15). Найди и убери причину; обновлять golden можно только "+ "на осознанной, ратифицированной смене поведения.\n%s", goldenFile, diffHint(string(want), capture)) } } // diffHint points a human at the first diverging line — the full capture is too big // for a useful t.Fatalf dump. func diffHint(want, got string) string { wl, gl := strings.Split(want, "\n"), strings.Split(got, "\n") n := len(wl) if len(gl) < n { n = len(gl) } for i := 0; i < n; i++ { if wl[i] != gl[i] { return fmt.Sprintf("first divergence at line %d:\n want: %s\n got: %s", i+1, wl[i], gl[i]) } } return fmt.Sprintf("line counts differ: want %d, got %d", len(wl), len(gl)) }