313 lines
14 KiB
Go
313 lines
14 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"textmachine/backend/internal/obs"
|
||
)
|
||
|
||
// golden_test.go — the refactor determinism guard (бэкенд-пакет №4, железное
|
||
// ограничение №1): on the static fixture book (testdata/golden/) it pins BIT-FOR-BIT
|
||
// the snapshotID + payload, every stage's request_hash and wire body, the chunk_status
|
||
// / retrieval_state rows and the final rendered texts — for a fresh run AND a resume
|
||
// run. Any refactoring step that changes a single byte of the wire or of a resolved
|
||
// verdict fails this test loudly: a changed request_hash = a missed checkpoint =
|
||
// --resnapshot = the whole book re-billed (D15), which no code-health win can pay for.
|
||
//
|
||
// The fixture deliberately walks the money-relevant paths in one book: a 2-chunk
|
||
// chapter (sticky window + injection), a spoiler-blocked term (since_ch: 2), a nested
|
||
// glossary key (紋章 inside 竜の紋章) exercising the collision/longest-match path, a
|
||
// CJK-echo draft cured by the single-hop escalation re-gate (ch2), and a hard refusal
|
||
// where the fallback also refuses → flag + skipped edit + exit 2 (ch3).
|
||
//
|
||
// D28.2 fixture expansion (D30.9 diff-class, this package): three ratified behaviours now
|
||
// PINNED by golden, not only by the dedicated unit tests —
|
||
// • FLOOR: fake-model floors max_tokens to 4000, fake-fallback to 6000 (per-model
|
||
// capabilities.min_max_tokens) → the wire max_tokens and the resolved capability move,
|
||
// and the escalation HOP takes ITS OWN model's floor (ch2 hop = 6000, primary = 4000);
|
||
// • UNION: ch5 fires an OBLIQUE-ONLY decl term (老人→старик) whose FINAL text carries the
|
||
// NOMINATIVE «старик» → postcheck_miss=0 only because the base-dst∪decl union checks the
|
||
// base (revert the union → this chunk false-flags a miss);
|
||
// • SANITIZER: ch6 draft leaks a service preamble → the output-sanitizer gate (ON in the
|
||
// fixture pipeline) flags it FlagSanitizerDefect and the edit is skipped.
|
||
// A cell with AMBIGUOUS>0 (an auto/draft candidate forcing a post-check) remains an OPTIONAL
|
||
// future extension, deliberately NOT added here.
|
||
//
|
||
// Update (ONLY on a deliberate, ratified behaviour change — never to "fix" a red
|
||
// refactor): TM_UPDATE_GOLDEN=1 go test ./internal/pipeline/ -run TestGolden
|
||
const goldenFile = "testdata/golden/capture.golden"
|
||
|
||
// goldenEchoMarker / goldenRefusalMarker / goldenUnionMarker / goldenSanitizerMarker are
|
||
// unique substrings of chapters 2/3/5/6 of the fixture source; the mock keys its
|
||
// echo / refusal / union / sanitizer behaviour off them.
|
||
const (
|
||
goldenEchoMarker = "響動計画"
|
||
goldenRefusalMarker = "拒絶計画"
|
||
goldenUnionMarker = "老人の秘密" // ch5 — oblique-only decl union case
|
||
goldenSanitizerMarker = "序文漏洩" // ch6 — sanitizer preamble-leak case
|
||
)
|
||
|
||
// goldenRespond is the deterministic mock provider brain. The response text is a pure
|
||
// function of the request body (no time, no randomness): default drafts/edits carry a
|
||
// body-hash suffix so every chunk×stage output is distinct, plus glossary dst forms so
|
||
// the post-check exercises both hit and miss.
|
||
func goldenRespond(body string) (text, finish string) {
|
||
var req struct {
|
||
Model string `json:"model"`
|
||
}
|
||
_ = json.Unmarshal([]byte(body), &req)
|
||
sum := sha256.Sum256([]byte(body))
|
||
tag := hex.EncodeToString(sum[:])[:12]
|
||
switch {
|
||
case strings.Contains(body, goldenRefusalMarker):
|
||
// Both the primary and the fallback refuse → the chunk stays flagged (1 hop).
|
||
return "Не могу помочь с этим фрагментом.", "refusal"
|
||
case strings.Contains(body, goldenSanitizerMarker) && !isEditBody(body):
|
||
// ch6 draft ALSO leaks a preamble AND carries a keying token («ПРЕАМБУЛА-СЦЕНА»). The
|
||
// sanitizer runs ONLY on the FINAL (edit) output (адверсариальное ревью: не санитайзить
|
||
// промежуточный черновик), so with isFinal this draft stays OK and the edit is the one
|
||
// flagged. This PINS isFinal: revert «isFinal &&» in classifyOutput and the DRAFT flags
|
||
// instead (edit skipped) → golden diff.
|
||
return "Вот перевод фрагмента: ПРЕАМБУЛА-СЦЕНА Судзуки открыл дверь.", "stop"
|
||
case strings.Contains(body, goldenUnionMarker) && !isEditBody(body):
|
||
// ch5 draft carries a keying token («СТАРИК-СЦЕНА») the edit branch below keys on.
|
||
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". СТАРИК-СЦЕНА Судзуки увидел старика.", "stop"
|
||
case strings.Contains(body, goldenEchoMarker) && !isEditBody(body):
|
||
if req.Model == "fake-fallback" {
|
||
// The escalation hop returns a clean translation → the re-gate passes it.
|
||
return "ЭСКАЛАЦИОННЫЙ ПЕРЕВОД " + tag + ". Судзуки вынес Драконью печать из Академии магии.", "stop"
|
||
}
|
||
// The primary draft echoes CJK instead of translating → cjk_artifact.
|
||
return "夜明け前、鈴木は再び書庫に戻り、竜の紋章を布に包んで持ち出した。", "stop"
|
||
case isEditBody(body) && strings.Contains(body, "ПРЕАМБУЛА-СЦЕНА"):
|
||
// ch6 edit leaks a service preamble → the output-sanitizer flags the FINAL stage
|
||
// (FlagSanitizerDefect). Colon after «фрагмента» is the high-precision signal.
|
||
return "Вот перевод фрагмента: Судзуки открыл дверь книгохранилища.", "stop"
|
||
case isEditBody(body) && strings.Contains(body, "СТАРИК-СЦЕНА"):
|
||
// ch5 edit — the FINAL text carries the NOMINATIVE «старик» (not an oblique decl
|
||
// form of 老人→старик), exercising the base-dst∪decl union → postcheck_miss=0.
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки увидел, как в тени ждал старик.", "stop"
|
||
case isEditBody(body):
|
||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки шёл по коридорам Академии магии.", "stop"
|
||
default:
|
||
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". Судзуки шёл по коридорам Академии магии.", "stop"
|
||
}
|
||
}
|
||
|
||
// setupGoldenProject copies the static fixture into a temp dir and writes models.yaml
|
||
// (the only runtime-generated file: it carries the live mock URL and a fresh
|
||
// prices_checked date — neither enters the snapshot, request hashes or wire bodies).
|
||
func setupGoldenProject(t *testing.T, providerURL string) string {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
src := filepath.Join("testdata", "golden")
|
||
for _, f := range []string{"book.yaml", "pipeline.yaml", "source.txt", "glossary-seed.yaml",
|
||
filepath.Join("prompts", "translator.md"), filepath.Join("prompts", "editor.md")} {
|
||
data, err := os.ReadFile(filepath.Join(src, f))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
writeFile(t, filepath.Join(dir, f), string(data))
|
||
}
|
||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||
prices_checked: %q
|
||
default_model: fake-model
|
||
providers:
|
||
fake:
|
||
kind: openai
|
||
base_url: %q
|
||
timeouts: { attempt_s: 5, max_attempts: 2, backoff_cap_s: 1 }
|
||
models:
|
||
fake-model:
|
||
provider: fake
|
||
price: { input_per_m: 1.0, cached_per_m: 0.1, cache_write_per_m: 0, output_per_m: 2.0 }
|
||
capabilities: { min_max_tokens: 4000 } # D28.2: floor the PRIMARY model — pins floored max_tokens on the wire
|
||
fake-fallback:
|
||
provider: fake
|
||
price: { input_per_m: 3.0, cached_per_m: 0.3, cache_write_per_m: 0, output_per_m: 6.0 }
|
||
capabilities: { min_max_tokens: 6000 } # D28.2: a DISTINCT hop floor — pins that the hop re-floors by ITS own model
|
||
`, time.Now().UTC().Format("2006-01-02"), providerURL))
|
||
return filepath.Join(dir, "book.yaml")
|
||
}
|
||
|
||
// captureGolden renders the full deterministic state of a run as one canonical text
|
||
// blob. Everything time/host-dependent (latency, timestamps, temp paths, provider
|
||
// URL) is deliberately excluded; everything byte/verdict-relevant is included.
|
||
func captureGolden(t *testing.T, label string, r *Runner, res *BookResult, wireBodies []string) string {
|
||
t.Helper()
|
||
var b strings.Builder
|
||
w := func(format string, args ...any) { fmt.Fprintf(&b, format+"\n", args...) }
|
||
fl := func(f float64) string { return strconv.FormatFloat(f, 'f', -1, 64) }
|
||
|
||
w("==== run %s ====", label)
|
||
snapID, payload, err := r.snapshotID()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
w("snapshot_id: %s", snapID)
|
||
w("snapshot_payload: %s", payload)
|
||
w("brief_hash: %s", r.Book.BriefHash())
|
||
w("memory_version: %s", r.memoryVersion())
|
||
|
||
w("-- book result --")
|
||
w("chunks=%d flagged=%d exit=%d total_usd=%s", len(res.Chunks), res.Flagged, res.ExitCode(), fl(res.TotalUSD))
|
||
for _, ch := range res.Chunks {
|
||
w("chunk ch%d/%d disposition=%s flag=%q final_text=%q cost=%s",
|
||
ch.Chapter, ch.ChunkIdx, ch.Disposition, ch.FlagReason, ch.FinalText, fl(ch.CostUSD))
|
||
for _, st := range ch.Stages {
|
||
w(" stage=%s role=%s model=%s resume=%t disp=%s flag=%q attempts=%d escalated=%t esc_model=%q finish=%q cum_usd=%s detail=%q",
|
||
st.Stage, st.Role, st.Model, st.FromResume, st.Disposition, st.FlagReason,
|
||
st.Attempts, st.Escalated, st.EscalationModel, st.FinishReason, fl(st.CumCostUSD), st.Detail)
|
||
w(" stage_text=%q", st.Text)
|
||
}
|
||
}
|
||
|
||
w("-- chunk_status --")
|
||
css, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
sort.Slice(css, func(i, j int) bool {
|
||
a, c := css[i], css[j]
|
||
if a.Chapter != c.Chapter {
|
||
return a.Chapter < c.Chapter
|
||
}
|
||
if a.ChunkIdx != c.ChunkIdx {
|
||
return a.ChunkIdx < c.ChunkIdx
|
||
}
|
||
return a.Stage < c.Stage
|
||
})
|
||
for _, cs := range css {
|
||
w("ch%d/%d %s snap_match=%t content_hash=%s disp=%s flag=%q attempts=%d final_hash=%s cost=%s escalated=%t esc_model=%q detail=%q",
|
||
cs.Chapter, cs.ChunkIdx, cs.Stage, cs.SnapshotID == snapID, cs.ContentHash, cs.Disposition,
|
||
cs.FlagReason, cs.Attempts, cs.FinalHash, fl(cs.CostUSD), cs.Escalated, cs.EscalationModel, cs.Detail)
|
||
}
|
||
|
||
w("-- request_log (insertion order) --")
|
||
rls, err := r.Store.RequestLogRows(r.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, rl := range rls {
|
||
w("ch%d/%d %s role=%s req=%s actual=%s hash=%s tm_hit=%d ok=%d finish=%q degraded=%q cost=%s tokens=%d/%d/%d/%d/%d err=%q",
|
||
rl.Chapter, rl.ChunkIdx, rl.Stage, rl.Role, rl.ModelRequested, rl.ModelActual, rl.RequestHash,
|
||
rl.TMHit, rl.OK, rl.FinishReason, rl.Degraded, fl(rl.CostUSD),
|
||
rl.PromptTokens, rl.CachedTokens, rl.CacheCreationTokens, rl.CompletionTokens, rl.ReasoningTokens, rl.Err)
|
||
}
|
||
|
||
w("-- retrieval_state --")
|
||
rss, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
sort.Slice(rss, func(i, j int) bool {
|
||
if rss[i].Chapter != rss[j].Chapter {
|
||
return rss[i].Chapter < rss[j].Chapter
|
||
}
|
||
return rss[i].ChunkIdx < rss[j].ChunkIdx
|
||
})
|
||
for _, rs := range rss {
|
||
w("ch%d/%d snap_match=%t exact=%d sticky=%d ambiguous=%d spoiler=%d evicted=%d postcheck_miss=%d style_flags=%d",
|
||
rs.Chapter, rs.ChunkIdx, rs.SnapshotID == snapID, rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged,
|
||
rs.NSpoilerBlocked, rs.NEvicted, rs.NPostcheckMiss, rs.NStyleFlags)
|
||
w(" injected_ids=%s postcheck_detail=%s style_detail=%s", rs.InjectedIDs, rs.PostcheckDetail, rs.StyleDetail)
|
||
}
|
||
|
||
w("-- wire bodies (call order) --")
|
||
for i, body := range wireBodies {
|
||
w("[%d] %s", i, body)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// TestGoldenDeterminism is the refactor guard: fresh run + resume run against the
|
||
// static fixture must match testdata/golden/capture.golden byte-for-byte.
|
||
func TestGoldenDeterminism(t *testing.T) {
|
||
rec := &reqRec{}
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
body, _ := io.ReadAll(r.Body)
|
||
rec.record(string(body))
|
||
text, finish := goldenRespond(string(body))
|
||
tb, _ := json.Marshal(text)
|
||
var req struct {
|
||
Model string `json:"model"`
|
||
}
|
||
_ = json.Unmarshal(body, &req)
|
||
fmt.Fprintf(w, `{"id":"fake","model":%q,"choices":[{"message":{"content":%s},"finish_reason":%q}],
|
||
"usage":{"prompt_tokens":1000,"completion_tokens":500,"prompt_tokens_details":{"cached_tokens":200}}}`,
|
||
req.Model, tb, finish)
|
||
}))
|
||
defer srv.Close()
|
||
bookPath := setupGoldenProject(t, srv.URL)
|
||
ctx := obs.WithReqInfo(t.Context(), obs.ReqInfo{TraceID: "golden-trace"})
|
||
|
||
// Run 1: fresh book, every call is a fresh reserve→call→settle.
|
||
r1 := newRunner(t, bookPath)
|
||
res1, err := r1.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
capture := captureGolden(t, "1 (fresh)", r1, res1, rec.all())
|
||
freshCalls := rec.count()
|
||
r1.Close()
|
||
|
||
// Run 2: resume from checkpoints/chunk_status — must make ZERO provider calls
|
||
// and reproduce the identical texts/verdicts (the wire-bodies section stays as
|
||
// captured in run 1; a resume that re-called the provider would append to it).
|
||
r2 := newRunner(t, bookPath)
|
||
defer r2.Close()
|
||
res2, err := r2.TranslateBook(ctx)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if rec.count() != freshCalls {
|
||
t.Fatalf("resume made %d extra provider calls", rec.count()-freshCalls)
|
||
}
|
||
capture += captureGolden(t, "2 (resume)", r2, res2, nil)
|
||
|
||
if os.Getenv("TM_UPDATE_GOLDEN") == "1" {
|
||
if err := os.WriteFile(goldenFile, []byte(capture), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Logf("golden updated: %s (%d bytes, %d provider calls)", goldenFile, len(capture), freshCalls)
|
||
return
|
||
}
|
||
want, err := os.ReadFile(goldenFile)
|
||
if err != nil {
|
||
t.Fatalf("golden file missing (generate once with TM_UPDATE_GOLDEN=1): %v", err)
|
||
}
|
||
if string(want) != capture {
|
||
t.Fatalf("golden mismatch: the run's bytes diverged from %s.\nЭто значит, что рефакторинг изменил wire-байты, хеши или вердикты — "+
|
||
"любое такое изменение = --resnapshot = переоплата книги (D15). Найди и убери причину; обновлять golden можно только "+
|
||
"на осознанной, ратифицированной смене поведения.\n%s", goldenFile, diffHint(string(want), capture))
|
||
}
|
||
}
|
||
|
||
// diffHint points a human at the first diverging line — the full capture is too big
|
||
// for a useful t.Fatalf dump.
|
||
func diffHint(want, got string) string {
|
||
wl, gl := strings.Split(want, "\n"), strings.Split(got, "\n")
|
||
n := len(wl)
|
||
if len(gl) < n {
|
||
n = len(gl)
|
||
}
|
||
for i := 0; i < n; i++ {
|
||
if wl[i] != gl[i] {
|
||
return fmt.Sprintf("first divergence at line %d:\n want: %s\n got: %s", i+1, wl[i], gl[i])
|
||
}
|
||
}
|
||
return fmt.Sprintf("line counts differ: want %d, got %d", len(wl), len(gl))
|
||
}
|