From dc30c7d61d76fe492cf56e3f50168272023d6b5a Mon Sep 17 00:00:00 2001 From: "Claude (backend session)" Date: Fri, 17 Jul 2026 02:16:48 +0300 Subject: [PATCH] Land track-A pack-1.5 with fix-list F1-F9: tmctl export surface with manifest and drift guards, fold-first sanitizer detection, bounded CJK-gloss whitelist, runnable zh-ru example --- backend/cmd/tmctl/export_cli_test.go | 73 +++++ backend/cmd/tmctl/invocation.go | 16 +- backend/cmd/tmctl/invocation_test.go | 4 +- backend/cmd/tmctl/main.go | 23 +- backend/cmd/tmctl/render.go | 55 +++- backend/example/book.yaml | 28 +- backend/example/chapter1-zh.txt | 11 + backend/internal/config/example_test.go | 56 ++++ backend/internal/pipeline/export.go | 206 ++++++++++++ backend/internal/pipeline/export_test.go | 305 +++++++++++++++++ backend/internal/pipeline/quality.go | 37 ++- backend/internal/pipeline/quality_test.go | 42 ++- backend/internal/pipeline/resume.go | 4 + backend/internal/pipeline/sanitizer.go | 306 +++++++++++++++--- backend/internal/pipeline/sanitizer_test.go | 132 +++++++- .../pipeline/testdata/golden/capture.golden | 174 +++++----- 16 files changed, 1303 insertions(+), 169 deletions(-) create mode 100644 backend/cmd/tmctl/export_cli_test.go create mode 100644 backend/example/chapter1-zh.txt create mode 100644 backend/internal/config/example_test.go create mode 100644 backend/internal/pipeline/export.go create mode 100644 backend/internal/pipeline/export_test.go diff --git a/backend/cmd/tmctl/export_cli_test.go b/backend/cmd/tmctl/export_cli_test.go new file mode 100644 index 0000000..9d3d03c --- /dev/null +++ b/backend/cmd/tmctl/export_cli_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "textmachine/backend/internal/pipeline" +) + +// export_cli_test.go pins the tmctl surface of the read-only export projection (D39 слой 6): the +// parse of the `export` command + `--plaintext`, the default stable-JSON render, and the human +// plaintext render (banner + text, flagged chunks marked, dropped chunks empty). + +func TestParseExportPlaintext(t *testing.T) { + inv, err := parseInvocation([]string{"export", "--config", "b.yaml", "--plaintext"}, &bytes.Buffer{}) + if err != nil { + t.Fatalf("export --plaintext must parse: %v", err) + } + if inv.cmd != "export" || inv.cfgPath != "b.yaml" || !inv.asPlaintext { + t.Fatalf("inv = %+v", inv) + } + // Default (no flag) is JSON, not plaintext. + inv2, err := parseInvocation([]string{"export", "--config", "b.yaml"}, &bytes.Buffer{}) + if err != nil || inv2.asPlaintext { + t.Fatalf("export default must be JSON (asPlaintext=false): inv=%+v err=%v", inv2, err) + } +} + +func sampleExport() *pipeline.BookExport { + return &pipeline.BookExport{BookID: "b1", Chunks: []pipeline.ChunkExport{ + {Chapter: 1, ChunkIdx: 0, SnapshotID: "snap", Disposition: "ok", FinalText: "Чистый текст."}, + {Chapter: 2, ChunkIdx: 0, SnapshotID: "snap", Disposition: "flagged", FlagReason: "sanitizer_stripped", FinalText: "Очищенный текст."}, + {Chapter: 3, ChunkIdx: 0, SnapshotID: "snap", Disposition: "flagged", FlagReason: "sanitizer_defect", FinalText: ""}, + }} +} + +func TestRenderExportJSONIsStable(t *testing.T) { + var b bytes.Buffer + if err := renderExport(&b, sampleExport(), false); err != nil { + t.Fatal(err) + } + // The default surface is valid JSON the polygon extractor can consume: it round-trips and carries + // every chunk's export bytes and metadata. + var got pipeline.BookExport + if err := json.Unmarshal(b.Bytes(), &got); err != nil { + t.Fatalf("export JSON must round-trip: %v\n%s", err, b.String()) + } + if got.BookID != "b1" || len(got.Chunks) != 3 { + t.Fatalf("round-tripped export = %+v", got) + } + if got.Chunks[1].FinalText != "Очищенный текст." || got.Chunks[1].FlagReason != "sanitizer_stripped" { + t.Fatalf("stripped chunk lost fidelity: %+v", got.Chunks[1]) + } +} + +func TestRenderExportPlaintextBanners(t *testing.T) { + var b bytes.Buffer + if err := renderExport(&b, sampleExport(), true); err != nil { + t.Fatal(err) + } + out := b.String() + for _, want := range []string{ + "=== ГЛАВА 1 ЧАНК 0 ===\nЧистый текст.", + "=== ГЛАВА 2 ЧАНК 0 — flagged (sanitizer_stripped) (утечка вычищена, проверь) ===\nОчищенный текст.", + "=== ГЛАВА 3 ЧАНК 0 — flagged (sanitizer_defect) (не переведён, помечен для человека) ===", + } { + if !strings.Contains(out, want) { + t.Fatalf("plaintext export must contain %q, got:\n%s", want, out) + } + } +} diff --git a/backend/cmd/tmctl/invocation.go b/backend/cmd/tmctl/invocation.go index e9ad929..f503fb9 100644 --- a/backend/cmd/tmctl/invocation.go +++ b/backend/cmd/tmctl/invocation.go @@ -15,11 +15,12 @@ import ( // invocation is one parsed tmctl command line. type invocation struct { - cmd string - cfgPath string - resnapshot bool - asJSON bool - sel pipeline.RedriveSelector + cmd string + cfgPath string + resnapshot bool + asJSON bool + asPlaintext bool + sel pipeline.RedriveSelector } // parseInvocation parses os.Args[1:] into an invocation. flagOut receives the @@ -37,7 +38,7 @@ type invocation struct { // main() → exit 1, оставляя 2 эксклюзивным для флагнутых чанков. func parseInvocation(args []string, flagOut io.Writer) (invocation, error) { if len(args) < 1 { - return invocation{}, fmt.Errorf("usage: tmctl --config book.yaml") + return invocation{}, fmt.Errorf("usage: tmctl --config book.yaml") } cmd, rest := args[0], args[1:] @@ -46,6 +47,7 @@ func parseInvocation(args []string, flagOut io.Writer) (invocation, error) { cfgPath := fs.String("config", "", "path to book.yaml") resnapshot := fs.Bool("resnapshot", false, "re-pin existing jobs to the current config snapshot (пере-перевод оплаченных чанков — явное согласие)") asJSON := fs.Bool("json", false, "status: emit the projection as JSON (stable disposition/flag_reason enums) for CI/IDE") + asPlaintext := fs.Bool("plaintext", false, "export: emit the concatenated human text instead of the default stable JSON") chapter := fs.Int("chapter", -1, "redrive: restrict to this chapter (default: any)") chunk := fs.Int("chunk", -1, "redrive: restrict to this chunk index within the chapter (default: any)") reason := fs.String("reason", "", "redrive: restrict to this flag_reason (default: any)") @@ -57,7 +59,7 @@ func parseInvocation(args []string, flagOut io.Writer) (invocation, error) { return invocation{}, fmt.Errorf("--config book.yaml is required") } return invocation{ - cmd: cmd, cfgPath: *cfgPath, resnapshot: *resnapshot, asJSON: *asJSON, + cmd: cmd, cfgPath: *cfgPath, resnapshot: *resnapshot, asJSON: *asJSON, asPlaintext: *asPlaintext, sel: pipeline.RedriveSelector{ Chapter: *chapter, ChunkIdx: *chunk, Reason: *reason, DryRun: *dryRun, }, diff --git a/backend/cmd/tmctl/invocation_test.go b/backend/cmd/tmctl/invocation_test.go index c33be93..c2b9eec 100644 --- a/backend/cmd/tmctl/invocation_test.go +++ b/backend/cmd/tmctl/invocation_test.go @@ -16,7 +16,9 @@ import ( func TestParseNoArgsUsage(t *testing.T) { _, err := parseInvocation(nil, &bytes.Buffer{}) - if err == nil || err.Error() != "usage: tmctl --config book.yaml" { + // The command list gained `export` (D39 слой 6 read-only surface, a deliberate contract extension); + // the rest of the usage text stays frozen. + if err == nil || err.Error() != "usage: tmctl --config book.yaml" { t.Fatalf("usage error text is frozen, got: %v", err) } } diff --git a/backend/cmd/tmctl/main.go b/backend/cmd/tmctl/main.go index 2b151c0..de42964 100644 --- a/backend/cmd/tmctl/main.go +++ b/backend/cmd/tmctl/main.go @@ -68,10 +68,12 @@ func run() error { return report(inv.cfgPath) case "status": return status(ctx, inv.cfgPath, inv.asJSON) + case "export": + return export(inv.cfgPath, inv.asPlaintext) case "redrive": return redrive(ctx, inv.cfgPath, inv.resnapshot, inv.sel) default: - return fmt.Errorf("unknown command %q (want translate|report|status|redrive)", inv.cmd) + return fmt.Errorf("unknown command %q (want translate|report|status|export|redrive)", inv.cmd) } } @@ -118,6 +120,25 @@ func report(cfgPath string) error { return renderQuality(os.Stdout, q) } +// export prints the READ-ONLY export projection (D39 слой 6 / D39.2-open №1): every chunk's FINAL +// export text EXACTLY by prod semantics (final_hash → checkpoint → exportNormalize), so the polygon +// extractor / reader-samples read the SAME bytes the backend ships instead of the raw checkpoint. Like +// report/status it is $0 and needs no provider keys (D20.4 — an audit surface must not demand keys). +// Default output is stable JSON (the machine surface); --plaintext emits the human concatenation. +func export(cfgPath string, asPlaintext bool) error { + r, err := pipeline.NewReadOnlyRunner(cfgPath, obs.NewLogger()) + if err != nil { + return err + } + defer r.Close() + + exp, err := r.Export() + if err != nil { + return err + } + return renderExport(os.Stdout, exp, asPlaintext) +} + // status prints the READ-ONLY progress projection (D15.3): 0 LLM, 0 replay. --json emits the // StatusReport with stable disposition/flag_reason enums for CI/IDE; the default is a human // dashboard (unit counts, per-chapter quality passports, money, secondary ETA). BOTH modes exit 2 diff --git a/backend/cmd/tmctl/render.go b/backend/cmd/tmctl/render.go index d343404..aa648ff 100644 --- a/backend/cmd/tmctl/render.go +++ b/backend/cmd/tmctl/render.go @@ -223,11 +223,62 @@ func renderQuality(w io.Writer, q *pipeline.QualityReport) error { q.MeanSentPerNarrPara, q.NarrativeSentences, q.NarrativeParagraphs) fmt.Fprintf(w, "СИГНАЛЫ: диалог-тире=%d · глоссарий-промахов=%d · число-дрейф=%d · trust-gated(сид)=%d\n", q.DialogueDashFlags, q.GlossaryMisses, q.NumberDriftFlags, q.TrustGated) - fmt.Fprintf(w, "УТЕЧКИ: CJK-strip чанков=%d (%.1f%%) · echo(cjk_artifact) чанков=%d (%.1f%%)\n", - q.CJKLeakChunks, 100*q.CJKLeakRate, q.EchoChunks, 100*q.EchoRate) + fmt.Fprintf(w, "СТРИПЫ/УТЕЧКИ: косметик-strip чанков=%d (%.1f%%, markdown+CJK) · echo(cjk_artifact) чанков=%d (%.1f%%)\n", + q.CosmeticStripChunks, 100*q.CosmeticStripRate, q.EchoChunks, 100*q.EchoRate) return nil } +// renderExport prints the read-only export projection (D39 слой 6 / D39.2-open №1): every chunk's +// FINAL export text EXACTLY as `tmctl translate` ships it (export-normalised), for the polygon +// extractor / reader-samples to read the same bytes the backend does instead of the raw checkpoint. +// The DEFAULT is stable JSON (the machine surface the extractor consumes); --plaintext emits the human +// concatenation (a per-chunk header banner + the text, flagged chunks marked). Pure $0 read — never an +// exit-2 sentinel (unlike translate/status): export is an audit projection, not a run verdict. +func renderExport(w io.Writer, exp *pipeline.BookExport, asPlaintext bool) error { + if asPlaintext { + // Manifest/drift summary first (F3/F4): a partial or drifted book is EXPLICIT, not silently + // exported as complete. + fmt.Fprintf(w, "# export %s — чанков всего=%d, экспортировано=%d, ожидают=%d", + exp.BookID, exp.TotalChunks, exp.TotalChunks-exp.PendingChunks, exp.PendingChunks) + if exp.GhostRows > 0 { + fmt.Fprintf(w, ", ghost-строк-отброшено=%d", exp.GhostRows) + } + if exp.ConfigDrift { + fmt.Fprintf(w, " ⚠ CONFIG-DRIFT (текущий конфиг рендерит другой снапшот — гейт/стадия могли измениться после прогона; %.12s)", exp.CurrentSnapshot) + } + fmt.Fprintln(w) + for _, ce := range exp.Chunks { + switch { + case ce.Disposition == "pending": + fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — pending (ещё не переведён) ===\n", ce.Chapter, ce.ChunkIdx) + case ce.Disposition == string(pipeline.DispOK): + fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d ===\n", ce.Chapter, ce.ChunkIdx) + case ce.FinalText != "": + // Cosmetic sanitizer strip (D35.4a): auto-cleaned remainder, flagged for a human. + fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — %s%s (утечка вычищена, проверь) ===\n", + ce.Chapter, ce.ChunkIdx, ce.Disposition, flagParen(ce.FlagReason)) + default: + // Substantive flag / upstream skip: no export text (D2 — contaminated output never ships). + fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — %s%s (не переведён, помечен для человека) ===\n", + ce.Chapter, ce.ChunkIdx, ce.Disposition, flagParen(ce.FlagReason)) + } + fmt.Fprintln(w, ce.FinalText) + } + return nil + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(exp) +} + +// flagParen renders a non-empty flag reason as " (reason)" for the plaintext export banner. +func flagParen(r string) string { + if r == "" { + return "" + } + return " (" + r + ")" +} + // renderStatusJSON emits the StatusReport as indented JSON (stable enums — the // ratified CI/IDE contract, D12/D15.3) and maps flagged>0 to the exit-2 sentinel. func renderStatusJSON(w io.Writer, rep *pipeline.StatusReport) error { diff --git a/backend/example/book.yaml b/backend/example/book.yaml index 1dd2733..4feef09 100644 --- a/backend/example/book.yaml +++ b/backend/example/book.yaml @@ -1,21 +1,35 @@ -# Пример проекта книги для приёмки Фазы 0: один чанк ja→ru через C1. -book_id: sample-ja -title: 夜明けの図書館 # «Библиотека на рассвете» — собственный тестовый фрагмент -source_lang: ja +# Пример проекта книги: один чанк zh→ru через C1 — прод-пара пайплайна (pipeline-c1 несёт +# промпты, ключёванные по паре, только для zh-ru: D39 слой 2). Собственный тестовый фрагмент. +book_id: sample-zh +title: 黎明藏书阁 # «Книгохранилище на рассвете» — собственный тестовый фрагмент +source_lang: zh target_lang: ru genre: ранобэ audience: взрослые читатели вебновелл adult: false venuti: 0.6 honorifics: keep -transcription: polivanov +transcription: palladius # zh→ru: система Палладия (пиньинь — латинская, не для ru-таргета) footnotes: minimal pipeline: ../configs/pipeline-c1.yaml models: ../configs/models.yaml -source_file: chapter1-chunk0.txt -project_db: sample-ja.db +source_file: chapter1-zh.txt +project_db: sample-zh.db ceilings: book_usd: 1.00 # потолок $ на книгу — ledger обязан работать и в тестах day_usd: 2.00 + +# --- Демонстрация pair-сеама (D39 слой 2), НЕ активная конфигурация ------------------ +# pipeline-c1 несёт промпты только для пары zh-ru (conventions написаны под китайский +# исходник). Книга с source_lang: ja даёт LangPair() = "ja-ru", которого в паке нет, и +# загрузка ПАДАЕТ ГРОМКО на loadTemplates (никогда молча не подставляем чужую пару). Чтобы +# увидеть сеам, поменяйте выше на блок ниже (ja-фрагмент лежит в chapter1-chunk0.txt): +# source_lang: ja +# target_lang: ru +# transcription: polivanov +# source_file: chapter1-chunk0.txt +# Запуск такого конфига fail-loud'ится с «no prompt for language pair "ja-ru"» — это +# сигнал добавить ja-ru пакет промптов, а НЕ баг. ja-ru пакет здесь НЕ сочиняется: +# контент пары = данные под реальную книгу (инвариант общности §0.1). diff --git a/backend/example/chapter1-zh.txt b/backend/example/chapter1-zh.txt new file mode 100644 index 0000000..71b51ee --- /dev/null +++ b/backend/example/chapter1-zh.txt @@ -0,0 +1,11 @@ +黎明前的藏书阁,被一片寂静笼罩着。 + +「师兄,我们真的可以进来吗?」美樱压低声音问道。她的手里,紧握着一把古旧的铜钥匙。 + +「放心,长老已经准许了,」拓海答道,「趁天亮之前,把那卷古籍找出来。」 + +两人在书架之间静静穿行。窗外,东方的天际泛起一丝鱼肚白。尘埃的气味,与旧纸的清香,在空气中浮动。美樱停下脚步,向一本书伸出手去。 + +「师兄……你看这个。封面上什么都没有写。」 + +拓海回过头的刹那,藏书阁深处传来一声轻响,仿佛有什么东西坠落了。 diff --git a/backend/internal/config/example_test.go b/backend/internal/config/example_test.go new file mode 100644 index 0000000..cf4beb9 --- /dev/null +++ b/backend/internal/config/example_test.go @@ -0,0 +1,56 @@ +package config + +import ( + "os" + "testing" +) + +// example_test.go pins that the shipped example project (example/book.yaml) is RUNNABLE through the +// pair-keyed prompt seam (D39 слой 2 / D39.2 T3). Reframed to the prod pair zh→ru, LoadBook + +// LoadPipeline resolve, every stage's prompt resolves for the book's pair and exists on disk, and a +// ja-ru pair still FAILS LOUD — the seam the example's commented-out ja block documents. The old ja +// framing regressed to a fail-loud after the pair seam landed (D39.2-open №4); this is the guard that +// the reframe fixed it without silently smuggling a ja book through the zh-ru conventions. +func TestExampleProjectIsRunnableZhRu(t *testing.T) { + book, err := LoadBook("../../example/book.yaml") + if err != nil { + t.Fatalf("example book must load (source_file etc. must exist): %v", err) + } + if book.LangPair() != "zh-ru" { + t.Fatalf("example must be reframed to the prod pair zh→ru, got LangPair() = %q", book.LangPair()) + } + models, err := LoadModels(book.ModelsFile) + if err != nil { + t.Fatalf("example models must load: %v", err) + } + pipe, err := LoadPipeline(book.Pipeline, models) + if err != nil { + t.Fatalf("example pipeline must load: %v", err) + } + if len(pipe.Stages) == 0 { + t.Fatal("example pipeline has no stages") + } + // zh-ru: every stage resolves a prompt for the book's pair AND the resolved file exists (no + // fail-loud, no dangling path) — the exact end-to-end the pair seam broke for the old ja framing. + for _, st := range pipe.Stages { + p, err := st.PromptPathFor(book.LangPair()) + if err != nil { + t.Fatalf("stage %q: zh-ru prompt must resolve: %v", st.Name, err) + } + if _, err := os.Stat(p); err != nil { + t.Fatalf("stage %q: resolved prompt %q must exist on disk: %v", st.Name, p, err) + } + } + // ja-ru: the seam still bites a pair the pack does not carry (what the commented-out ja block in + // example/book.yaml documents) — loadTemplates fails on the first such stage, so at least one must. + jaFailed := false + for _, st := range pipe.Stages { + if _, err := st.PromptPathFor("ja-ru"); err != nil { + jaFailed = true + break + } + } + if !jaFailed { + t.Fatal("a ja-ru pair must fail loud through the example pipeline (the pair seam)") + } +} diff --git a/backend/internal/pipeline/export.go b/backend/internal/pipeline/export.go new file mode 100644 index 0000000..8d88409 --- /dev/null +++ b/backend/internal/pipeline/export.go @@ -0,0 +1,206 @@ +package pipeline + +import ( + "fmt" + + "textmachine/backend/internal/store" +) + +// export.go: the read-only EXPORT projection (D39 слой 6 / D39.2-open №1; the deferred `tmctl export` +// of D15.2-этап-Б in its MINIMAL read form). It emits every chunk's FINAL export text EXACTLY by prod +// semantics, so the polygon extractor (records.json / exp12_extract) and reader-samples read the SAME +// export-normalised bytes `tmctl translate` ships — instead of the RAW stored checkpoint, which skips +// the export contract for a clean chunk. The owner chose a BACKEND surface over a Python mirror of +// exportNormalize (which would drift against the Go contract). It is a PURE READ-ONLY projection like +// Status/QualityReport: $0, no LLM, no keys, no snapshot touch — it reads the persisted chunk_status + +// the $0 final checkpoint and applies the very same exportNormalize the ChunkOutcome does. Only the READ +// half is built here; the annotation/override half of the D15.2 `export` command is deliberately NOT. + +// exportPending marks a manifest chunk that has not (yet) reached the final stage — no exported text. +const exportPending = "pending" + +// ChunkExport is one chunk's final export record: the metadata plus the export-normalised final text. +type ChunkExport struct { + Chapter int `json:"chapter"` + ChunkIdx int `json:"chunk_idx"` + SnapshotID string `json:"snapshot_id,omitempty"` + // Disposition is the CHUNK-level verdict mirroring ChunkOutcome: "ok" only when the final stage is + // ok; "flagged" for a flagged OR upstream-skipped final row; "pending" for a manifest chunk with no + // final-stage row (not yet translated). + Disposition string `json:"disposition"` + FlagReason string `json:"flag_reason,omitempty"` + Detail string `json:"detail,omitempty"` + // FinalText is the export-normalised final text EXACTLY as `tmctl translate` ships it: + // exportNormalize(final checkpoint response) for an ok or cosmetic-stripped chunk, "" for a + // substantive flag / upstream-skip / pending chunk (exportNormalize("")=="", so the paths coincide + // byte-for-byte with ChunkOutcome.FinalText). + FinalText string `json:"final_text"` +} + +// BookExport is the whole-book export projection (chunks in manifest — (chapter, chunk_idx) — order, so +// the JSON is deterministic across runs). The counters make a partial/empty/drifted book EXPLICIT +// instead of silently exporting as complete (F4/F3). +type BookExport struct { + BookID string `json:"book_id"` + // TotalChunks is the current source manifest size (the honest denominator); ExportedChunks + + // PendingChunks == TotalChunks. GhostRows are stored final rows dropped as OUTSIDE the manifest. + TotalChunks int `json:"total_chunks"` + PendingChunks int `json:"pending_chunks"` + GhostRows int `json:"ghost_rows,omitempty"` + // ConfigDrift is true when the CURRENT config renders a snapshot different from the one the stored + // rows carry (a gate flip / prompt bump / stage rename since the run) — the gate/stage re-derivation + // below may then not match what translate actually did. CurrentSnapshot is that projected id. + ConfigDrift bool `json:"config_drift"` + CurrentSnapshot string `json:"current_snapshot,omitempty"` + // Chunks is always non-nil (an empty book exports [] not null). + Chunks []ChunkExport `json:"chunks"` +} + +// Export builds the read-only per-chunk export projection. It opens no jobs, reserves nothing, makes no +// LLM call — it joins the $0 source manifest against the persisted chunk_status (final-stage rows) and, +// for a chunk that produced exported text, the $0 final checkpoint, applying exportNormalize exactly as +// translateChunk does. Safe to run whenever `report`/`status` are (the same exclusive-lock rule). +// Deterministic over the store + source. +// +// CAVEATS (surfaced, never silent): +// - CONFIG/GATE DRIFT (F3): the gate flag AND the final-stage NAME are read from the CURRENT config. +// A config edit since the run (a glossary post-check gate FLIP, a prompt bump, a final-stage rename) +// changes the projected snapshot; Export sets ConfigDrift + WARNs so the gate re-derivation / row +// lookup mismatch is visible (a run under the same config reads correctly; `status` shows it too). +// - MANIFEST (F4): a chunk absent from the current source is a GHOST row (dropped + counted + WARNed); +// a manifest chunk with no final row is PENDING; both keep a partial/edited book from exporting as +// if complete (the D37-skew the polygon extractor must not inherit). +func (r *Runner) Export() (*BookExport, error) { + statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID) + if err != nil { + return nil, err + } + // The glossary post-check GATE (opt-in) flips a chunk to flagged/glossary_miss at the CHUNK level + // AFTER the stage loop, so it is NEVER written to chunk_status — every stage row stays DispOK + // (chunkrun.translateChunk records only retrieval_state). Export must re-derive it, exactly as + // Status does: a raw chunk_status read would report a gate-flagged chunk as clean ok AND ship the + // contaminated text `tmctl translate` deliberately withheld. Read retrieval_state only when the gate + // is on (else the map stays empty and the branch is a no-op). + gateOn := r.Pipeline.Gates.Glossary.PostcheckGate + missByChunk := map[chunkKey]int{} + if gateOn { + states, err := r.Store.RetrievalStatesForBook(r.Book.BookID) + if err != nil { + return nil, err + } + for _, rs := range states { + missByChunk[chunkKey{rs.Chapter, rs.ChunkIdx}] = rs.NPostcheckMiss // CONFIRMED-miss count + } + } + lastStage := "" + if n := len(r.Pipeline.Stages); n > 0 { + lastStage = r.Pipeline.Stages[n-1].Name + } + // Index the final-stage rows by chunk key (the exported text + verdict live there) and record the + // snapshot ids the rows carry (for the drift check). + finalRow := map[chunkKey]store.ChunkStatus{} + snapSeen := map[string]bool{} + for _, cs := range statuses { + if cs.Stage != lastStage { + continue // the exported text and the chunk's final verdict live on the final stage's row + } + finalRow[chunkKey{cs.Chapter, cs.ChunkIdx}] = cs + if cs.SnapshotID != "" { + snapSeen[cs.SnapshotID] = true + } + } + + // F4: the $0 source manifest is the authoritative chunk set — join it so a PENDING chunk is explicit + // and a GHOST (deleted-source) row is not exported as if live. + manifest, err := r.bookChunks() + if err != nil { + return nil, err + } + exp := &BookExport{BookID: r.Book.BookID, Chunks: []ChunkExport{}} + inManifest := map[chunkKey]bool{} + for _, ch := range manifest { + k := chunkKey{ch.Chapter, ch.ChunkIdx} + inManifest[k] = true + exp.TotalChunks++ + cs, ok := finalRow[k] + if !ok { + exp.PendingChunks++ + exp.Chunks = append(exp.Chunks, ChunkExport{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: exportPending}) + continue + } + ce, err := r.chunkExport(cs, gateOn, missByChunk[k]) + if err != nil { + return nil, err + } + exp.Chunks = append(exp.Chunks, ce) + } + // F4: GHOST rows — a stored final row whose chunk is NOT in the current manifest (source shrunk since + // the run). Dropped (never exported as if live) but counted + WARNed so the drop is loud. + for k := range finalRow { + if !inManifest[k] { + exp.GhostRows++ + } + } + if exp.GhostRows > 0 { + r.Log.Warn("export: dropped chunk_status rows outside the current source manifest (source shrunk since the run?)", + "book", r.Book.BookID, "ghost_rows", exp.GhostRows) + } + + // F3: config/gate drift — only meaningful on a single stored snapshot (multi-snapshot drift is + // already visible in the per-chunk snapshot_id). Compute the CURRENT config's projected snapshot + // read-only and compare; surface a mismatch as a flag + WARN, never silently. + if len(snapSeen) == 1 { + var stored string + for s := range snapSeen { + stored = s + } + if cur, serr := r.currentSnapshotProjected(); serr != nil { + r.Log.Warn("export: config-drift check failed; drift state unknown (reported as none)", "err", serr) + } else if cur != stored { + exp.ConfigDrift = true + exp.CurrentSnapshot = cur + r.Log.Warn("export: CONFIG-DRIFT — current config renders a different snapshot than the stored rows; the gate/stage re-derivation may not match the run (translate would require --resnapshot)", + "book", r.Book.BookID, "stored", stored, "current", cur) + } + } + return exp, nil +} + +// chunkExport computes one chunk's export record from its final-stage row, exactly by prod FinalText +// semantics (chunkrun.translateChunk): an ok chunk exports exportNormalize(primary checkpoint), a +// cosmetic sanitizer-strip exports exportNormalize(its derived stripped checkpoint) — both via +// final_hash — the glossary gate override exports "", and every other flag / upstream skip exports "". +func (r *Runner) chunkExport(cs store.ChunkStatus, gateOn bool, gateMiss int) (ChunkExport, error) { + ce := ChunkExport{Chapter: cs.Chapter, ChunkIdx: cs.ChunkIdx, SnapshotID: cs.SnapshotID, Detail: cs.Detail} + // Post-check gate override (mirrors translateChunk): an otherwise-ok chunk (DispOK ⇒ every stage ok) + // with a CONFIRMED miss is flagged glossary_miss and exports NOTHING — the withheld text never ships. + if gateOn && cs.Disposition == string(DispOK) && gateMiss > 0 { + ce.Disposition = string(DispFlagged) + ce.FlagReason = string(FlagGlossaryMiss) + return ce, nil + } + if cs.Disposition == string(DispOK) { + ce.Disposition = string(DispOK) + if cs.FinalHash == "" { + // F9: an ok chunk MUST carry a final_hash — an empty one is an inconsistent store. Fail LOUD + // (parity with the missing-checkpoint case), never silently export "". + return ce, fmt.Errorf("pipeline: export ch%d/chunk%d: DispOK row has an empty final_hash (inconsistent store)", cs.Chapter, cs.ChunkIdx) + } + } else { + // A flagged final row OR a skipped one (an upstream stage flagged) is a flagged CHUNK; the final + // row carries the propagated flag reason (chunkrun records it on the skipped row too). + ce.Disposition = string(DispFlagged) + ce.FlagReason = cs.FlagReason + } + if cs.FinalHash != "" && (cs.Disposition == string(DispOK) || cs.FlagReason == string(FlagSanitizerStripped)) { + cp, err := r.Store.GetCheckpoint(cs.FinalHash) + if err != nil { + return ce, err + } + if cp == nil { + return ce, fmt.Errorf("pipeline: export ch%d/chunk%d: chunk_status.final_hash %.12s has no checkpoint (inconsistent store)", cs.Chapter, cs.ChunkIdx, cs.FinalHash) + } + ce.FinalText = exportNormalize(cp.ResponseText) + } + return ce, nil +} diff --git a/backend/internal/pipeline/export_test.go b/backend/internal/pipeline/export_test.go new file mode 100644 index 0000000..6bc9739 --- /dev/null +++ b/backend/internal/pipeline/export_test.go @@ -0,0 +1,305 @@ +package pipeline + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "textmachine/backend/internal/obs" +) + +// export_test.go pins the read-only export surface (D39 слой 6 / D39.2-open №1): `tmctl export` must +// emit each chunk's FINAL text EXACTLY as `tmctl translate` ships it (export-normalised), for BOTH a +// fresh run and a $0 resume — the whole point being that the polygon extractor reads the SAME bytes +// the backend does, not the RAW stored checkpoint (which skips the export contract for a clean chunk). + +// TestExportMatchesTranslateFinalText runs a real 3-chunk book across the three export outcomes and +// asserts Export() reproduces BookResult's FinalText byte-for-byte — and that a clean chunk carrying a +// normalize-relevant artifact is exported NORMALISED (not the raw checkpoint), which is the fix. +func TestExportMatchesTranslateFinalText(t *testing.T) { + rec := &reqRec{} + // Three chapters (\f-separated) → three 1-chunk chapters. The editor (final stage) emits: + // ch1: clean prose carrying a U+3000 indent + fullwidth digits → OK, but export must NORMALISE it; + // ch2: a sparse contentless CJK-leak run (below the echo threshold) → cosmetic sanitizer strip; + // ch3: a leaked service preamble → SUBSTANTIVE sanitizer defect → dropped (FinalText ""). + const ( + src1 = "ГЛАВААА" + src2 = "ГЛАВАББ" + src3 = "ГЛАВАВВ" + ) + editCleanArtifact := "Судзуки шёл по коридору. 123 шага до двери." // U+3000 + «123» + editCosmeticLeak := "Он поднял кулак 羅漢 и ударил врага." // sparse Han run → strip + editPreamble := "Вот перевод фрагмента:\nОн молча ушёл в туман." // preamble → dropped + srv := newJSONProvider(rec, func(body string) (string, string) { + if !isEditBody(body) { + return "черновик перевода этой главы.", "stop" // any draft — clean, non-echo + } + switch { + case strings.Contains(body, src1): + return editCleanArtifact, "stop" + case strings.Contains(body, src2): + return editCosmeticLeak, "stop" + default: // src3 + return editPreamble, "stop" + } + }) + defer srv.Close() + + // Enable the output-sanitizer so the cosmetic-strip (ch2) and substantive-drop (ch3) paths engage. + bookPath := setupProjectOpts(t, srv.URL, projectOpts{ + source: src1 + "\f" + src2 + "\f" + src3, + gatesYAML: "gates:\n sanitizer:\n enabled: true\n", + }) + ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) + + r1 := newRunner(t, bookPath) + res, err := r1.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + if len(res.Chunks) != 3 { + t.Fatalf("want 3 chunks, got %d: %+v", len(res.Chunks), res.Chunks) + } + + // The authoritative FinalText per chunk, straight from the translate result. + type want struct { + final string + disp Disposition + flag FlagReason + } + byChunk := map[[2]int]want{} + for _, ch := range res.Chunks { + byChunk[[2]int{ch.Chapter, ch.ChunkIdx}] = want{ch.FinalText, ch.Disposition, ch.FlagReason} + } + + // The clean chunk MUST be export-normalised, not raw: no U+3000, no fullwidth digits survive. + ch1 := byChunk[[2]int{1, 0}] + if strings.ContainsRune(ch1.final, ' ') || strings.Contains(ch1.final, "123") { + t.Fatalf("ch1 FinalText was not export-normalised: %q", ch1.final) + } + if ch1.final != "Судзуки шёл по коридору. 123 шага до двери." { + t.Fatalf("ch1 FinalText = %q, want the normalised form", ch1.final) + } + // The cosmetic-leak chunk is flagged-stripped with the Han run removed but the prose kept. + ch2 := byChunk[[2]int{2, 0}] + if ch2.disp != DispFlagged || ch2.flag != FlagSanitizerStripped || strings.Contains(ch2.final, "羅漢") || ch2.final == "" { + t.Fatalf("ch2 expected stripped-non-empty flagged export, got %+v", ch2) + } + // The preamble chunk is a substantive drop: flagged with an EMPTY export. + ch3 := byChunk[[2]int{3, 0}] + if ch3.disp != DispFlagged || ch3.final != "" { + t.Fatalf("ch3 expected substantive drop with empty export, got %+v", ch3) + } + + assertExportEquals := func(tag string, r *Runner) { + t.Helper() + exp, err := r.Export() + if err != nil { + t.Fatalf("%s: Export: %v", tag, err) + } + if len(exp.Chunks) != 3 { + t.Fatalf("%s: want 3 export chunks, got %d", tag, len(exp.Chunks)) + } + for _, ce := range exp.Chunks { + w, ok := byChunk[[2]int{ce.Chapter, ce.ChunkIdx}] + if !ok { + t.Fatalf("%s: export emitted an unknown chunk %d/%d", tag, ce.Chapter, ce.ChunkIdx) + } + if ce.FinalText != w.final { + t.Errorf("%s: ch%d/%d export text = %q, want translate FinalText %q", tag, ce.Chapter, ce.ChunkIdx, ce.FinalText, w.final) + } + if ce.Disposition != string(w.disp) || ce.FlagReason != string(w.flag) { + t.Errorf("%s: ch%d/%d export disp/flag = %s/%s, want %s/%s", tag, ce.Chapter, ce.ChunkIdx, ce.Disposition, ce.FlagReason, w.disp, w.flag) + } + } + } + + assertExportEquals("fresh", r1) + r1.Close() + + // Resume (a new process): the book is served from checkpoints at $0, and Export must reproduce the + // identical bytes — the derived stripped checkpoint and the raw ones both re-normalise deterministically. + callsBefore := rec.count() + r2 := newRunner(t, bookPath) + defer r2.Close() + res2, err := r2.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + if rec.count() != callsBefore { + t.Fatalf("resume must not call the provider again: %d -> %d", callsBefore, rec.count()) + } + if res2.TotalUSD != 0 { + t.Fatalf("resume must be $0, got %v", res2.TotalUSD) + } + assertExportEquals("resume", r2) +} + +// TestExportGlossaryGateWithheld pins the chunk-level glossary post-check gate case (adversarial-review +// finding): the gate flips a chunk to flagged/glossary_miss AFTER the stage loop and never writes it to +// chunk_status, so a raw chunk_status read would report the chunk as clean ok and ship the text +// `translate` withheld. Export must re-derive the gate flag from retrieval_state (like Status) so it +// matches translate byte-for-byte: flagged, glossary_miss, EMPTY export. +func TestExportGlossaryGateWithheld(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isEditBody(body) { + return "ОТРЕДАКТИРОВАННЫЙ.", "stop" // final text drops the approved name → confirmed miss + } + return "Некто пошёл в библиотеку.", "stop" + }) + defer srv.Close() + // Opt-in hard gate + a seed whose approved 鈴木→Судзуки is missed in the final. + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true}) + ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) + + r := newRunner(t, bookPath) + defer r.Close() + res, err := r.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + // Precondition: translate withheld this chunk (flagged glossary_miss, empty text). + if len(res.Chunks) != 1 || res.Chunks[0].FlagReason != FlagGlossaryMiss || res.Chunks[0].FinalText != "" { + t.Fatalf("precondition: want 1 chunk flagged glossary_miss with empty text, got %+v", res.Chunks) + } + exp, err := r.Export() + if err != nil { + t.Fatalf("Export: %v", err) + } + if len(exp.Chunks) != 1 { + t.Fatalf("want 1 export chunk, got %d", len(exp.Chunks)) + } + ce := exp.Chunks[0] + // Export MUST match translate — NOT ship the withheld text as clean ok. + if ce.Disposition != string(DispFlagged) || ce.FlagReason != string(FlagGlossaryMiss) || ce.FinalText != "" { + t.Fatalf("export leaked a gate-withheld chunk: disp=%s reason=%s final=%q (want flagged/glossary_miss/\"\")", + ce.Disposition, ce.FlagReason, ce.FinalText) + } +} + +// TestExportManifestPending pins F4: a PARTIAL book (a ceiling stopped the run mid-way) exports the +// untranslated chunks as explicit "pending" rows against the $0 manifest — not silently as a complete +// book (the D37-skew the polygon extractor must not inherit). Chunks is non-nil. +func TestExportManifestPending(t *testing.T) { + srv := newJSONProvider(&reqRec{}, draftEdit) + defer srv.Close() + // Three chapters; a book ceiling that pays for ~one chapter (2 calls ≈ $0.0036) then denies. + bookPath := setupProjectOpts(t, srv.URL, projectOpts{ + source: "ГЛАВАА\fГЛАВАБ\fГЛАВАВ", regenerate: 0, bookUSD: 0.005, + }) + ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) + + r := newRunner(t, bookPath) + defer r.Close() + // The ceiling stop is an infra error — expected; we only need the partial store state it leaves. + _, _ = r.TranslateBook(ctx) + + exp, err := r.Export() + if err != nil { + t.Fatalf("Export: %v", err) + } + if exp.TotalChunks != 3 { + t.Fatalf("manifest must be 3 chunks, got total=%d", exp.TotalChunks) + } + if exp.PendingChunks < 1 { + t.Fatalf("a ceiling-stopped book must have pending chunks, got pending=%d (%+v)", exp.PendingChunks, exp.Chunks) + } + if len(exp.Chunks) != exp.TotalChunks { + t.Fatalf("every manifest chunk must have a row: len=%d total=%d", len(exp.Chunks), exp.TotalChunks) + } + var pending int + for _, ce := range exp.Chunks { + if ce.Disposition == "pending" { + pending++ + if ce.FinalText != "" { + t.Errorf("a pending chunk must export no text: %+v", ce) + } + } + } + if pending != exp.PendingChunks { + t.Errorf("pending rows (%d) must match the counter (%d)", pending, exp.PendingChunks) + } +} + +// TestExportDetectsConfigDrift pins F3: after a wire-affecting config edit (a prompt_version bump) since +// the run, Export flags ConfigDrift — the gate/stage re-derivation may not match what translate did. +func TestExportDetectsConfigDrift(t *testing.T) { + srv := newJSONProvider(&reqRec{}, draftEdit) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0}) + ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) + + r1 := newRunner(t, bookPath) + if _, err := r1.TranslateBook(ctx); err != nil { + t.Fatal(err) + } + r1.Close() + + // Unchanged config → no drift. + r2 := newRunner(t, bookPath) + exp, err := r2.Export() + if err != nil { + t.Fatal(err) + } + if exp.ConfigDrift { + t.Fatalf("an unchanged config must not drift (current=%s)", exp.CurrentSnapshot) + } + r2.Close() + + // Bump a wire-affecting field → the current config renders a new snapshot → drift. + dir := filepath.Dir(bookPath) + body, err := os.ReadFile(filepath.Join(dir, "pipeline.yaml")) + if err != nil { + t.Fatal(err) + } + changed := strings.Replace(string(body), "prompt_version: v-test", "prompt_version: v-test2", 1) + if changed == string(body) { + t.Fatal("setup: prompt_version token not found") + } + writeFile(t, filepath.Join(dir, "pipeline.yaml"), changed) + + r3 := newRunner(t, bookPath) + defer r3.Close() + exp3, err := r3.Export() + if err != nil { + t.Fatal(err) + } + if !exp3.ConfigDrift || exp3.CurrentSnapshot == "" { + t.Errorf("a prompt_version bump since the run must surface as ConfigDrift: %+v", exp3) + } +} + +// TestExportFailLoudOnInconsistentStore pins F9: a DispOK final row with an EMPTY final_hash is an +// inconsistent store; Export fails LOUD rather than silently exporting "". +func TestExportFailLoudOnInconsistentStore(t *testing.T) { + srv := newJSONProvider(&reqRec{}, draftEdit) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0}) + ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()}) + + r := newRunner(t, bookPath) + defer r.Close() + res, err := r.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + // Corrupt the final-stage row: DispOK but no final_hash (simulating store inconsistency). + lastStage := r.Pipeline.Stages[len(r.Pipeline.Stages)-1].Name + ch := res.Chunks[0] + cs, err := r.Store.GetChunkStatus(r.Book.BookID, ch.Chapter, ch.ChunkIdx, lastStage) + if err != nil || cs == nil { + t.Fatalf("read final row: %v cs=%v", err, cs) + } + cs.FinalHash = "" + if err := r.Store.UpsertChunkStatus(*cs); err != nil { + t.Fatal(err) + } + if _, err := r.Export(); err == nil { + t.Fatal("Export must fail loud on a DispOK row with an empty final_hash") + } else if !strings.Contains(err.Error(), "empty final_hash") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/backend/internal/pipeline/quality.go b/backend/internal/pipeline/quality.go index 116198e..523caff 100644 --- a/backend/internal/pipeline/quality.go +++ b/backend/internal/pipeline/quality.go @@ -36,17 +36,19 @@ type QualityReport struct { MeanSentPerNarrPara float64 `json:"mean_sentences_per_narrative_paragraph"` // Deterministic signal aggregates (all observability, never a disposition). - DialogueDashFlags int `json:"dialogue_dash_flags"` // Rosenthal dialogue-dash inconsistencies - GlossaryMisses int `json:"glossary_misses"` // CONFIRMED post-check misses (D10 consistency) - NumberDriftFlags int `json:"number_drift_flags"` // reflow number drift + 万/億 magnitude drift - TrustGated int `json:"trust_gated"` // lower-trust suppressions refused (seed hygiene) - CJKLeakChunks int `json:"cjk_leak_chunks"` // chunks the sanitizer stripped a CJK leak from - EchoChunks int `json:"echo_chunks"` // chunks flagged cjk_artifact (untranslated echo) + DialogueDashFlags int `json:"dialogue_dash_flags"` // Rosenthal dialogue-dash inconsistencies + GlossaryMisses int `json:"glossary_misses"` // CONFIRMED post-check misses (D10 consistency) + NumberDriftFlags int `json:"number_drift_flags"` // reflow number drift + 万/億 magnitude drift + TrustGated int `json:"trust_gated"` // lower-trust suppressions refused (seed hygiene) + CosmeticStripChunks int `json:"cosmetic_strip_chunks"` // chunks the sanitizer auto-stripped (markdown header OR CJK leak — F6: was mislabeled cjk_leak) + EchoChunks int `json:"echo_chunks"` // chunks flagged cjk_artifact (untranslated echo) // Rates over ProcessedChunks (0..1) for the two "instant unreadability" families, so the numbers - // read as a bounded fraction of processed chunks rather than a bare count. - CJKLeakRate float64 `json:"cjk_leak_rate"` - EchoRate float64 `json:"echo_rate"` + // read as a bounded fraction of processed chunks rather than a bare count. CosmeticStripRate covers + // BOTH strip classes (a markdown-only strip is NOT a CJK leak — F6, D39.4: the old cjk_leak_rate + // counted every sanitizer_stripped chunk and read as false CJK leakage). + CosmeticStripRate float64 `json:"cosmetic_strip_rate"` + EchoRate float64 `json:"echo_rate"` Chunks []ChunkQuality `json:"chunks,omitempty"` } @@ -114,6 +116,13 @@ func (r *Runner) QualityReport() (*QualityReport, error) { return q } + // The glossary post-check GATE flips a chunk to withheld (flagged glossary_miss, empty export) at + // the CHUNK level without a chunk_status row (like Export/Status re-derive it). F5 (D39.4): the + // structural KPI must EXCLUDE these — their export is "", so counting their (withheld) text in + // TextChunks/KPI diverges from what `tmctl export` and `translate` actually ship. + gateOn := r.Pipeline.Gates.Glossary.PostcheckGate + withheld := map[chunkKey]bool{} + // Aggregate the stored retrieval-state signals (glossary consistency, style breakdown, trust-gated). for _, rs := range states { q := chunkOf(chunkKey{rs.Chapter, rs.ChunkIdx}) @@ -121,6 +130,9 @@ func (r *Runner) QualityReport() (*QualityReport, error) { q.TrustGated += rs.NTrustGatedSuppress rep.GlossaryMisses += rs.NPostcheckMiss rep.TrustGated += rs.NTrustGatedSuppress + if gateOn && rs.NPostcheckMiss > 0 { + withheld[chunkKey{rs.Chapter, rs.ChunkIdx}] = true + } if rs.NStyleFlags > 0 && rs.StyleDetail != "" { var cg cheapGateResult if json.Unmarshal([]byte(rs.StyleDetail), &cg) == nil { @@ -159,7 +171,7 @@ func (r *Runner) QualityReport() (*QualityReport, error) { chunkOf(k) // ensure the chunk row exists even if it has no retrieval-state signals } if cs.FlagReason == string(FlagSanitizerStripped) { - rep.CJKLeakChunks++ // a stripped chunk carried a markdown/CJK cosmetic leak (detail says which) + rep.CosmeticStripChunks++ // a stripped chunk carried a markdown OR CJK cosmetic leak (F6) } // Structural KPI: recompute over the exported final text (ok, or the cosmetic-stripped export). if cs.FinalHash == "" { @@ -168,6 +180,9 @@ func (r *Runner) QualityReport() (*QualityReport, error) { if cs.Disposition != string(DispOK) && cs.FlagReason != string(FlagSanitizerStripped) { continue // a dropped chunk exported nothing } + if withheld[k] { + continue // F5: the glossary gate withheld this chunk's text — it exports nothing + } cp, cperr := r.Store.GetCheckpoint(cs.FinalHash) if cperr != nil { return nil, cperr @@ -187,7 +202,7 @@ func (r *Runner) QualityReport() (*QualityReport, error) { rep.MeanSentPerNarrPara = float64(rep.NarrativeSentences) / float64(rep.NarrativeParagraphs) } if rep.ProcessedChunks > 0 { - rep.CJKLeakRate = float64(rep.CJKLeakChunks) / float64(rep.ProcessedChunks) + rep.CosmeticStripRate = float64(rep.CosmeticStripChunks) / float64(rep.ProcessedChunks) rep.EchoRate = float64(rep.EchoChunks) / float64(rep.ProcessedChunks) } for _, k := range order { diff --git a/backend/internal/pipeline/quality_test.go b/backend/internal/pipeline/quality_test.go index b7e1904..1c903bc 100644 --- a/backend/internal/pipeline/quality_test.go +++ b/backend/internal/pipeline/quality_test.go @@ -66,8 +66,8 @@ func TestQualityReportAggregates(t *testing.T) { t.Fatalf("chunk counts: total=%d processed=%d text=%d, want 1/1/1", q.TotalChunks, q.ProcessedChunks, q.TextChunks) } // Rates are bounded [0,1] over ProcessedChunks (the disjoint-set bug fix). - if q.EchoRate < 0 || q.EchoRate > 1 || q.CJKLeakRate < 0 || q.CJKLeakRate > 1 { - t.Errorf("rates must be within [0,1], got echo=%.3f cjk=%.3f", q.EchoRate, q.CJKLeakRate) + if q.EchoRate < 0 || q.EchoRate > 1 || q.CosmeticStripRate < 0 || q.CosmeticStripRate > 1 { + t.Errorf("rates must be within [0,1], got echo=%.3f cosmetic=%.3f", q.EchoRate, q.CosmeticStripRate) } if q.NarrativeSentences != 3 || q.NarrativeParagraphs != 2 { t.Errorf("structural KPI = %d sent / %d para, want 3/2", q.NarrativeSentences, q.NarrativeParagraphs) @@ -76,10 +76,46 @@ func TestQualityReportAggregates(t *testing.T) { t.Errorf("mean sentences/narrative-paragraph = %.3f, want 1.5", q.MeanSentPerNarrPara) } // A clean run: no glossary misses, no echo, no CJK strip, no trust-gated, no dialogue-dash issue. - if q.GlossaryMisses != 0 || q.EchoChunks != 0 || q.CJKLeakChunks != 0 || q.TrustGated != 0 || q.DialogueDashFlags != 0 { + if q.GlossaryMisses != 0 || q.EchoChunks != 0 || q.CosmeticStripChunks != 0 || q.TrustGated != 0 || q.DialogueDashFlags != 0 { t.Errorf("clean run must have zero defect signals, got %+v", q) } if len(q.Chunks) != 1 || q.Chunks[0].NarrativeParagraphs != 2 { t.Errorf("per-chunk row missing/wrong: %+v", q.Chunks) } } + +// TestQualityReportExcludesGateWithheld pins F5 (D39.4): a chunk the glossary post-check GATE withheld +// (flagged glossary_miss, empty export) must NOT be counted in TextChunks / the structural KPI — its +// text never ships, so counting it would diverge from `tmctl export`/`translate`. +func TestQualityReportExcludesGateWithheld(t *testing.T) { + srv := newJSONProvider(&reqRec{}, func(body string) (string, string) { + if isEditBody(body) { + return "ОТРЕДАКТИРОВАННЫЙ.", "stop" // final drops the approved name → confirmed miss + } + return "Некто пошёл в библиотеку.", "stop" + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true}) + + r := newRunner(t, bookPath) + defer r.Close() + if _, err := r.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + q, err := r.QualityReport() + if err != nil { + t.Fatalf("QualityReport: %v", err) + } + // The single chunk is gate-withheld → processed but NOT text-bearing. + if q.ProcessedChunks != 1 { + t.Fatalf("the withheld chunk still reached the final stage: processed=%d", q.ProcessedChunks) + } + if q.TextChunks != 0 || q.NarrativeSentences != 0 || q.NarrativeParagraphs != 0 { + t.Errorf("gate-withheld chunk must be excluded from the KPI, got text=%d sent=%d para=%d", + q.TextChunks, q.NarrativeSentences, q.NarrativeParagraphs) + } + // It IS visible as a glossary miss (the observability signal is kept). + if q.GlossaryMisses == 0 { + t.Errorf("the confirmed miss must still be reported: %+v", q) + } +} diff --git a/backend/internal/pipeline/resume.go b/backend/internal/pipeline/resume.go index 950f7b3..b361b39 100644 --- a/backend/internal/pipeline/resume.go +++ b/backend/internal/pipeline/resume.go @@ -70,6 +70,10 @@ func (r *Runner) resumeFromChunkStatus(ctx context.Context, st config.Stage, ch // checkpoint, so report ITS finish_reason ("sanitized_export") rather than a bare "" that // contradicts the referenced row (adversarial review NIT; telemetry-only). sr.FinishReason = cp.FinishReason + // F7 (D39.4): mirror the ok branch's model attribution — the derived checkpoint carries the + // ACTUAL model that produced the flagged output; without this the resumed telemetry row is + // attributed to the config slug (st.Model) instead of the model that actually answered. + sr.Model = cp.ModelActual } rl := r.baseRequestLog(st, ch, st.Model, cs.FinalHash) rl.ModelActual = sr.Model diff --git a/backend/internal/pipeline/sanitizer.go b/backend/internal/pipeline/sanitizer.go index 479cec2..fe562a6 100644 --- a/backend/internal/pipeline/sanitizer.go +++ b/backend/internal/pipeline/sanitizer.go @@ -78,7 +78,16 @@ import ( // delegates that fold too. This shifts which chunks flag and a stripped chunk's export → a loud // --resnapshot + golden re-capture (invariant №8). [exportNormalize itself is an EPHEMERAL export // projection, not a persisted input; only stripCosmetic's output rides a checkpoint, versioned here.] -const sanitizerVersion = "sanitizer-v5" +// v6 (D39.2 T2, owner decision 16.07: KEEP glosses; + D39.4 adversarial fix-list F1/F2): (a) a +// whitelisted CJK term gloss — a balanced Han/kana run inside parens DIRECTLY after a Cyrillic/Latin +// head word, «Кулак Ло Ханя (羅漢拳)» — is NOT a leak: detectCJKLeak no longer fires on it and +// stripCosmetic/exportNormalize preserve its bytes; a BARE Han run stays a leak. (b) F1: ALL six classes +// now DETECT over the export-normalised (folded) text, closing the fullwidth-signature gate bypass +// (a «###»/«:» variant used to read clean while export manufactured the ASCII defect). (c) F2: a gloss +// bounds its non-CJK content (inner length, proportional pinyin-token count, per-token length, +// lowercase), so a leaked English clause after a token Han char is not whitelisted. All shift which +// chunks flag and a stripped chunk's export → a loud --resnapshot + golden re-capture (invariant №8). +const sanitizerVersion = "sanitizer-v6" // sanitizer tuning constants (versioned by sanitizerVersion). const ( @@ -137,29 +146,46 @@ func (s sanitizerResult) summary() string { } // sanitizeOutput runs all six classes over one chunk's FINAL translated text. +// +// FOLD-FIRST (F1, D39.4 — adversarial-found gate bypass): ALL six classes DETECT over the +// export-normalised (recoverableFold-ed) text, not the raw text. Otherwise a FULLWIDTH variation of a +// signature slips past the ASCII regexes while exportNormalize still MANUFACTURES the ASCII defect in +// the shipped bytes: «Вот отредактированный перевод:»(U+FF1A fullwidth colon), «### Глава 5» +// (fullwidth hash), a trailing «Примечание:…», fullwidth-Latin — all read total=0 on the raw text yet +// export a real ASCII preamble/header/note/Latin leak (their ASCII twins ARE flagged, so it was a pure +// bypass; the pre-D39 v4 folded the whole fullwidth block and its re-check caught this — a regression). +// Only DETECTION folds; the strip/export paths (stripCosmetic/exportNormalize) still operate on the +// ORIGINAL text so they preserve the original/gloss bytes. Gloss spans are re-anchored consistently: +// the mask and detectCJKLeak both re-derive glossSegments over the SAME folded text they scan. +// recoverableFold is idempotent, so detectCJKLeak's own internal fold is a no-op on the folded input. func sanitizeOutput(text string) sanitizerResult { var r sanitizerResult - if det := detectLeadingPreamble(text); det != "" { + folded := recoverableFold(text) + // The two SCRIPT-based substantive classes (Latin insertion, broken word) additionally run with + // whitelisted CJK glosses masked out (D39.2 T2): a gloss's own pinyin/Han must not read as a leaked + // Latin phrase or a homoglyph token and drop the chunk. Masking is over the FOLDED text too. + glossMasked := maskGlossSpans(folded) + if det := detectLeadingPreamble(folded); det != "" { r.Preamble++ r.Detail = append(r.Detail, det) } - if det := detectTrailingNote(text); det != "" { + if det := detectTrailingNote(folded); det != "" { r.TrailingNote++ r.Detail = append(r.Detail, det) } - if n, det := detectMarkdownHeaders(text); n > 0 { + if n, det := detectMarkdownHeaders(folded); n > 0 { r.MarkdownHeader = n r.Detail = append(r.Detail, det...) } - if det := detectLatinInsertion(text); det != "" { + if det := detectLatinInsertion(glossMasked); det != "" { r.LatinInsert++ r.Detail = append(r.Detail, det) } - if n, det := detectBrokenWords(text); n > 0 { + if n, det := detectBrokenWords(glossMasked); n > 0 { r.BrokenWord = n r.Detail = append(r.Detail, det...) } - if n, det := detectCJKLeak(text); n > 0 { + if n, det := detectCJKLeak(folded); n > 0 { r.CJKLeak = n r.Detail = append(r.Detail, det...) } @@ -394,33 +420,196 @@ func isCJKLeakRune(r rune) bool { return false } +// --- CJK-gloss whitelist (D39.2 T2, owner decision 16.07: keep glosses) --------- + +// glossParenRE finds a candidate parenthesised CJK gloss: a Cyrillic/Latin HEAD letter, an optional +// short whitespace gap, then a single-level (NO nested paren, NO newline) «(...)» in ASCII OR fullwidth +// parens. The gap class covers a plain space, a NON-BREAKING space, and the CJK-native IDEOGRAPHIC space +// U+3000 (all bind a term to its gloss). U+3000 is load-bearing for detection↔strip CONSISTENCY (D39.4 +// review finding 1): the FOLD-FIRST detector sees U+3000 folded to a plain space and recognises the +// gloss, but stripCosmetic/exportNormalize detect glosses on the ORIGINAL bytes — so without U+3000 in +// the gap class a U+3000-bound gloss would read CLEAN in detection yet be DESTROYED by the strip. The +// head requirement is load-bearing — a paren block with no letter head (line start, after punctuation, a +// bare «(羅漢拳)») does NOT match, so its Han stays a leak. Group 1 is the whole paren block (the +// protected span); isValidGloss then validates its inner content. RE2 \p classes are Unicode. +var glossParenRE = regexp.MustCompile(`[\p{Cyrillic}\p{Latin}][ \x{00A0}\x{3000}]{0,2}(\([^()\n]*\)|([^()\n]*))`) + +// Gloss bounds (D39.2 T2 + D39.4 F2): a whitelisted gloss is a short TERM with at most a proportional +// pinyin romanisation, NOT a leaked clause smuggled in parens. maxGlossCJKRunes bounds the CJK term +// (羅漢拳, 五行八卦掌); the F2 bounds keep the non-CJK content pinyin-plausible so «拳 the iron fist that +// breaks the sky» is NOT whitelisted. [Tunable: flagged to the orchestrator; raise if a real term is +// dropped in practice.] +const ( + maxGlossCJKRunes = 6 // CJK chars in the term (九阳真经=4, 五行八卦掌=5) + maxGlossInnerRunes = 40 // whole inner-content backstop (a clause is long) + maxGlossLatinTokenRunes = 24 // one romanisation word ≈ 4×maxGlossCJKRunes: the JOINED pinyin of a ≤6-char term («jiǔyángzhēnjīng»=15, «wǔxíngbāguàzhǎng»=16) must fit, while a long English word («antidisestablishmentarianism»=28) exceeds it (D39.4 review finding 2) +) + +// isValidGloss reports whether a matched paren block «(inner)» is a whitelisted CJK TERM gloss: inner +// carries 1..maxGlossCJKRunes CJK letters (Han/kana — the original term) plus at most a PROPORTIONAL +// pinyin romanisation (Latin tokens ≤ cjk+1, each lowercase and ≤maxGlossLatinTokenRunes), and NOTHING +// outside the gloss charset (CJK letters + Latin/digits/combining tone marks + the separators space «·» +// «'» «-»), within maxGlossInnerRunes total. A parenthetical with Cyrillic prose, CJK PUNCTUATION +// («、」»), a nested fullwidth paren, a too-long CJK run (a clause, not a term), or a non-proportional / +// uppercase / over-long Latin run (a leaked English phrase, F2) fails — so any CJK inside stays a leak +// (precision over recall: a false whitelist would smuggle a real leak past the detector to export). +func isValidGloss(block string) bool { + rs := []rune(block) + if len(rs) < 3 { // «(x)» is the minimum + return false + } + inner := rs[1 : len(rs)-1] // inner content, without the opening/closing paren + if len(inner) > maxGlossInnerRunes { + return false // a term gloss is short; a long run is a clause (F2) + } + cjk, latinToks, tokLen := 0, 0, 0 + for _, r := range inner { + switch { + case unicode.Is(unicode.Han, r), unicode.Is(unicode.Hiragana, r), unicode.Is(unicode.Katakana, r): + cjk++ + if tokLen > 0 { // a CJK char ends the current Latin token + latinToks++ + tokLen = 0 + } + case unicode.Is(unicode.Latin, r): + if unicode.IsUpper(r) { + return false // pinyin romanisation is lowercase; an uppercase Latin run reads as prose (F2) + } + tokLen++ + if tokLen > maxGlossLatinTokenRunes { + return false // a token longer than any romanisation word → not pinyin (F2) + } + case unicode.IsDigit(r), unicode.Is(unicode.Mn, r): + // a pinyin tone digit / combining tone mark stays within the current Latin token + if tokLen > 0 { + tokLen++ + } + case r == ' ' || r == '·' || r == '\'' || r == '-': + if tokLen > 0 { // an intra-gloss separator ends the current Latin token + latinToks++ + tokLen = 0 + } + default: + return false + } + } + if tokLen > 0 { + latinToks++ + } + // The Latin tokens must be PROPORTIONAL to the term — pinyin transliterates the Han roughly + // one token per character (+1 slack) — so a 1-CJK head with a 7-word English clause is rejected (F2). + return cjk >= 1 && cjk <= maxGlossCJKRunes && latinToks <= cjk+1 +} + +// maskGlossSpans replaces every whitelisted gloss block with a single space, so the SUBSTANTIVE +// whole-text sanitizer classes that would otherwise mis-fire on a gloss's own bytes run gloss-blind: the +// Latin-insertion class must not read a space-separated PINYIN romanisation «(五行八卦掌 wu xing ba gua +// zhang)» as a leaked Latin phrase, and the broken-word class must not read a Han-glued pinyin token as a +// homoglyph — both would drop the whole chunk to a placeholder, defeating the "keep glosses" intent. The +// CJK-leak/strip/normalize paths use glossSegments directly (they must preserve the gloss bytes); these +// substantive classes only need the gloss GONE from their view, so a plain mask suffices. Pure. +func maskGlossSpans(text string) string { + spans := glossSpans(text) + if len(spans) == 0 { + return text + } + var b strings.Builder + b.Grow(len(text)) + pos := 0 + for _, sp := range spans { + b.WriteString(text[pos:sp[0]]) + b.WriteByte(' ') // a single space breaks token adjacency and drops the gloss's pinyin/Han from view + pos = sp[1] + } + b.WriteString(text[pos:]) + return b.String() +} + +// glossSpans returns the byte ranges of the whitelisted CJK glosses in text (the paren blocks, +// including their parens), left-to-right and non-overlapping (FindAll semantics). Detected on the +// ORIGINAL text so the span bytes can be preserved verbatim (a fullwidth char that is PART of a gloss +// is not folded — owner decision 16.07). Two adjacent glosses «(拳) и (腿)» yield two spans. +func glossSpans(text string) [][2]int { + m := glossParenRE.FindAllStringSubmatchIndex(text, -1) + if m == nil { + return nil + } + var spans [][2]int + for _, idx := range m { + s, e := idx[2], idx[3] // group 1 = the paren block + if s < 0 { + continue + } + if isValidGloss(text[s:e]) { + spans = append(spans, [2]int{s, e}) + } + } + return spans +} + +// textSegment is one slice of a chunk split by glossSegments: a normal segment (folded/stripped/ +// detected) or a gloss segment (preserved verbatim). +type textSegment struct { + text string + gloss bool +} + +// glossSegments splits text into alternating normal / gloss segments on the whitelisted gloss spans, so +// the three export-contract functions treat a gloss the SAME way: normal segments are folded/detected/ +// stripped, gloss segments are passed through untouched. The whole text is one normal segment when no +// gloss is present. +func glossSegments(text string) []textSegment { + spans := glossSpans(text) + if len(spans) == 0 { + return []textSegment{{text: text}} + } + var segs []textSegment + pos := 0 + for _, sp := range spans { + if sp[0] > pos { + segs = append(segs, textSegment{text: text[pos:sp[0]]}) + } + segs = append(segs, textSegment{text: text[sp[0]:sp[1]], gloss: true}) + pos = sp[1] + } + if pos < len(text) { + segs = append(segs, textSegment{text: text[pos:]}) + } + return segs +} + // detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail per -// distinct run. It runs over the RECOVERABLE-FOLD of the text (D39 слой 6): a fullwidth «123» or a -// «、» is a wrong-glyph rendering the export contract silently fixes, so it is NOT a leak — only a -// genuinely contentless Han/kana/CJK-bracket run is. A single run («却有») is enough to fire — the -// class is COSMETIC, so the whole chunk is not lost: stripCosmetic removes the run and the remainder -// exports (flagged for review). +// distinct run. It runs over the RECOVERABLE-FOLD of each NON-gloss segment (D39 слой 6): a fullwidth +// «123» or a «、» is a wrong-glyph rendering the export contract silently fixes, so it is NOT a leak — +// only a genuinely contentless Han/kana/CJK-bracket run is. A whitelisted CJK term gloss «(羅漢拳)» is +// skipped entirely (D39.2 T2 — owner keeps glosses). A single run («却有») is enough to fire — the class +// is COSMETIC, so the whole chunk is not lost: stripCosmetic removes the run and the remainder exports. func detectCJKLeak(text string) (int, []string) { var det []string seen := map[string]bool{} n := 0 - rs := []rune(recoverableFold(text)) - for i := 0; i < len(rs); { - if !isCJKLeakRune(rs[i]) { - i++ - continue + for _, seg := range glossSegments(text) { + if seg.gloss { + continue // a whitelisted gloss is legitimate original-term content, not a leak } - j := i - for j < len(rs) && isCJKLeakRune(rs[j]) { - j++ + rs := []rune(recoverableFold(seg.text)) + for i := 0; i < len(rs); { + if !isCJKLeakRune(rs[i]) { + i++ + continue + } + j := i + for j < len(rs) && isCJKLeakRune(rs[j]) { + j++ + } + n++ + run := string(rs[i:j]) + if !seen[run] { + seen[run] = true + det = append(det, "CJK-утечка в ru-выходе: "+preview(run)) + } + i = j } - n++ - run := string(rs[i:j]) - if !seen[run] { - seen[run] = true - det = append(det, "CJK-утечка в ru-выходе: "+preview(run)) - } - i = j } return n, det } @@ -466,14 +655,25 @@ var ideographicPunctFold = strings.NewReplacer("、", ",", "。", ".") // free, no re-bill). Deliberately conservative on whitespace: it does NOT collapse internal doubled // spaces (a clean chunk's spacing is left as the model wrote it). func exportNormalize(text string) string { - out := recoverableFold(text) - // NFC composes a canonically-decomposed base+mark (a «й» stored as и+U+0306, a «ё» as е+U+0308) - // BEFORE the combining-strip below, so the strip only removes the NON-composable stress accents - // (U+0301 on a vowel), never corrupting «й»/«ё» into «и»/«е». NFC is canonical-only, so it leaves - // «…» and other compatibility characters intact (unlike NFKC). - out = norm.NFC.String(out) - out = stripCombiningOnCyrillic(out) - return trailingHSpaceRE.ReplaceAllString(out, "") + var b strings.Builder + b.Grow(len(text)) + for _, seg := range glossSegments(text) { + if seg.gloss { + // A whitelisted CJK term gloss is preserved VERBATIM — including any fullwidth char that is + // part of the gloss, which is NOT folded (owner decision 16.07, D39.2 T2). + b.WriteString(seg.text) + continue + } + out := recoverableFold(seg.text) + // NFC composes a canonically-decomposed base+mark (a «й» stored as и+U+0306, a «ё» as е+U+0308) + // BEFORE the combining-strip below, so the strip only removes the NON-composable stress accents + // (U+0301 on a vowel), never corrupting «й»/«ё» into «и»/«е». NFC is canonical-only, so it leaves + // «…» and other compatibility characters intact (unlike NFKC). + out = norm.NFC.String(out) + out = stripCombiningOnCyrillic(out) + b.WriteString(out) + } + return trailingHSpaceRE.ReplaceAllString(b.String(), "") } // stripCombiningOnCyrillic drops a combining mark (unicode.Mn) that sits directly on a CYRILLIC base @@ -506,22 +706,32 @@ func stripCombiningOnCyrillic(s string) string { // non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it // sanitizer_stripped). It never crosses newlines when tidying, so paragraph structure is kept. func stripCosmetic(text string) string { - // 1) markdown header prefixes → keep the heading text, drop the «#### » artifact. + // 1) markdown header prefixes → keep the heading text, drop the «#### » artifact. Line-leading, so + // it never touches a mid-line gloss span computed below. out := leadingMarkdownStripRE.ReplaceAllString(text, "$1") - // 2) fold the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «123»→«123», U+3000→space, - // «、»→«,» «。»→«.») via the shared export-contract fold — no longer a hand-rolled NFKC - // (L5-stripcosmetic-duplicates-nfkc). After this only genuinely contentless CJK remains to drop. - out = recoverableFold(out) - // 3) drop the contentless CJK-leak runes (Han/kana/CJK brackets & symbols) the fold cannot - // recover, each leaving a SINGLE SPACE so adjacent Russian words do not merge («нашёл特产древний» - // → «нашёл древний», not «нашёлдревний»). + // 2+3) per segment: fold the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «123»→«123», + // U+3000→space, «、»→«,» «。»→«.») via the shared export-contract fold (L5-stripcosmetic-duplicates-nfkc) + // and drop the contentless CJK-leak runes the fold cannot recover, each leaving a SINGLE SPACE so + // adjacent Russian words do not merge («нашёл特产древний»→«нашёл древний»). A whitelisted CJK term + // gloss «(羅漢拳)» has its TERM BYTES preserved (D39.2 T2 — owner keeps glosses), so its parens no + // longer empty out and its term is not dropped. [The step-4 whitespace tidy below runs over the + // reassembled string, so it may normalise STRAY whitespace INSIDE a malformed gloss («(羅漢拳 )» → + // «(羅漢拳)»); this is benign and does NOT diverge from what `tmctl translate`/`tmctl export` ship, + // since translate's stripped-export path is exportNormalize(stripCosmetic(x)) and Export reads that + // same derived checkpoint — both go through THIS function, so the shipped bytes agree.] var b strings.Builder b.Grow(len(out)) - for _, r := range out { - if isCJKLeakRune(r) { - b.WriteByte(' ') - } else { - b.WriteRune(r) + for _, seg := range glossSegments(out) { + if seg.gloss { + b.WriteString(seg.text) + continue + } + for _, r := range recoverableFold(seg.text) { + if isCJKLeakRune(r) { + b.WriteByte(' ') + } else { + b.WriteRune(r) + } } } out = b.String() diff --git a/backend/internal/pipeline/sanitizer_test.go b/backend/internal/pipeline/sanitizer_test.go index aeda4ad..3b2033f 100644 --- a/backend/internal/pipeline/sanitizer_test.go +++ b/backend/internal/pipeline/sanitizer_test.go @@ -291,8 +291,9 @@ func TestStripCosmetic(t *testing.T) { {"Индекс ABC найден.", "Индекс ABC найден."}, // ideographic comma/period fold to plain — the clause boundary is kept. {"Он вошёл в дом、и дверь закрылась。", "Он вошёл в дом,и дверь закрылась."}, - // a parenthetical Han gloss is stripped and the emptied «()» removed (adversarial review). - {"Кулак Ло Хань (羅漢拳) был силён.", "Кулак Ло Хань был силён."}, + // a whitelisted CJK term gloss is now PRESERVED (D39.2 T2 — owner keeps glosses); a co-occurring + // BARE Han run (no head, not in parens) is still stripped, and the gloss parens do not empty out. + {"Кулак Ло Хань (羅漢拳) ударил 特产 врага.", "Кулак Ло Хань (羅漢拳) ударил врага."}, // a legit ratio «2 : 1» is NOT reformatted (colon dropped from the tidy set). {"### Итог\n\nСчёт был 2 : 1 却有 в пользу гостей.", "Итог\n\nСчёт был 2 : 1 в пользу гостей."}, // U+3000 ideographic space normalised to a plain space (the indent artifact). @@ -348,6 +349,133 @@ func TestExportNormalize(t *testing.T) { } } +// TestSanitizerCJKGlossWhitelist pins the CJK-gloss whitelist (D39.2 T2, owner decision 16.07: KEEP +// glosses): a balanced CJK run inside parens directly after a Cyrillic/Latin head word is NOT a leak — +// detectCJKLeak does not flag it and stripCosmetic/exportNormalize preserve its bytes; a bare Han run +// (no head, line start, after punctuation, CJK punctuation inside, nested paren) is STILL a leak. +func TestSanitizerCJKGlossWhitelist(t *testing.T) { + // Whitelisted glosses: NOT flagged, PRESERVED verbatim by both strip and normalize. + preserved := []string{ + "Кулак Ло Хань (羅漢拳) был силён.", // the owner's canonical case (ASCII parens) + "Он изучил дар (資質) древнего клана.", // gloss with no space before paren-adjacent head + "Приём Ло Хань(羅漢拳) сработал.", // head directly adjacent to «(» (no space) + "Два приёма: Кулак (拳) и Нога (腿) в бою.", // two glosses in a row — both preserved + "Стиль Тайцзи (太極拳 tàijíquán) он знал.", // Han + pinyin with tone marks and a space + "Техника Zhang (張三豐) древних мастеров.", // Latin head word, Han gloss + "Название клана(羅漢)на воротах.", // FULLWIDTH parens — preserved verbatim, not folded + "Кулак (羅漢拳) был силён.", // NBSP between head and paren (finding B) — term kept + "Приём (五行八卦掌 wu xing ba gua zhang) стар.", // 5 space-separated PINYIN syllables (finding A) + "Он изучил приём (九阳真经 jiǔyángzhēnjīng) там.", // JOINED pinyin of a 4-char term (D39.4 finding 2) + } + for _, s := range preserved { + if got := sanitizeOutput(s); got.CJKLeak != 0 { + t.Errorf("gloss FALSE-flagged as leak in %q: %v", preview(s), got.Detail) + } + if got := stripCosmetic(s); got != s { + t.Errorf("stripCosmetic altered a preserved gloss:\n in %q\n got %q", s, got) + } + if got := exportNormalize(s); got != s { + t.Errorf("exportNormalize altered a preserved gloss:\n in %q\n got %q", s, got) + } + } + // FINDING A: a space-separated pinyin gloss must NOT trip the Latin-insertion class and DROP the whole + // chunk (detectLatinInsertion runs gloss-masked). The chunk must be fully clean (nothing fired). + if got := sanitizeOutput("Приём (五行八卦掌 wu xing ba gua zhang) стар."); got.total() != 0 { + t.Errorf("pinyin gloss must not flag the chunk (finding A), got %+v", got) + } + // FINDING B: a DOUBLE space between head and paren — the gloss TERM is kept and the chunk not flagged + // (the tidy normalises the double space to one, so it is not byte-verbatim, hence a separate check). + if got := sanitizeOutput("Кулак (羅漢拳) был силён."); got.CJKLeak != 0 { + t.Errorf("double-space gloss FALSE-flagged (finding B): %+v", got) + } + if got := stripCosmetic("Кулак (羅漢拳) был силён."); !strings.Contains(got, "羅漢拳") { + t.Errorf("double-space gloss term dropped (finding B): %q", got) + } + // D39.4 finding 1: a U+3000 (ideographic-space) gap binds the term to the gloss — the term is kept + // and the chunk not flagged (the U+3000 in the head normalises to a plain space, so not byte-verbatim). + if got := sanitizeOutput("Кулак (羅漢拳) был силён."); got.CJKLeak != 0 { + t.Errorf("U+3000-bound gloss FALSE-flagged (finding 1): %+v", got) + } + if got := stripCosmetic("Кулак (羅漢拳) был силён."); !strings.Contains(got, "羅漢拳") { + t.Errorf("U+3000-bound gloss term dropped (finding 1): %q", got) + } + // NOT a whitelisted gloss → the CJK stays a leak (bare run / no letter head / CJK punctuation / + // nesting / a CLAUSE too long to be a term — finding D). + leak := []string{ + "Судзуки нашёл 特产 древнего клана.", // bare Han, no parens at all + "(羅漢拳) — так назвали приём.", // paren gloss at LINE START, no letter head → leak + "Он сказал: (太極) и ушёл.", // after «:» + space, no letter head → leak + "Приём 5 (羅漢拳) в списке.", // digit head, not a letter → not a gloss + "Клан (羅、漢) распался.", // CJK PUNCTUATION «、» inside the parens → not a term gloss + "Он ушёл (却有很多事情要做), но вернулся.", // 8-CJK CLAUSE, not a term gloss (finding D) → leak + "Техника (九字真言破邪法) была забыта тогда.", // 7 CJK > maxGlossCJKRunes → leak (finding D bound) + "Кулак (拳 the iron fist that breaks the sky) взлетел.", // F2: English clause after a token Han → leak + "Приём (拳 antidisestablishmentarianism) стар.", // F2: single over-long non-pinyin Latin word → leak + "Стиль (拳 iron fist of sky) забыт.", // F2: non-proportional Latin (3 toks vs 1 CJK) → leak + } + for _, s := range leak { + if got := sanitizeOutput(s); got.CJKLeak == 0 { + t.Errorf("expected a CJK leak (not a valid gloss) in %q: %+v", preview(s), got) + } + } + // The emptied-bracket regression must NOT reappear: a preserved gloss keeps its parens and term, so + // stripCosmetic never yields «()» — and a real bare leak next to a gloss strips cleanly. + if got := stripCosmetic("Кулак (羅漢拳) и 特产 рядом."); got != "Кулак (羅漢拳) и рядом." { + t.Errorf("mixed gloss+leak strip = %q, want the gloss kept and the bare leak dropped", got) + } + // D39.4 finding 1: a U+3000-BOUND gloss must survive the strip of a CO-OCCURRING bare leak — the + // fold-first detector and the original-bytes strip must agree the U+3000 gloss is protected. + if got := stripCosmetic("Кулак (羅漢拳) и 特产 рядом."); !strings.Contains(got, "羅漢拳") { + t.Errorf("finding 1: U+3000-bound gloss destroyed alongside a bare leak: %q", got) + } +} + +// TestSanitizerFoldFirstBypass pins F1 (D39.4): a FULLWIDTH variation of a signature must be detected, +// not slip past the ASCII regexes while exportNormalize manufactures the ASCII defect in the export. +// All six classes detect over the export-normalised (folded) text. +func TestSanitizerFoldFirstBypass(t *testing.T) { + fire := map[string]string{ + "fullwidth-colon-preamble": "Вот отредактированный перевод:\n\nСудзуки шёл.", // U+FF1A colon + "fullwidth-markdown": "### Глава 5\n\nСудзуки открыл дверь.", // U+FF03 hash ×3 + "fullwidth-trailing-note": "Судзуки ушёл.\n\nПримечание:имя оставлено как есть.", // U+FF1A trailing + "fullwidth-latin-phrase": "Он прочитал: the quick brown fox today.", // fullwidth latin + } + for name, s := range fire { + if got := sanitizeOutput(s); got.total() == 0 { + t.Errorf("F1 %s: a fullwidth signature slipped past the detector (bypass): %q → %+v", name, preview(s), got) + } + } +} + +// TestSanitizerExportNormalizeInvariant pins the F1 gate/export-contract invariant: for any text the +// sanitizer calls CLEAN, its export-normalised form is ALSO clean — the detector and exportNormalize +// agree, so no fullwidth defect is manufactured in the export of a "clean" chunk. Runs over the whole +// legitimate-prose corpus the class tests use. +func TestSanitizerExportNormalizeInvariant(t *testing.T) { + corpus := []string{ + "Судзуки шёл по коридорам Академии магии, вспоминая вчерашнюю долгую лекцию.", + "— Сегодня я открою дверь книгохранилища, — прошептал он, сжимая ключ.", + "Цена была 123 золотых, и он расплатился не торгуясь.", + " Судзуки медленно шёл по длинному коридору к башне.", + "Цена была 123 золотых.", + "Он вошёл в дом、и дверь закрылась.", + "Он источа́ло тепло за́мка.", + "Он ждал… но никто не пришёл. № 5, ½ часа.", + "Кулак Ло Хань (羅漢拳) был силён.", + "Стиль Тайцзи (太極拳 tàijíquán) он знал.", + "Он купил iPhone, iPad, MacBook, Apple Watch и Google Pixel вчера.", + "На гербе была выбита надпись: sic transit gloria mundi.", + } + for _, s := range corpus { + if sanitizeOutput(s).total() != 0 { + continue // not a clean input — the invariant is stated for clean verdicts + } + if got := sanitizeOutput(exportNormalize(s)); got.total() != 0 { + t.Errorf("F1 invariant: clean %q became a defect after exportNormalize: %+v", preview(s), got) + } + } +} + // TestSanitizerDeterministic asserts the verdict is a pure function of the text (resume-safe). func TestSanitizerDeterministic(t *testing.T) { s := "### Заголовок\nВот отредактированный перевод: текст. Стольь странно.\nПримечание: правка." diff --git a/backend/internal/pipeline/testdata/golden/capture.golden b/backend/internal/pipeline/testdata/golden/capture.golden index 874f350..fd1919c 100644 --- a/backend/internal/pipeline/testdata/golden/capture.golden +++ b/backend/internal/pipeline/testdata/golden/capture.golden @@ -1,6 +1,6 @@ ==== run 1 (fresh) ==== -snapshot_id: 79309d253aeaa641272a3fb53eca471567cecf1d2e8339a7fba2e2f1d4c37708 -snapshot_payload: {"brief_hash":"c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df","chunker_version":"chunker-v4-utf8-quotedepth","estimator_version":"estimator-v0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v1-refusal+echo015+loop","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"stm_depth":0,"overlap_tokens":0,"cache_ttl":""},"memory_version":"636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v2","sanitizer":{"enabled":true,"version":"sanitizer-v5"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"26d0245a56d180eafd200ebe5311aab2488142c4a98dcddee21ea457814aa21a","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}},{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]} +snapshot_id: 325d971e2b6ec79a129891f969d16a3ddea1410f345f52d9d47e3f50c70726b2 +snapshot_payload: {"brief_hash":"c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df","chunker_version":"chunker-v4-utf8-quotedepth","estimator_version":"estimator-v0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v1-refusal+echo015+loop","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"stm_depth":0,"overlap_tokens":0,"cache_ttl":""},"memory_version":"636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v2","sanitizer":{"enabled":true,"version":"sanitizer-v6"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"26d0245a56d180eafd200ebe5311aab2488142c4a98dcddee21ea457814aa21a","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}},{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]} brief_hash: c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df memory_version: 636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53 -- book result -- @@ -51,44 +51,44 @@ chunk ch8/0 disposition=flagged flag="sanitizer_stripped" final_text="Судзу stage=edit role=editor model=fake-model resume=false disp=flagged flag="sanitizer_stripped" attempts=1 escalated=false esc_model="" finish="stop" cum_usd=0.00182 detail="CJK-утечка в ru-выходе: 特产" stage_text="" recovered="Судзуки нашёл древний камень на каменном алтаре." -- chunk_status -- -ch1/0 draft snap_match=true content_hash=331c6f32c7f3ed15447e2639a2621d24a180e561a2aee92115324f492db5de3e disp=ok flag="" attempts=1 final_hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 cost=0.00182 escalated=false esc_model="" detail="" -ch1/0 edit snap_match=true content_hash=a559165b4d1fc3f9e75472ca654f4d039ae1e0cad1541c043e7aa68fbea5415a disp=ok flag="" attempts=1 final_hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f cost=0.00182 escalated=false esc_model="" detail="" -ch1/1 draft snap_match=true content_hash=12fe3ccdf062f43e011363a33ff85c9b6842a8f23d790991fe9afab51a8becc8 disp=ok flag="" attempts=1 final_hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 cost=0.00182 escalated=false esc_model="" detail="" -ch1/1 edit snap_match=true content_hash=b11a1740e5af98af20e207672555da8d411311e6c1e26595b456b656e8de59ef disp=ok flag="" attempts=1 final_hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 cost=0.00182 escalated=false esc_model="" detail="" -ch2/0 draft snap_match=true content_hash=2693325bc747687acf090bc263fe8c17783726b949c6d8a2d072da3401be6827 disp=ok flag="" attempts=1 final_hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a cost=0.00728 escalated=true esc_model="fake-fallback" detail="" -ch2/0 edit snap_match=true content_hash=b467337f05775296985a6d79454b518302812edb57d398001b8852d0663c5bbb disp=ok flag="" attempts=1 final_hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 cost=0.00182 escalated=false esc_model="" detail="" +ch1/0 draft snap_match=true content_hash=331c6f32c7f3ed15447e2639a2621d24a180e561a2aee92115324f492db5de3e disp=ok flag="" attempts=1 final_hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 cost=0.00182 escalated=false esc_model="" detail="" +ch1/0 edit snap_match=true content_hash=a559165b4d1fc3f9e75472ca654f4d039ae1e0cad1541c043e7aa68fbea5415a disp=ok flag="" attempts=1 final_hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 cost=0.00182 escalated=false esc_model="" detail="" +ch1/1 draft snap_match=true content_hash=12fe3ccdf062f43e011363a33ff85c9b6842a8f23d790991fe9afab51a8becc8 disp=ok flag="" attempts=1 final_hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 cost=0.00182 escalated=false esc_model="" detail="" +ch1/1 edit snap_match=true content_hash=b11a1740e5af98af20e207672555da8d411311e6c1e26595b456b656e8de59ef disp=ok flag="" attempts=1 final_hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 cost=0.00182 escalated=false esc_model="" detail="" +ch2/0 draft snap_match=true content_hash=2693325bc747687acf090bc263fe8c17783726b949c6d8a2d072da3401be6827 disp=ok flag="" attempts=1 final_hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 cost=0.00728 escalated=true esc_model="fake-fallback" detail="" +ch2/0 edit snap_match=true content_hash=b467337f05775296985a6d79454b518302812edb57d398001b8852d0663c5bbb disp=ok flag="" attempts=1 final_hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 cost=0.00182 escalated=false esc_model="" detail="" ch3/0 draft snap_match=true content_hash=733fc6a89215efe112cfadd1d1fb361bb8a7ab1013994248a49da0dbd137548e disp=flagged flag="hard_refusal" attempts=1 final_hash= cost=0.00728 escalated=true esc_model="fake-fallback" detail="provider finish_reason=refusal" ch3/0 edit snap_match=true content_hash= disp=skipped flag="hard_refusal" attempts=0 final_hash= cost=0 escalated=false esc_model="" detail="skipped: an upstream stage was flagged (hard_refusal)" -ch4/0 draft snap_match=true content_hash=7aabdcb89dea365c95363b2662144ca1a10b307d377f47a8e7cc19b5b872d287 disp=ok flag="" attempts=1 final_hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece cost=0.00182 escalated=false esc_model="" detail="" -ch4/0 edit snap_match=true content_hash=2854dfe1648b0cf0cbb683cb7ce94c5eedc9223b2ce20c99dc008c7add6317ef disp=ok flag="" attempts=1 final_hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 cost=0.00182 escalated=false esc_model="" detail="" -ch5/0 draft snap_match=true content_hash=d8aba42cabb41853a81875ebc725bb516dc2412e29de3a03bc1a68e90f8cf157 disp=ok flag="" attempts=1 final_hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 cost=0.00182 escalated=false esc_model="" detail="" -ch5/0 edit snap_match=true content_hash=99ebb10d754a2f3ae3a00d62734ad1c39160f083350217eda5bda6e91373f4de disp=ok flag="" attempts=1 final_hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb cost=0.00182 escalated=false esc_model="" detail="" -ch6/0 draft snap_match=true content_hash=9375b05780724cc354eba15a595da9a5f964219c0887042484256eed11fb566f disp=ok flag="" attempts=1 final_hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f cost=0.00182 escalated=false esc_model="" detail="" +ch4/0 draft snap_match=true content_hash=7aabdcb89dea365c95363b2662144ca1a10b307d377f47a8e7cc19b5b872d287 disp=ok flag="" attempts=1 final_hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c cost=0.00182 escalated=false esc_model="" detail="" +ch4/0 edit snap_match=true content_hash=2854dfe1648b0cf0cbb683cb7ce94c5eedc9223b2ce20c99dc008c7add6317ef disp=ok flag="" attempts=1 final_hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 cost=0.00182 escalated=false esc_model="" detail="" +ch5/0 draft snap_match=true content_hash=d8aba42cabb41853a81875ebc725bb516dc2412e29de3a03bc1a68e90f8cf157 disp=ok flag="" attempts=1 final_hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 cost=0.00182 escalated=false esc_model="" detail="" +ch5/0 edit snap_match=true content_hash=99ebb10d754a2f3ae3a00d62734ad1c39160f083350217eda5bda6e91373f4de disp=ok flag="" attempts=1 final_hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 cost=0.00182 escalated=false esc_model="" detail="" +ch6/0 draft snap_match=true content_hash=9375b05780724cc354eba15a595da9a5f964219c0887042484256eed11fb566f disp=ok flag="" attempts=1 final_hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 cost=0.00182 escalated=false esc_model="" detail="" ch6/0 edit snap_match=true content_hash=f9a70ce8a43db4a89eda5e53a6f29ba5f0c2a35559a9d35b73f771220bb262ee disp=flagged flag="sanitizer_defect" attempts=1 final_hash= cost=0.00182 escalated=false esc_model="" detail="ведущая служебная преамбула: Вот перевод фрагмента:" -ch7/0 draft snap_match=true content_hash=d90c335cba3d1547a22bb3f656443cb432f84c12a577d43509cfe472cf86a2ac disp=ok flag="" attempts=1 final_hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 cost=0.00182 escalated=false esc_model="" detail="" -ch7/0 edit snap_match=true content_hash=f003d8ad25546238e63b98d28a08f942c8e3cf57251df694de81bea293a6dd26 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:3f4f20574caef4ee6153e60f6acc5f41e9ed5afe63b2cafd69396bafb18f9051 cost=0.00182 escalated=false esc_model="" detail="markdown-заголовок в выходе: ### Глава 7" -ch8/0 draft snap_match=true content_hash=fbd27af11da2275cd1d47a0224970a5f8d073c6164f614efb6522ea96fa1e021 disp=ok flag="" attempts=1 final_hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb cost=0.00182 escalated=false esc_model="" detail="" -ch8/0 edit snap_match=true content_hash=40c2dd5b7b713800e8453930779bf2c88e2a296056f594bc1a4f5c495956915a disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:31dcd0e8a631cbce9ef7873f139bc415902858a708ea16602c8fc38f77cf3b66 cost=0.00182 escalated=false esc_model="" detail="CJK-утечка в ru-выходе: 特产" +ch7/0 draft snap_match=true content_hash=d90c335cba3d1547a22bb3f656443cb432f84c12a577d43509cfe472cf86a2ac disp=ok flag="" attempts=1 final_hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a cost=0.00182 escalated=false esc_model="" detail="" +ch7/0 edit snap_match=true content_hash=f003d8ad25546238e63b98d28a08f942c8e3cf57251df694de81bea293a6dd26 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:e1f2765b9f6d3cc4c461471cd7c143476614c602b6b08843bd239c77beb3b102 cost=0.00182 escalated=false esc_model="" detail="markdown-заголовок в выходе: ### Глава 7" +ch8/0 draft snap_match=true content_hash=fbd27af11da2275cd1d47a0224970a5f8d073c6164f614efb6522ea96fa1e021 disp=ok flag="" attempts=1 final_hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 cost=0.00182 escalated=false esc_model="" detail="" +ch8/0 edit snap_match=true content_hash=40c2dd5b7b713800e8453930779bf2c88e2a296056f594bc1a4f5c495956915a disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:067cfd0a649bfcd3db7fd04e8beff2a610dba06e292664e0160f06d16eef5209 cost=0.00182 escalated=false esc_model="" detail="CJK-утечка в ru-выходе: 特产" -- request_log (insertion order) -- -ch1/0 draft role=translator req=fake-model actual=fake-model hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/0 edit role=editor req=fake-model actual=fake-model hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/1 draft role=translator req=fake-model actual=fake-model hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/1 edit role=editor req=fake-model actual=fake-model hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 draft role=translator req=fake-model actual=fake-model hash=afeec2c4175c036a72332a9d6e27ab48a5584bb3819ca5a3c5b95ad01160ed74 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" -ch2/0 edit role=editor req=fake-model actual=fake-model hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch3/0 draft role=translator req=fake-model actual=fake-model hash=cb19e60076fb427f7294b8a0b80e2d810af3630d16353892f83a49b947affdd0 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=bc3af2566b80f34217bcbf5898895ab00887355f3f1e8a427d5d504683d41902 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" -ch4/0 draft role=translator req=fake-model actual=fake-model hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch4/0 edit role=editor req=fake-model actual=fake-model hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch5/0 draft role=translator req=fake-model actual=fake-model hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch5/0 edit role=editor req=fake-model actual=fake-model hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch6/0 draft role=translator req=fake-model actual=fake-model hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch6/0 edit role=editor req=fake-model actual=fake-model hash=b76db9cd65fd0cde6bcc4d2d8cc77ae9628e23ebeff2653cb59740b458f1960c tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch7/0 draft role=translator req=fake-model actual=fake-model hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch7/0 edit role=editor req=fake-model actual=fake-model hash=327f47fe5e882624cb3c01c1e0dcfde566e8b695cbe482de8e4916eb90d6f45b tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch8/0 draft role=translator req=fake-model actual=fake-model hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch8/0 edit role=editor req=fake-model actual=fake-model hash=5b836ac514f218026f8f61f4d8ef6e1a68b7625ec2dde3cb95e196b6ac55864d tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/0 draft role=translator req=fake-model actual=fake-model hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/0 edit role=editor req=fake-model actual=fake-model hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/1 draft role=translator req=fake-model actual=fake-model hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/1 edit role=editor req=fake-model actual=fake-model hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 draft role=translator req=fake-model actual=fake-model hash=5f4763e07c67f541958817ce6e7a65c8277e1d806558d3e218e92362ba858866 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" +ch2/0 edit role=editor req=fake-model actual=fake-model hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch3/0 draft role=translator req=fake-model actual=fake-model hash=be200a513ccbfbb410ffc7788967610e2323aeda610c4d25296c8f06ddf3aead tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=4e5ad1c52475c43b5f98f0a369187a11367e4c739b67d5d9269b9949b718de33 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" +ch4/0 draft role=translator req=fake-model actual=fake-model hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch4/0 edit role=editor req=fake-model actual=fake-model hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch5/0 draft role=translator req=fake-model actual=fake-model hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch5/0 edit role=editor req=fake-model actual=fake-model hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch6/0 draft role=translator req=fake-model actual=fake-model hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch6/0 edit role=editor req=fake-model actual=fake-model hash=ae7cb45838ddec818f5e6c8f71410db7f00365dc8d1a41ec1a94af1d490e2178 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch7/0 draft role=translator req=fake-model actual=fake-model hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch7/0 edit role=editor req=fake-model actual=fake-model hash=9394388fc09349b7e4014e12ae991f041c5912307964d83ae165a8c99a3aac2e tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch8/0 draft role=translator req=fake-model actual=fake-model hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch8/0 edit role=editor req=fake-model actual=fake-model hash=fd0a61bd15ec394e35f291a3f114469a0fca591b3bc529ef8bd79686fe3bef93 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" -- retrieval_state -- ch1/0 snap_match=true exact=3 sticky=0 ambiguous=0 spoiler=1 evicted=0 postcheck_miss=1 style_flags=0 trust_gated=0 injected_ids=["紋章\u001f\u001f0\u001f0","鈴木\u001f\u001f0\u001f0","魔法学院\u001f\u001f0\u001f0"] postcheck_detail=[{"src":"紋章","dst":"герб","disp":"confirmed"}] style_detail= trust_gate_detail= @@ -129,8 +129,8 @@ ch8/0 snap_match=true exact=1 sticky=0 ambiguous=0 spoiler=0 evicted=0 postcheck [17] {"max_tokens":4000,"messages":[{"role":"system","content":"Переводи художественный текст с ja на ru. Жанр: ранобэ. Аудитория: взрослые. Баланс Венути: 0.50. Хонорифики: keep. Транскрипция: polivanov."},{"role":"system","content":"ГЛОССАРИЙ (используй эти утверждённые переводы имён и терминов последовательно; строки с пометкой ⟨проверить⟩ — неподтверждённые кандидаты):\n鈴木 → Судзуки"},{"role":"user","content":"第八章 漢字漏洩\n\n鈴木は石板の文字をなぞり、古い漢字の意味を思い出そうとした。だが記憶は霧のように薄れ、指先だけが淡く光っていた。"}],"model":"fake-model","stream":false,"temperature":0.3} [18] {"max_tokens":4000,"messages":[{"role":"system","content":"Ты — монолингвальный редактор русского текста. Правь стиль, не меняя смысла. Книга: Золотая книга."},{"role":"system","content":"КАНОНИЧЕСКИЕ ПЕРЕВОДЫ имён и терминов (в черновике эти сущности должны быть переданы ИМЕННО этими формами — приводи к ним любые расхождения, склоняя по контексту; не вводи иных вариантов и не меняй ничего другого):\n- «Судзуки»"},{"role":"user","content":"Черновик перевода для редактуры: ЧЕРНОВИК ПЕРЕВОДА 0bc6a4817acb. ИЕРОГЛИФ-СЦЕНА Судзуки нашёл камень."}],"model":"fake-model","stream":false,"temperature":0.4} ==== run 2 (resume) ==== -snapshot_id: 79309d253aeaa641272a3fb53eca471567cecf1d2e8339a7fba2e2f1d4c37708 -snapshot_payload: {"brief_hash":"c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df","chunker_version":"chunker-v4-utf8-quotedepth","estimator_version":"estimator-v0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v1-refusal+echo015+loop","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"stm_depth":0,"overlap_tokens":0,"cache_ttl":""},"memory_version":"636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v2","sanitizer":{"enabled":true,"version":"sanitizer-v5"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"26d0245a56d180eafd200ebe5311aab2488142c4a98dcddee21ea457814aa21a","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}},{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]} +snapshot_id: 325d971e2b6ec79a129891f969d16a3ddea1410f345f52d9d47e3f50c70726b2 +snapshot_payload: {"brief_hash":"c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df","chunker_version":"chunker-v4-utf8-quotedepth","estimator_version":"estimator-v0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v1-refusal+echo015+loop","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"stm_depth":0,"overlap_tokens":0,"cache_ttl":""},"memory_version":"636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v2","sanitizer":{"enabled":true,"version":"sanitizer-v6"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"26d0245a56d180eafd200ebe5311aab2488142c4a98dcddee21ea457814aa21a","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}},{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]} brief_hash: c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df memory_version: 636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53 -- book result -- @@ -181,61 +181,61 @@ chunk ch8/0 disposition=flagged flag="sanitizer_stripped" final_text="Судзу stage=edit role=editor model=fake-model resume=true disp=flagged flag="sanitizer_stripped" attempts=1 escalated=false esc_model="" finish="sanitized_export" cum_usd=0.00182 detail="CJK-утечка в ru-выходе: 特产" stage_text="" recovered="Судзуки нашёл древний камень на каменном алтаре." -- chunk_status -- -ch1/0 draft snap_match=true content_hash=331c6f32c7f3ed15447e2639a2621d24a180e561a2aee92115324f492db5de3e disp=ok flag="" attempts=1 final_hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 cost=0.00182 escalated=false esc_model="" detail="" -ch1/0 edit snap_match=true content_hash=a559165b4d1fc3f9e75472ca654f4d039ae1e0cad1541c043e7aa68fbea5415a disp=ok flag="" attempts=1 final_hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f cost=0.00182 escalated=false esc_model="" detail="" -ch1/1 draft snap_match=true content_hash=12fe3ccdf062f43e011363a33ff85c9b6842a8f23d790991fe9afab51a8becc8 disp=ok flag="" attempts=1 final_hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 cost=0.00182 escalated=false esc_model="" detail="" -ch1/1 edit snap_match=true content_hash=b11a1740e5af98af20e207672555da8d411311e6c1e26595b456b656e8de59ef disp=ok flag="" attempts=1 final_hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 cost=0.00182 escalated=false esc_model="" detail="" -ch2/0 draft snap_match=true content_hash=2693325bc747687acf090bc263fe8c17783726b949c6d8a2d072da3401be6827 disp=ok flag="" attempts=1 final_hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a cost=0.00728 escalated=true esc_model="fake-fallback" detail="" -ch2/0 edit snap_match=true content_hash=b467337f05775296985a6d79454b518302812edb57d398001b8852d0663c5bbb disp=ok flag="" attempts=1 final_hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 cost=0.00182 escalated=false esc_model="" detail="" +ch1/0 draft snap_match=true content_hash=331c6f32c7f3ed15447e2639a2621d24a180e561a2aee92115324f492db5de3e disp=ok flag="" attempts=1 final_hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 cost=0.00182 escalated=false esc_model="" detail="" +ch1/0 edit snap_match=true content_hash=a559165b4d1fc3f9e75472ca654f4d039ae1e0cad1541c043e7aa68fbea5415a disp=ok flag="" attempts=1 final_hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 cost=0.00182 escalated=false esc_model="" detail="" +ch1/1 draft snap_match=true content_hash=12fe3ccdf062f43e011363a33ff85c9b6842a8f23d790991fe9afab51a8becc8 disp=ok flag="" attempts=1 final_hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 cost=0.00182 escalated=false esc_model="" detail="" +ch1/1 edit snap_match=true content_hash=b11a1740e5af98af20e207672555da8d411311e6c1e26595b456b656e8de59ef disp=ok flag="" attempts=1 final_hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 cost=0.00182 escalated=false esc_model="" detail="" +ch2/0 draft snap_match=true content_hash=2693325bc747687acf090bc263fe8c17783726b949c6d8a2d072da3401be6827 disp=ok flag="" attempts=1 final_hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 cost=0.00728 escalated=true esc_model="fake-fallback" detail="" +ch2/0 edit snap_match=true content_hash=b467337f05775296985a6d79454b518302812edb57d398001b8852d0663c5bbb disp=ok flag="" attempts=1 final_hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 cost=0.00182 escalated=false esc_model="" detail="" ch3/0 draft snap_match=true content_hash=733fc6a89215efe112cfadd1d1fb361bb8a7ab1013994248a49da0dbd137548e disp=flagged flag="hard_refusal" attempts=1 final_hash= cost=0.00728 escalated=true esc_model="fake-fallback" detail="provider finish_reason=refusal" ch3/0 edit snap_match=true content_hash= disp=skipped flag="hard_refusal" attempts=0 final_hash= cost=0 escalated=false esc_model="" detail="skipped: an upstream stage was flagged (hard_refusal)" -ch4/0 draft snap_match=true content_hash=7aabdcb89dea365c95363b2662144ca1a10b307d377f47a8e7cc19b5b872d287 disp=ok flag="" attempts=1 final_hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece cost=0.00182 escalated=false esc_model="" detail="" -ch4/0 edit snap_match=true content_hash=2854dfe1648b0cf0cbb683cb7ce94c5eedc9223b2ce20c99dc008c7add6317ef disp=ok flag="" attempts=1 final_hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 cost=0.00182 escalated=false esc_model="" detail="" -ch5/0 draft snap_match=true content_hash=d8aba42cabb41853a81875ebc725bb516dc2412e29de3a03bc1a68e90f8cf157 disp=ok flag="" attempts=1 final_hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 cost=0.00182 escalated=false esc_model="" detail="" -ch5/0 edit snap_match=true content_hash=99ebb10d754a2f3ae3a00d62734ad1c39160f083350217eda5bda6e91373f4de disp=ok flag="" attempts=1 final_hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb cost=0.00182 escalated=false esc_model="" detail="" -ch6/0 draft snap_match=true content_hash=9375b05780724cc354eba15a595da9a5f964219c0887042484256eed11fb566f disp=ok flag="" attempts=1 final_hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f cost=0.00182 escalated=false esc_model="" detail="" +ch4/0 draft snap_match=true content_hash=7aabdcb89dea365c95363b2662144ca1a10b307d377f47a8e7cc19b5b872d287 disp=ok flag="" attempts=1 final_hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c cost=0.00182 escalated=false esc_model="" detail="" +ch4/0 edit snap_match=true content_hash=2854dfe1648b0cf0cbb683cb7ce94c5eedc9223b2ce20c99dc008c7add6317ef disp=ok flag="" attempts=1 final_hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 cost=0.00182 escalated=false esc_model="" detail="" +ch5/0 draft snap_match=true content_hash=d8aba42cabb41853a81875ebc725bb516dc2412e29de3a03bc1a68e90f8cf157 disp=ok flag="" attempts=1 final_hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 cost=0.00182 escalated=false esc_model="" detail="" +ch5/0 edit snap_match=true content_hash=99ebb10d754a2f3ae3a00d62734ad1c39160f083350217eda5bda6e91373f4de disp=ok flag="" attempts=1 final_hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 cost=0.00182 escalated=false esc_model="" detail="" +ch6/0 draft snap_match=true content_hash=9375b05780724cc354eba15a595da9a5f964219c0887042484256eed11fb566f disp=ok flag="" attempts=1 final_hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 cost=0.00182 escalated=false esc_model="" detail="" ch6/0 edit snap_match=true content_hash=f9a70ce8a43db4a89eda5e53a6f29ba5f0c2a35559a9d35b73f771220bb262ee disp=flagged flag="sanitizer_defect" attempts=1 final_hash= cost=0.00182 escalated=false esc_model="" detail="ведущая служебная преамбула: Вот перевод фрагмента:" -ch7/0 draft snap_match=true content_hash=d90c335cba3d1547a22bb3f656443cb432f84c12a577d43509cfe472cf86a2ac disp=ok flag="" attempts=1 final_hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 cost=0.00182 escalated=false esc_model="" detail="" -ch7/0 edit snap_match=true content_hash=f003d8ad25546238e63b98d28a08f942c8e3cf57251df694de81bea293a6dd26 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:3f4f20574caef4ee6153e60f6acc5f41e9ed5afe63b2cafd69396bafb18f9051 cost=0.00182 escalated=false esc_model="" detail="markdown-заголовок в выходе: ### Глава 7" -ch8/0 draft snap_match=true content_hash=fbd27af11da2275cd1d47a0224970a5f8d073c6164f614efb6522ea96fa1e021 disp=ok flag="" attempts=1 final_hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb cost=0.00182 escalated=false esc_model="" detail="" -ch8/0 edit snap_match=true content_hash=40c2dd5b7b713800e8453930779bf2c88e2a296056f594bc1a4f5c495956915a disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:31dcd0e8a631cbce9ef7873f139bc415902858a708ea16602c8fc38f77cf3b66 cost=0.00182 escalated=false esc_model="" detail="CJK-утечка в ru-выходе: 特产" +ch7/0 draft snap_match=true content_hash=d90c335cba3d1547a22bb3f656443cb432f84c12a577d43509cfe472cf86a2ac disp=ok flag="" attempts=1 final_hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a cost=0.00182 escalated=false esc_model="" detail="" +ch7/0 edit snap_match=true content_hash=f003d8ad25546238e63b98d28a08f942c8e3cf57251df694de81bea293a6dd26 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:e1f2765b9f6d3cc4c461471cd7c143476614c602b6b08843bd239c77beb3b102 cost=0.00182 escalated=false esc_model="" detail="markdown-заголовок в выходе: ### Глава 7" +ch8/0 draft snap_match=true content_hash=fbd27af11da2275cd1d47a0224970a5f8d073c6164f614efb6522ea96fa1e021 disp=ok flag="" attempts=1 final_hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 cost=0.00182 escalated=false esc_model="" detail="" +ch8/0 edit snap_match=true content_hash=40c2dd5b7b713800e8453930779bf2c88e2a296056f594bc1a4f5c495956915a disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:067cfd0a649bfcd3db7fd04e8beff2a610dba06e292664e0160f06d16eef5209 cost=0.00182 escalated=false esc_model="" detail="CJK-утечка в ru-выходе: 特产" -- request_log (insertion order) -- -ch1/0 draft role=translator req=fake-model actual=fake-model hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/0 edit role=editor req=fake-model actual=fake-model hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/1 draft role=translator req=fake-model actual=fake-model hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/1 edit role=editor req=fake-model actual=fake-model hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 draft role=translator req=fake-model actual=fake-model hash=afeec2c4175c036a72332a9d6e27ab48a5584bb3819ca5a3c5b95ad01160ed74 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" -ch2/0 edit role=editor req=fake-model actual=fake-model hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch3/0 draft role=translator req=fake-model actual=fake-model hash=cb19e60076fb427f7294b8a0b80e2d810af3630d16353892f83a49b947affdd0 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=bc3af2566b80f34217bcbf5898895ab00887355f3f1e8a427d5d504683d41902 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" -ch4/0 draft role=translator req=fake-model actual=fake-model hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch4/0 edit role=editor req=fake-model actual=fake-model hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch5/0 draft role=translator req=fake-model actual=fake-model hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch5/0 edit role=editor req=fake-model actual=fake-model hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch6/0 draft role=translator req=fake-model actual=fake-model hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch6/0 edit role=editor req=fake-model actual=fake-model hash=b76db9cd65fd0cde6bcc4d2d8cc77ae9628e23ebeff2653cb59740b458f1960c tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch7/0 draft role=translator req=fake-model actual=fake-model hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch7/0 edit role=editor req=fake-model actual=fake-model hash=327f47fe5e882624cb3c01c1e0dcfde566e8b695cbe482de8e4916eb90d6f45b tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch8/0 draft role=translator req=fake-model actual=fake-model hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch8/0 edit role=editor req=fake-model actual=fake-model hash=5b836ac514f218026f8f61f4d8ef6e1a68b7625ec2dde3cb95e196b6ac55864d tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" -ch1/0 draft role=translator req=fake-model actual=fake-model hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch1/0 edit role=editor req=fake-model actual=fake-model hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch1/1 draft role=translator req=fake-model actual=fake-model hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch1/1 edit role=editor req=fake-model actual=fake-model hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch2/0 draft role=translator req=fake-model actual=fake-fallback hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch2/0 edit role=editor req=fake-model actual=fake-model hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch1/0 draft role=translator req=fake-model actual=fake-model hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/0 edit role=editor req=fake-model actual=fake-model hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/1 draft role=translator req=fake-model actual=fake-model hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/1 edit role=editor req=fake-model actual=fake-model hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 draft role=translator req=fake-model actual=fake-model hash=5f4763e07c67f541958817ce6e7a65c8277e1d806558d3e218e92362ba858866 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" +ch2/0 edit role=editor req=fake-model actual=fake-model hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch3/0 draft role=translator req=fake-model actual=fake-model hash=be200a513ccbfbb410ffc7788967610e2323aeda610c4d25296c8f06ddf3aead tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=4e5ad1c52475c43b5f98f0a369187a11367e4c739b67d5d9269b9949b718de33 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err="" +ch4/0 draft role=translator req=fake-model actual=fake-model hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch4/0 edit role=editor req=fake-model actual=fake-model hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch5/0 draft role=translator req=fake-model actual=fake-model hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch5/0 edit role=editor req=fake-model actual=fake-model hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch6/0 draft role=translator req=fake-model actual=fake-model hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch6/0 edit role=editor req=fake-model actual=fake-model hash=ae7cb45838ddec818f5e6c8f71410db7f00365dc8d1a41ec1a94af1d490e2178 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch7/0 draft role=translator req=fake-model actual=fake-model hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch7/0 edit role=editor req=fake-model actual=fake-model hash=9394388fc09349b7e4014e12ae991f041c5912307964d83ae165a8c99a3aac2e tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch8/0 draft role=translator req=fake-model actual=fake-model hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch8/0 edit role=editor req=fake-model actual=fake-model hash=fd0a61bd15ec394e35f291a3f114469a0fca591b3bc529ef8bd79686fe3bef93 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err="" +ch1/0 draft role=translator req=fake-model actual=fake-model hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch1/0 edit role=editor req=fake-model actual=fake-model hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch1/1 draft role=translator req=fake-model actual=fake-model hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch1/1 edit role=editor req=fake-model actual=fake-model hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch2/0 draft role=translator req=fake-model actual=fake-fallback hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch2/0 edit role=editor req=fake-model actual=fake-model hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" ch3/0 draft role=translator req=fake-model actual=fake-model hash= tm_hit=1 ok=0 finish="" degraded="hard_refusal" cost=0 tokens=0/0/0/0/0 err="" -ch4/0 draft role=translator req=fake-model actual=fake-model hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch4/0 edit role=editor req=fake-model actual=fake-model hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch5/0 draft role=translator req=fake-model actual=fake-model hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch5/0 edit role=editor req=fake-model actual=fake-model hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch6/0 draft role=translator req=fake-model actual=fake-model hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch4/0 draft role=translator req=fake-model actual=fake-model hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch4/0 edit role=editor req=fake-model actual=fake-model hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch5/0 draft role=translator req=fake-model actual=fake-model hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch5/0 edit role=editor req=fake-model actual=fake-model hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch6/0 draft role=translator req=fake-model actual=fake-model hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" ch6/0 edit role=editor req=fake-model actual=fake-model hash= tm_hit=1 ok=0 finish="" degraded="sanitizer_defect" cost=0 tokens=0/0/0/0/0 err="" -ch7/0 draft role=translator req=fake-model actual=fake-model hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch7/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:3f4f20574caef4ee6153e60f6acc5f41e9ed5afe63b2cafd69396bafb18f9051 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err="" -ch8/0 draft role=translator req=fake-model actual=fake-model hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" -ch8/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:31dcd0e8a631cbce9ef7873f139bc415902858a708ea16602c8fc38f77cf3b66 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err="" +ch7/0 draft role=translator req=fake-model actual=fake-model hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch7/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:e1f2765b9f6d3cc4c461471cd7c143476614c602b6b08843bd239c77beb3b102 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err="" +ch8/0 draft role=translator req=fake-model actual=fake-model hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err="" +ch8/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:067cfd0a649bfcd3db7fd04e8beff2a610dba06e292664e0160f06d16eef5209 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err="" -- retrieval_state -- ch1/0 snap_match=true exact=3 sticky=0 ambiguous=0 spoiler=1 evicted=0 postcheck_miss=1 style_flags=0 trust_gated=0 injected_ids=["紋章\u001f\u001f0\u001f0","鈴木\u001f\u001f0\u001f0","魔法学院\u001f\u001f0\u001f0"] postcheck_detail=[{"src":"紋章","dst":"герб","disp":"confirmed"}] style_detail= trust_gate_detail=