Land track-A pack-1.5 with fix-list F1-F9: tmctl export surface with manifest and drift guards, fold-first sanitizer detection, bounded CJK-gloss whitelist, runnable zh-ru example
This commit is contained in:
parent
9dae0ed13d
commit
dc30c7d61d
16 changed files with 1303 additions and 169 deletions
73
backend/cmd/tmctl/export_cli_test.go
Normal file
73
backend/cmd/tmctl/export_cli_test.go
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/pipeline"
|
||||
)
|
||||
|
||||
// export_cli_test.go pins the tmctl surface of the read-only export projection (D39 слой 6): the
|
||||
// parse of the `export` command + `--plaintext`, the default stable-JSON render, and the human
|
||||
// plaintext render (banner + text, flagged chunks marked, dropped chunks empty).
|
||||
|
||||
func TestParseExportPlaintext(t *testing.T) {
|
||||
inv, err := parseInvocation([]string{"export", "--config", "b.yaml", "--plaintext"}, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatalf("export --plaintext must parse: %v", err)
|
||||
}
|
||||
if inv.cmd != "export" || inv.cfgPath != "b.yaml" || !inv.asPlaintext {
|
||||
t.Fatalf("inv = %+v", inv)
|
||||
}
|
||||
// Default (no flag) is JSON, not plaintext.
|
||||
inv2, err := parseInvocation([]string{"export", "--config", "b.yaml"}, &bytes.Buffer{})
|
||||
if err != nil || inv2.asPlaintext {
|
||||
t.Fatalf("export default must be JSON (asPlaintext=false): inv=%+v err=%v", inv2, err)
|
||||
}
|
||||
}
|
||||
|
||||
func sampleExport() *pipeline.BookExport {
|
||||
return &pipeline.BookExport{BookID: "b1", Chunks: []pipeline.ChunkExport{
|
||||
{Chapter: 1, ChunkIdx: 0, SnapshotID: "snap", Disposition: "ok", FinalText: "Чистый текст."},
|
||||
{Chapter: 2, ChunkIdx: 0, SnapshotID: "snap", Disposition: "flagged", FlagReason: "sanitizer_stripped", FinalText: "Очищенный текст."},
|
||||
{Chapter: 3, ChunkIdx: 0, SnapshotID: "snap", Disposition: "flagged", FlagReason: "sanitizer_defect", FinalText: ""},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestRenderExportJSONIsStable(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
if err := renderExport(&b, sampleExport(), false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The default surface is valid JSON the polygon extractor can consume: it round-trips and carries
|
||||
// every chunk's export bytes and metadata.
|
||||
var got pipeline.BookExport
|
||||
if err := json.Unmarshal(b.Bytes(), &got); err != nil {
|
||||
t.Fatalf("export JSON must round-trip: %v\n%s", err, b.String())
|
||||
}
|
||||
if got.BookID != "b1" || len(got.Chunks) != 3 {
|
||||
t.Fatalf("round-tripped export = %+v", got)
|
||||
}
|
||||
if got.Chunks[1].FinalText != "Очищенный текст." || got.Chunks[1].FlagReason != "sanitizer_stripped" {
|
||||
t.Fatalf("stripped chunk lost fidelity: %+v", got.Chunks[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderExportPlaintextBanners(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
if err := renderExport(&b, sampleExport(), true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := b.String()
|
||||
for _, want := range []string{
|
||||
"=== ГЛАВА 1 ЧАНК 0 ===\nЧистый текст.",
|
||||
"=== ГЛАВА 2 ЧАНК 0 — flagged (sanitizer_stripped) (утечка вычищена, проверь) ===\nОчищенный текст.",
|
||||
"=== ГЛАВА 3 ЧАНК 0 — flagged (sanitizer_defect) (не переведён, помечен для человека) ===",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("plaintext export must contain %q, got:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,11 +15,12 @@ import (
|
|||
|
||||
// invocation is one parsed tmctl command line.
|
||||
type invocation struct {
|
||||
cmd string
|
||||
cfgPath string
|
||||
resnapshot bool
|
||||
asJSON bool
|
||||
sel pipeline.RedriveSelector
|
||||
cmd string
|
||||
cfgPath string
|
||||
resnapshot bool
|
||||
asJSON bool
|
||||
asPlaintext bool
|
||||
sel pipeline.RedriveSelector
|
||||
}
|
||||
|
||||
// parseInvocation parses os.Args[1:] into an invocation. flagOut receives the
|
||||
|
|
@ -37,7 +38,7 @@ type invocation struct {
|
|||
// main() → exit 1, оставляя 2 эксклюзивным для флагнутых чанков.
|
||||
func parseInvocation(args []string, flagOut io.Writer) (invocation, error) {
|
||||
if len(args) < 1 {
|
||||
return invocation{}, fmt.Errorf("usage: tmctl <translate|report|status|redrive> --config book.yaml")
|
||||
return invocation{}, fmt.Errorf("usage: tmctl <translate|report|status|export|redrive> --config book.yaml")
|
||||
}
|
||||
cmd, rest := args[0], args[1:]
|
||||
|
||||
|
|
@ -46,6 +47,7 @@ func parseInvocation(args []string, flagOut io.Writer) (invocation, error) {
|
|||
cfgPath := fs.String("config", "", "path to book.yaml")
|
||||
resnapshot := fs.Bool("resnapshot", false, "re-pin existing jobs to the current config snapshot (пере-перевод оплаченных чанков — явное согласие)")
|
||||
asJSON := fs.Bool("json", false, "status: emit the projection as JSON (stable disposition/flag_reason enums) for CI/IDE")
|
||||
asPlaintext := fs.Bool("plaintext", false, "export: emit the concatenated human text instead of the default stable JSON")
|
||||
chapter := fs.Int("chapter", -1, "redrive: restrict to this chapter (default: any)")
|
||||
chunk := fs.Int("chunk", -1, "redrive: restrict to this chunk index within the chapter (default: any)")
|
||||
reason := fs.String("reason", "", "redrive: restrict to this flag_reason (default: any)")
|
||||
|
|
@ -57,7 +59,7 @@ func parseInvocation(args []string, flagOut io.Writer) (invocation, error) {
|
|||
return invocation{}, fmt.Errorf("--config book.yaml is required")
|
||||
}
|
||||
return invocation{
|
||||
cmd: cmd, cfgPath: *cfgPath, resnapshot: *resnapshot, asJSON: *asJSON,
|
||||
cmd: cmd, cfgPath: *cfgPath, resnapshot: *resnapshot, asJSON: *asJSON, asPlaintext: *asPlaintext,
|
||||
sel: pipeline.RedriveSelector{
|
||||
Chapter: *chapter, ChunkIdx: *chunk, Reason: *reason, DryRun: *dryRun,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ import (
|
|||
|
||||
func TestParseNoArgsUsage(t *testing.T) {
|
||||
_, err := parseInvocation(nil, &bytes.Buffer{})
|
||||
if err == nil || err.Error() != "usage: tmctl <translate|report|status|redrive> --config book.yaml" {
|
||||
// The command list gained `export` (D39 слой 6 read-only surface, a deliberate contract extension);
|
||||
// the rest of the usage text stays frozen.
|
||||
if err == nil || err.Error() != "usage: tmctl <translate|report|status|export|redrive> --config book.yaml" {
|
||||
t.Fatalf("usage error text is frozen, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,10 +68,12 @@ func run() error {
|
|||
return report(inv.cfgPath)
|
||||
case "status":
|
||||
return status(ctx, inv.cfgPath, inv.asJSON)
|
||||
case "export":
|
||||
return export(inv.cfgPath, inv.asPlaintext)
|
||||
case "redrive":
|
||||
return redrive(ctx, inv.cfgPath, inv.resnapshot, inv.sel)
|
||||
default:
|
||||
return fmt.Errorf("unknown command %q (want translate|report|status|redrive)", inv.cmd)
|
||||
return fmt.Errorf("unknown command %q (want translate|report|status|export|redrive)", inv.cmd)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -118,6 +120,25 @@ func report(cfgPath string) error {
|
|||
return renderQuality(os.Stdout, q)
|
||||
}
|
||||
|
||||
// export prints the READ-ONLY export projection (D39 слой 6 / D39.2-open №1): every chunk's FINAL
|
||||
// export text EXACTLY by prod semantics (final_hash → checkpoint → exportNormalize), so the polygon
|
||||
// extractor / reader-samples read the SAME bytes the backend ships instead of the raw checkpoint. Like
|
||||
// report/status it is $0 and needs no provider keys (D20.4 — an audit surface must not demand keys).
|
||||
// Default output is stable JSON (the machine surface); --plaintext emits the human concatenation.
|
||||
func export(cfgPath string, asPlaintext bool) error {
|
||||
r, err := pipeline.NewReadOnlyRunner(cfgPath, obs.NewLogger())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
exp, err := r.Export()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return renderExport(os.Stdout, exp, asPlaintext)
|
||||
}
|
||||
|
||||
// status prints the READ-ONLY progress projection (D15.3): 0 LLM, 0 replay. --json emits the
|
||||
// StatusReport with stable disposition/flag_reason enums for CI/IDE; the default is a human
|
||||
// dashboard (unit counts, per-chapter quality passports, money, secondary ETA). BOTH modes exit 2
|
||||
|
|
|
|||
|
|
@ -223,11 +223,62 @@ func renderQuality(w io.Writer, q *pipeline.QualityReport) error {
|
|||
q.MeanSentPerNarrPara, q.NarrativeSentences, q.NarrativeParagraphs)
|
||||
fmt.Fprintf(w, "СИГНАЛЫ: диалог-тире=%d · глоссарий-промахов=%d · число-дрейф=%d · trust-gated(сид)=%d\n",
|
||||
q.DialogueDashFlags, q.GlossaryMisses, q.NumberDriftFlags, q.TrustGated)
|
||||
fmt.Fprintf(w, "УТЕЧКИ: CJK-strip чанков=%d (%.1f%%) · echo(cjk_artifact) чанков=%d (%.1f%%)\n",
|
||||
q.CJKLeakChunks, 100*q.CJKLeakRate, q.EchoChunks, 100*q.EchoRate)
|
||||
fmt.Fprintf(w, "СТРИПЫ/УТЕЧКИ: косметик-strip чанков=%d (%.1f%%, markdown+CJK) · echo(cjk_artifact) чанков=%d (%.1f%%)\n",
|
||||
q.CosmeticStripChunks, 100*q.CosmeticStripRate, q.EchoChunks, 100*q.EchoRate)
|
||||
return nil
|
||||
}
|
||||
|
||||
// renderExport prints the read-only export projection (D39 слой 6 / D39.2-open №1): every chunk's
|
||||
// FINAL export text EXACTLY as `tmctl translate` ships it (export-normalised), for the polygon
|
||||
// extractor / reader-samples to read the same bytes the backend does instead of the raw checkpoint.
|
||||
// The DEFAULT is stable JSON (the machine surface the extractor consumes); --plaintext emits the human
|
||||
// concatenation (a per-chunk header banner + the text, flagged chunks marked). Pure $0 read — never an
|
||||
// exit-2 sentinel (unlike translate/status): export is an audit projection, not a run verdict.
|
||||
func renderExport(w io.Writer, exp *pipeline.BookExport, asPlaintext bool) error {
|
||||
if asPlaintext {
|
||||
// Manifest/drift summary first (F3/F4): a partial or drifted book is EXPLICIT, not silently
|
||||
// exported as complete.
|
||||
fmt.Fprintf(w, "# export %s — чанков всего=%d, экспортировано=%d, ожидают=%d",
|
||||
exp.BookID, exp.TotalChunks, exp.TotalChunks-exp.PendingChunks, exp.PendingChunks)
|
||||
if exp.GhostRows > 0 {
|
||||
fmt.Fprintf(w, ", ghost-строк-отброшено=%d", exp.GhostRows)
|
||||
}
|
||||
if exp.ConfigDrift {
|
||||
fmt.Fprintf(w, " ⚠ CONFIG-DRIFT (текущий конфиг рендерит другой снапшот — гейт/стадия могли измениться после прогона; %.12s)", exp.CurrentSnapshot)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
for _, ce := range exp.Chunks {
|
||||
switch {
|
||||
case ce.Disposition == "pending":
|
||||
fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — pending (ещё не переведён) ===\n", ce.Chapter, ce.ChunkIdx)
|
||||
case ce.Disposition == string(pipeline.DispOK):
|
||||
fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d ===\n", ce.Chapter, ce.ChunkIdx)
|
||||
case ce.FinalText != "":
|
||||
// Cosmetic sanitizer strip (D35.4a): auto-cleaned remainder, flagged for a human.
|
||||
fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — %s%s (утечка вычищена, проверь) ===\n",
|
||||
ce.Chapter, ce.ChunkIdx, ce.Disposition, flagParen(ce.FlagReason))
|
||||
default:
|
||||
// Substantive flag / upstream skip: no export text (D2 — contaminated output never ships).
|
||||
fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — %s%s (не переведён, помечен для человека) ===\n",
|
||||
ce.Chapter, ce.ChunkIdx, ce.Disposition, flagParen(ce.FlagReason))
|
||||
}
|
||||
fmt.Fprintln(w, ce.FinalText)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(exp)
|
||||
}
|
||||
|
||||
// flagParen renders a non-empty flag reason as " (reason)" for the plaintext export banner.
|
||||
func flagParen(r string) string {
|
||||
if r == "" {
|
||||
return ""
|
||||
}
|
||||
return " (" + r + ")"
|
||||
}
|
||||
|
||||
// renderStatusJSON emits the StatusReport as indented JSON (stable enums — the
|
||||
// ratified CI/IDE contract, D12/D15.3) and maps flagged>0 to the exit-2 sentinel.
|
||||
func renderStatusJSON(w io.Writer, rep *pipeline.StatusReport) error {
|
||||
|
|
|
|||
|
|
@ -1,21 +1,35 @@
|
|||
# Пример проекта книги для приёмки Фазы 0: один чанк ja→ru через C1.
|
||||
book_id: sample-ja
|
||||
title: 夜明けの図書館 # «Библиотека на рассвете» — собственный тестовый фрагмент
|
||||
source_lang: ja
|
||||
# Пример проекта книги: один чанк zh→ru через C1 — прод-пара пайплайна (pipeline-c1 несёт
|
||||
# промпты, ключёванные по паре, только для zh-ru: D39 слой 2). Собственный тестовый фрагмент.
|
||||
book_id: sample-zh
|
||||
title: 黎明藏书阁 # «Книгохранилище на рассвете» — собственный тестовый фрагмент
|
||||
source_lang: zh
|
||||
target_lang: ru
|
||||
genre: ранобэ
|
||||
audience: взрослые читатели вебновелл
|
||||
adult: false
|
||||
venuti: 0.6
|
||||
honorifics: keep
|
||||
transcription: polivanov
|
||||
transcription: palladius # zh→ru: система Палладия (пиньинь — латинская, не для ru-таргета)
|
||||
footnotes: minimal
|
||||
|
||||
pipeline: ../configs/pipeline-c1.yaml
|
||||
models: ../configs/models.yaml
|
||||
source_file: chapter1-chunk0.txt
|
||||
project_db: sample-ja.db
|
||||
source_file: chapter1-zh.txt
|
||||
project_db: sample-zh.db
|
||||
|
||||
ceilings:
|
||||
book_usd: 1.00 # потолок $ на книгу — ledger обязан работать и в тестах
|
||||
day_usd: 2.00
|
||||
|
||||
# --- Демонстрация pair-сеама (D39 слой 2), НЕ активная конфигурация ------------------
|
||||
# pipeline-c1 несёт промпты только для пары zh-ru (conventions написаны под китайский
|
||||
# исходник). Книга с source_lang: ja даёт LangPair() = "ja-ru", которого в паке нет, и
|
||||
# загрузка ПАДАЕТ ГРОМКО на loadTemplates (никогда молча не подставляем чужую пару). Чтобы
|
||||
# увидеть сеам, поменяйте выше на блок ниже (ja-фрагмент лежит в chapter1-chunk0.txt):
|
||||
# source_lang: ja
|
||||
# target_lang: ru
|
||||
# transcription: polivanov
|
||||
# source_file: chapter1-chunk0.txt
|
||||
# Запуск такого конфига fail-loud'ится с «no prompt for language pair "ja-ru"» — это
|
||||
# сигнал добавить ja-ru пакет промптов, а НЕ баг. ja-ru пакет здесь НЕ сочиняется:
|
||||
# контент пары = данные под реальную книгу (инвариант общности §0.1).
|
||||
|
|
|
|||
11
backend/example/chapter1-zh.txt
Normal file
11
backend/example/chapter1-zh.txt
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
黎明前的藏书阁,被一片寂静笼罩着。
|
||||
|
||||
「师兄,我们真的可以进来吗?」美樱压低声音问道。她的手里,紧握着一把古旧的铜钥匙。
|
||||
|
||||
「放心,长老已经准许了,」拓海答道,「趁天亮之前,把那卷古籍找出来。」
|
||||
|
||||
两人在书架之间静静穿行。窗外,东方的天际泛起一丝鱼肚白。尘埃的气味,与旧纸的清香,在空气中浮动。美樱停下脚步,向一本书伸出手去。
|
||||
|
||||
「师兄……你看这个。封面上什么都没有写。」
|
||||
|
||||
拓海回过头的刹那,藏书阁深处传来一声轻响,仿佛有什么东西坠落了。
|
||||
56
backend/internal/config/example_test.go
Normal file
56
backend/internal/config/example_test.go
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// example_test.go pins that the shipped example project (example/book.yaml) is RUNNABLE through the
|
||||
// pair-keyed prompt seam (D39 слой 2 / D39.2 T3). Reframed to the prod pair zh→ru, LoadBook +
|
||||
// LoadPipeline resolve, every stage's prompt resolves for the book's pair and exists on disk, and a
|
||||
// ja-ru pair still FAILS LOUD — the seam the example's commented-out ja block documents. The old ja
|
||||
// framing regressed to a fail-loud after the pair seam landed (D39.2-open №4); this is the guard that
|
||||
// the reframe fixed it without silently smuggling a ja book through the zh-ru conventions.
|
||||
func TestExampleProjectIsRunnableZhRu(t *testing.T) {
|
||||
book, err := LoadBook("../../example/book.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("example book must load (source_file etc. must exist): %v", err)
|
||||
}
|
||||
if book.LangPair() != "zh-ru" {
|
||||
t.Fatalf("example must be reframed to the prod pair zh→ru, got LangPair() = %q", book.LangPair())
|
||||
}
|
||||
models, err := LoadModels(book.ModelsFile)
|
||||
if err != nil {
|
||||
t.Fatalf("example models must load: %v", err)
|
||||
}
|
||||
pipe, err := LoadPipeline(book.Pipeline, models)
|
||||
if err != nil {
|
||||
t.Fatalf("example pipeline must load: %v", err)
|
||||
}
|
||||
if len(pipe.Stages) == 0 {
|
||||
t.Fatal("example pipeline has no stages")
|
||||
}
|
||||
// zh-ru: every stage resolves a prompt for the book's pair AND the resolved file exists (no
|
||||
// fail-loud, no dangling path) — the exact end-to-end the pair seam broke for the old ja framing.
|
||||
for _, st := range pipe.Stages {
|
||||
p, err := st.PromptPathFor(book.LangPair())
|
||||
if err != nil {
|
||||
t.Fatalf("stage %q: zh-ru prompt must resolve: %v", st.Name, err)
|
||||
}
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
t.Fatalf("stage %q: resolved prompt %q must exist on disk: %v", st.Name, p, err)
|
||||
}
|
||||
}
|
||||
// ja-ru: the seam still bites a pair the pack does not carry (what the commented-out ja block in
|
||||
// example/book.yaml documents) — loadTemplates fails on the first such stage, so at least one must.
|
||||
jaFailed := false
|
||||
for _, st := range pipe.Stages {
|
||||
if _, err := st.PromptPathFor("ja-ru"); err != nil {
|
||||
jaFailed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !jaFailed {
|
||||
t.Fatal("a ja-ru pair must fail loud through the example pipeline (the pair seam)")
|
||||
}
|
||||
}
|
||||
206
backend/internal/pipeline/export.go
Normal file
206
backend/internal/pipeline/export.go
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"textmachine/backend/internal/store"
|
||||
)
|
||||
|
||||
// export.go: the read-only EXPORT projection (D39 слой 6 / D39.2-open №1; the deferred `tmctl export`
|
||||
// of D15.2-этап-Б in its MINIMAL read form). It emits every chunk's FINAL export text EXACTLY by prod
|
||||
// semantics, so the polygon extractor (records.json / exp12_extract) and reader-samples read the SAME
|
||||
// export-normalised bytes `tmctl translate` ships — instead of the RAW stored checkpoint, which skips
|
||||
// the export contract for a clean chunk. The owner chose a BACKEND surface over a Python mirror of
|
||||
// exportNormalize (which would drift against the Go contract). It is a PURE READ-ONLY projection like
|
||||
// Status/QualityReport: $0, no LLM, no keys, no snapshot touch — it reads the persisted chunk_status +
|
||||
// the $0 final checkpoint and applies the very same exportNormalize the ChunkOutcome does. Only the READ
|
||||
// half is built here; the annotation/override half of the D15.2 `export` command is deliberately NOT.
|
||||
|
||||
// exportPending marks a manifest chunk that has not (yet) reached the final stage — no exported text.
|
||||
const exportPending = "pending"
|
||||
|
||||
// ChunkExport is one chunk's final export record: the metadata plus the export-normalised final text.
|
||||
type ChunkExport struct {
|
||||
Chapter int `json:"chapter"`
|
||||
ChunkIdx int `json:"chunk_idx"`
|
||||
SnapshotID string `json:"snapshot_id,omitempty"`
|
||||
// Disposition is the CHUNK-level verdict mirroring ChunkOutcome: "ok" only when the final stage is
|
||||
// ok; "flagged" for a flagged OR upstream-skipped final row; "pending" for a manifest chunk with no
|
||||
// final-stage row (not yet translated).
|
||||
Disposition string `json:"disposition"`
|
||||
FlagReason string `json:"flag_reason,omitempty"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
// FinalText is the export-normalised final text EXACTLY as `tmctl translate` ships it:
|
||||
// exportNormalize(final checkpoint response) for an ok or cosmetic-stripped chunk, "" for a
|
||||
// substantive flag / upstream-skip / pending chunk (exportNormalize("")=="", so the paths coincide
|
||||
// byte-for-byte with ChunkOutcome.FinalText).
|
||||
FinalText string `json:"final_text"`
|
||||
}
|
||||
|
||||
// BookExport is the whole-book export projection (chunks in manifest — (chapter, chunk_idx) — order, so
|
||||
// the JSON is deterministic across runs). The counters make a partial/empty/drifted book EXPLICIT
|
||||
// instead of silently exporting as complete (F4/F3).
|
||||
type BookExport struct {
|
||||
BookID string `json:"book_id"`
|
||||
// TotalChunks is the current source manifest size (the honest denominator); ExportedChunks +
|
||||
// PendingChunks == TotalChunks. GhostRows are stored final rows dropped as OUTSIDE the manifest.
|
||||
TotalChunks int `json:"total_chunks"`
|
||||
PendingChunks int `json:"pending_chunks"`
|
||||
GhostRows int `json:"ghost_rows,omitempty"`
|
||||
// ConfigDrift is true when the CURRENT config renders a snapshot different from the one the stored
|
||||
// rows carry (a gate flip / prompt bump / stage rename since the run) — the gate/stage re-derivation
|
||||
// below may then not match what translate actually did. CurrentSnapshot is that projected id.
|
||||
ConfigDrift bool `json:"config_drift"`
|
||||
CurrentSnapshot string `json:"current_snapshot,omitempty"`
|
||||
// Chunks is always non-nil (an empty book exports [] not null).
|
||||
Chunks []ChunkExport `json:"chunks"`
|
||||
}
|
||||
|
||||
// Export builds the read-only per-chunk export projection. It opens no jobs, reserves nothing, makes no
|
||||
// LLM call — it joins the $0 source manifest against the persisted chunk_status (final-stage rows) and,
|
||||
// for a chunk that produced exported text, the $0 final checkpoint, applying exportNormalize exactly as
|
||||
// translateChunk does. Safe to run whenever `report`/`status` are (the same exclusive-lock rule).
|
||||
// Deterministic over the store + source.
|
||||
//
|
||||
// CAVEATS (surfaced, never silent):
|
||||
// - CONFIG/GATE DRIFT (F3): the gate flag AND the final-stage NAME are read from the CURRENT config.
|
||||
// A config edit since the run (a glossary post-check gate FLIP, a prompt bump, a final-stage rename)
|
||||
// changes the projected snapshot; Export sets ConfigDrift + WARNs so the gate re-derivation / row
|
||||
// lookup mismatch is visible (a run under the same config reads correctly; `status` shows it too).
|
||||
// - MANIFEST (F4): a chunk absent from the current source is a GHOST row (dropped + counted + WARNed);
|
||||
// a manifest chunk with no final row is PENDING; both keep a partial/edited book from exporting as
|
||||
// if complete (the D37-skew the polygon extractor must not inherit).
|
||||
func (r *Runner) Export() (*BookExport, error) {
|
||||
statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The glossary post-check GATE (opt-in) flips a chunk to flagged/glossary_miss at the CHUNK level
|
||||
// AFTER the stage loop, so it is NEVER written to chunk_status — every stage row stays DispOK
|
||||
// (chunkrun.translateChunk records only retrieval_state). Export must re-derive it, exactly as
|
||||
// Status does: a raw chunk_status read would report a gate-flagged chunk as clean ok AND ship the
|
||||
// contaminated text `tmctl translate` deliberately withheld. Read retrieval_state only when the gate
|
||||
// is on (else the map stays empty and the branch is a no-op).
|
||||
gateOn := r.Pipeline.Gates.Glossary.PostcheckGate
|
||||
missByChunk := map[chunkKey]int{}
|
||||
if gateOn {
|
||||
states, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rs := range states {
|
||||
missByChunk[chunkKey{rs.Chapter, rs.ChunkIdx}] = rs.NPostcheckMiss // CONFIRMED-miss count
|
||||
}
|
||||
}
|
||||
lastStage := ""
|
||||
if n := len(r.Pipeline.Stages); n > 0 {
|
||||
lastStage = r.Pipeline.Stages[n-1].Name
|
||||
}
|
||||
// Index the final-stage rows by chunk key (the exported text + verdict live there) and record the
|
||||
// snapshot ids the rows carry (for the drift check).
|
||||
finalRow := map[chunkKey]store.ChunkStatus{}
|
||||
snapSeen := map[string]bool{}
|
||||
for _, cs := range statuses {
|
||||
if cs.Stage != lastStage {
|
||||
continue // the exported text and the chunk's final verdict live on the final stage's row
|
||||
}
|
||||
finalRow[chunkKey{cs.Chapter, cs.ChunkIdx}] = cs
|
||||
if cs.SnapshotID != "" {
|
||||
snapSeen[cs.SnapshotID] = true
|
||||
}
|
||||
}
|
||||
|
||||
// F4: the $0 source manifest is the authoritative chunk set — join it so a PENDING chunk is explicit
|
||||
// and a GHOST (deleted-source) row is not exported as if live.
|
||||
manifest, err := r.bookChunks()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exp := &BookExport{BookID: r.Book.BookID, Chunks: []ChunkExport{}}
|
||||
inManifest := map[chunkKey]bool{}
|
||||
for _, ch := range manifest {
|
||||
k := chunkKey{ch.Chapter, ch.ChunkIdx}
|
||||
inManifest[k] = true
|
||||
exp.TotalChunks++
|
||||
cs, ok := finalRow[k]
|
||||
if !ok {
|
||||
exp.PendingChunks++
|
||||
exp.Chunks = append(exp.Chunks, ChunkExport{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: exportPending})
|
||||
continue
|
||||
}
|
||||
ce, err := r.chunkExport(cs, gateOn, missByChunk[k])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exp.Chunks = append(exp.Chunks, ce)
|
||||
}
|
||||
// F4: GHOST rows — a stored final row whose chunk is NOT in the current manifest (source shrunk since
|
||||
// the run). Dropped (never exported as if live) but counted + WARNed so the drop is loud.
|
||||
for k := range finalRow {
|
||||
if !inManifest[k] {
|
||||
exp.GhostRows++
|
||||
}
|
||||
}
|
||||
if exp.GhostRows > 0 {
|
||||
r.Log.Warn("export: dropped chunk_status rows outside the current source manifest (source shrunk since the run?)",
|
||||
"book", r.Book.BookID, "ghost_rows", exp.GhostRows)
|
||||
}
|
||||
|
||||
// F3: config/gate drift — only meaningful on a single stored snapshot (multi-snapshot drift is
|
||||
// already visible in the per-chunk snapshot_id). Compute the CURRENT config's projected snapshot
|
||||
// read-only and compare; surface a mismatch as a flag + WARN, never silently.
|
||||
if len(snapSeen) == 1 {
|
||||
var stored string
|
||||
for s := range snapSeen {
|
||||
stored = s
|
||||
}
|
||||
if cur, serr := r.currentSnapshotProjected(); serr != nil {
|
||||
r.Log.Warn("export: config-drift check failed; drift state unknown (reported as none)", "err", serr)
|
||||
} else if cur != stored {
|
||||
exp.ConfigDrift = true
|
||||
exp.CurrentSnapshot = cur
|
||||
r.Log.Warn("export: CONFIG-DRIFT — current config renders a different snapshot than the stored rows; the gate/stage re-derivation may not match the run (translate would require --resnapshot)",
|
||||
"book", r.Book.BookID, "stored", stored, "current", cur)
|
||||
}
|
||||
}
|
||||
return exp, nil
|
||||
}
|
||||
|
||||
// chunkExport computes one chunk's export record from its final-stage row, exactly by prod FinalText
|
||||
// semantics (chunkrun.translateChunk): an ok chunk exports exportNormalize(primary checkpoint), a
|
||||
// cosmetic sanitizer-strip exports exportNormalize(its derived stripped checkpoint) — both via
|
||||
// final_hash — the glossary gate override exports "", and every other flag / upstream skip exports "".
|
||||
func (r *Runner) chunkExport(cs store.ChunkStatus, gateOn bool, gateMiss int) (ChunkExport, error) {
|
||||
ce := ChunkExport{Chapter: cs.Chapter, ChunkIdx: cs.ChunkIdx, SnapshotID: cs.SnapshotID, Detail: cs.Detail}
|
||||
// Post-check gate override (mirrors translateChunk): an otherwise-ok chunk (DispOK ⇒ every stage ok)
|
||||
// with a CONFIRMED miss is flagged glossary_miss and exports NOTHING — the withheld text never ships.
|
||||
if gateOn && cs.Disposition == string(DispOK) && gateMiss > 0 {
|
||||
ce.Disposition = string(DispFlagged)
|
||||
ce.FlagReason = string(FlagGlossaryMiss)
|
||||
return ce, nil
|
||||
}
|
||||
if cs.Disposition == string(DispOK) {
|
||||
ce.Disposition = string(DispOK)
|
||||
if cs.FinalHash == "" {
|
||||
// F9: an ok chunk MUST carry a final_hash — an empty one is an inconsistent store. Fail LOUD
|
||||
// (parity with the missing-checkpoint case), never silently export "".
|
||||
return ce, fmt.Errorf("pipeline: export ch%d/chunk%d: DispOK row has an empty final_hash (inconsistent store)", cs.Chapter, cs.ChunkIdx)
|
||||
}
|
||||
} else {
|
||||
// A flagged final row OR a skipped one (an upstream stage flagged) is a flagged CHUNK; the final
|
||||
// row carries the propagated flag reason (chunkrun records it on the skipped row too).
|
||||
ce.Disposition = string(DispFlagged)
|
||||
ce.FlagReason = cs.FlagReason
|
||||
}
|
||||
if cs.FinalHash != "" && (cs.Disposition == string(DispOK) || cs.FlagReason == string(FlagSanitizerStripped)) {
|
||||
cp, err := r.Store.GetCheckpoint(cs.FinalHash)
|
||||
if err != nil {
|
||||
return ce, err
|
||||
}
|
||||
if cp == nil {
|
||||
return ce, fmt.Errorf("pipeline: export ch%d/chunk%d: chunk_status.final_hash %.12s has no checkpoint (inconsistent store)", cs.Chapter, cs.ChunkIdx, cs.FinalHash)
|
||||
}
|
||||
ce.FinalText = exportNormalize(cp.ResponseText)
|
||||
}
|
||||
return ce, nil
|
||||
}
|
||||
305
backend/internal/pipeline/export_test.go
Normal file
305
backend/internal/pipeline/export_test.go
Normal file
|
|
@ -0,0 +1,305 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/obs"
|
||||
)
|
||||
|
||||
// export_test.go pins the read-only export surface (D39 слой 6 / D39.2-open №1): `tmctl export` must
|
||||
// emit each chunk's FINAL text EXACTLY as `tmctl translate` ships it (export-normalised), for BOTH a
|
||||
// fresh run and a $0 resume — the whole point being that the polygon extractor reads the SAME bytes
|
||||
// the backend does, not the RAW stored checkpoint (which skips the export contract for a clean chunk).
|
||||
|
||||
// TestExportMatchesTranslateFinalText runs a real 3-chunk book across the three export outcomes and
|
||||
// asserts Export() reproduces BookResult's FinalText byte-for-byte — and that a clean chunk carrying a
|
||||
// normalize-relevant artifact is exported NORMALISED (not the raw checkpoint), which is the fix.
|
||||
func TestExportMatchesTranslateFinalText(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
// Three chapters (\f-separated) → three 1-chunk chapters. The editor (final stage) emits:
|
||||
// ch1: clean prose carrying a U+3000 indent + fullwidth digits → OK, but export must NORMALISE it;
|
||||
// ch2: a sparse contentless CJK-leak run (below the echo threshold) → cosmetic sanitizer strip;
|
||||
// ch3: a leaked service preamble → SUBSTANTIVE sanitizer defect → dropped (FinalText "").
|
||||
const (
|
||||
src1 = "ГЛАВААА"
|
||||
src2 = "ГЛАВАББ"
|
||||
src3 = "ГЛАВАВВ"
|
||||
)
|
||||
editCleanArtifact := "Судзуки шёл по коридору. 123 шага до двери." // U+3000 + «123»
|
||||
editCosmeticLeak := "Он поднял кулак 羅漢 и ударил врага." // sparse Han run → strip
|
||||
editPreamble := "Вот перевод фрагмента:\nОн молча ушёл в туман." // preamble → dropped
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if !isEditBody(body) {
|
||||
return "черновик перевода этой главы.", "stop" // any draft — clean, non-echo
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(body, src1):
|
||||
return editCleanArtifact, "stop"
|
||||
case strings.Contains(body, src2):
|
||||
return editCosmeticLeak, "stop"
|
||||
default: // src3
|
||||
return editPreamble, "stop"
|
||||
}
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
// Enable the output-sanitizer so the cosmetic-strip (ch2) and substantive-drop (ch3) paths engage.
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||||
source: src1 + "\f" + src2 + "\f" + src3,
|
||||
gatesYAML: "gates:\n sanitizer:\n enabled: true\n",
|
||||
})
|
||||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
res, err := r1.TranslateBook(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(res.Chunks) != 3 {
|
||||
t.Fatalf("want 3 chunks, got %d: %+v", len(res.Chunks), res.Chunks)
|
||||
}
|
||||
|
||||
// The authoritative FinalText per chunk, straight from the translate result.
|
||||
type want struct {
|
||||
final string
|
||||
disp Disposition
|
||||
flag FlagReason
|
||||
}
|
||||
byChunk := map[[2]int]want{}
|
||||
for _, ch := range res.Chunks {
|
||||
byChunk[[2]int{ch.Chapter, ch.ChunkIdx}] = want{ch.FinalText, ch.Disposition, ch.FlagReason}
|
||||
}
|
||||
|
||||
// The clean chunk MUST be export-normalised, not raw: no U+3000, no fullwidth digits survive.
|
||||
ch1 := byChunk[[2]int{1, 0}]
|
||||
if strings.ContainsRune(ch1.final, ' ') || strings.Contains(ch1.final, "123") {
|
||||
t.Fatalf("ch1 FinalText was not export-normalised: %q", ch1.final)
|
||||
}
|
||||
if ch1.final != "Судзуки шёл по коридору. 123 шага до двери." {
|
||||
t.Fatalf("ch1 FinalText = %q, want the normalised form", ch1.final)
|
||||
}
|
||||
// The cosmetic-leak chunk is flagged-stripped with the Han run removed but the prose kept.
|
||||
ch2 := byChunk[[2]int{2, 0}]
|
||||
if ch2.disp != DispFlagged || ch2.flag != FlagSanitizerStripped || strings.Contains(ch2.final, "羅漢") || ch2.final == "" {
|
||||
t.Fatalf("ch2 expected stripped-non-empty flagged export, got %+v", ch2)
|
||||
}
|
||||
// The preamble chunk is a substantive drop: flagged with an EMPTY export.
|
||||
ch3 := byChunk[[2]int{3, 0}]
|
||||
if ch3.disp != DispFlagged || ch3.final != "" {
|
||||
t.Fatalf("ch3 expected substantive drop with empty export, got %+v", ch3)
|
||||
}
|
||||
|
||||
assertExportEquals := func(tag string, r *Runner) {
|
||||
t.Helper()
|
||||
exp, err := r.Export()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: Export: %v", tag, err)
|
||||
}
|
||||
if len(exp.Chunks) != 3 {
|
||||
t.Fatalf("%s: want 3 export chunks, got %d", tag, len(exp.Chunks))
|
||||
}
|
||||
for _, ce := range exp.Chunks {
|
||||
w, ok := byChunk[[2]int{ce.Chapter, ce.ChunkIdx}]
|
||||
if !ok {
|
||||
t.Fatalf("%s: export emitted an unknown chunk %d/%d", tag, ce.Chapter, ce.ChunkIdx)
|
||||
}
|
||||
if ce.FinalText != w.final {
|
||||
t.Errorf("%s: ch%d/%d export text = %q, want translate FinalText %q", tag, ce.Chapter, ce.ChunkIdx, ce.FinalText, w.final)
|
||||
}
|
||||
if ce.Disposition != string(w.disp) || ce.FlagReason != string(w.flag) {
|
||||
t.Errorf("%s: ch%d/%d export disp/flag = %s/%s, want %s/%s", tag, ce.Chapter, ce.ChunkIdx, ce.Disposition, ce.FlagReason, w.disp, w.flag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assertExportEquals("fresh", r1)
|
||||
r1.Close()
|
||||
|
||||
// Resume (a new process): the book is served from checkpoints at $0, and Export must reproduce the
|
||||
// identical bytes — the derived stripped checkpoint and the raw ones both re-normalise deterministically.
|
||||
callsBefore := rec.count()
|
||||
r2 := newRunner(t, bookPath)
|
||||
defer r2.Close()
|
||||
res2, err := r2.TranslateBook(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.count() != callsBefore {
|
||||
t.Fatalf("resume must not call the provider again: %d -> %d", callsBefore, rec.count())
|
||||
}
|
||||
if res2.TotalUSD != 0 {
|
||||
t.Fatalf("resume must be $0, got %v", res2.TotalUSD)
|
||||
}
|
||||
assertExportEquals("resume", r2)
|
||||
}
|
||||
|
||||
// TestExportGlossaryGateWithheld pins the chunk-level glossary post-check gate case (adversarial-review
|
||||
// finding): the gate flips a chunk to flagged/glossary_miss AFTER the stage loop and never writes it to
|
||||
// chunk_status, so a raw chunk_status read would report the chunk as clean ok and ship the text
|
||||
// `translate` withheld. Export must re-derive the gate flag from retrieval_state (like Status) so it
|
||||
// matches translate byte-for-byte: flagged, glossary_miss, EMPTY export.
|
||||
func TestExportGlossaryGateWithheld(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
return "ОТРЕДАКТИРОВАННЫЙ.", "stop" // final text drops the approved name → confirmed miss
|
||||
}
|
||||
return "Некто пошёл в библиотеку.", "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
// Opt-in hard gate + a seed whose approved 鈴木→Судзуки is missed in the final.
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true})
|
||||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
res, err := r.TranslateBook(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Precondition: translate withheld this chunk (flagged glossary_miss, empty text).
|
||||
if len(res.Chunks) != 1 || res.Chunks[0].FlagReason != FlagGlossaryMiss || res.Chunks[0].FinalText != "" {
|
||||
t.Fatalf("precondition: want 1 chunk flagged glossary_miss with empty text, got %+v", res.Chunks)
|
||||
}
|
||||
exp, err := r.Export()
|
||||
if err != nil {
|
||||
t.Fatalf("Export: %v", err)
|
||||
}
|
||||
if len(exp.Chunks) != 1 {
|
||||
t.Fatalf("want 1 export chunk, got %d", len(exp.Chunks))
|
||||
}
|
||||
ce := exp.Chunks[0]
|
||||
// Export MUST match translate — NOT ship the withheld text as clean ok.
|
||||
if ce.Disposition != string(DispFlagged) || ce.FlagReason != string(FlagGlossaryMiss) || ce.FinalText != "" {
|
||||
t.Fatalf("export leaked a gate-withheld chunk: disp=%s reason=%s final=%q (want flagged/glossary_miss/\"\")",
|
||||
ce.Disposition, ce.FlagReason, ce.FinalText)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExportManifestPending pins F4: a PARTIAL book (a ceiling stopped the run mid-way) exports the
|
||||
// untranslated chunks as explicit "pending" rows against the $0 manifest — not silently as a complete
|
||||
// book (the D37-skew the polygon extractor must not inherit). Chunks is non-nil.
|
||||
func TestExportManifestPending(t *testing.T) {
|
||||
srv := newJSONProvider(&reqRec{}, draftEdit)
|
||||
defer srv.Close()
|
||||
// Three chapters; a book ceiling that pays for ~one chapter (2 calls ≈ $0.0036) then denies.
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||||
source: "ГЛАВАА\fГЛАВАБ\fГЛАВАВ", regenerate: 0, bookUSD: 0.005,
|
||||
})
|
||||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
// The ceiling stop is an infra error — expected; we only need the partial store state it leaves.
|
||||
_, _ = r.TranslateBook(ctx)
|
||||
|
||||
exp, err := r.Export()
|
||||
if err != nil {
|
||||
t.Fatalf("Export: %v", err)
|
||||
}
|
||||
if exp.TotalChunks != 3 {
|
||||
t.Fatalf("manifest must be 3 chunks, got total=%d", exp.TotalChunks)
|
||||
}
|
||||
if exp.PendingChunks < 1 {
|
||||
t.Fatalf("a ceiling-stopped book must have pending chunks, got pending=%d (%+v)", exp.PendingChunks, exp.Chunks)
|
||||
}
|
||||
if len(exp.Chunks) != exp.TotalChunks {
|
||||
t.Fatalf("every manifest chunk must have a row: len=%d total=%d", len(exp.Chunks), exp.TotalChunks)
|
||||
}
|
||||
var pending int
|
||||
for _, ce := range exp.Chunks {
|
||||
if ce.Disposition == "pending" {
|
||||
pending++
|
||||
if ce.FinalText != "" {
|
||||
t.Errorf("a pending chunk must export no text: %+v", ce)
|
||||
}
|
||||
}
|
||||
}
|
||||
if pending != exp.PendingChunks {
|
||||
t.Errorf("pending rows (%d) must match the counter (%d)", pending, exp.PendingChunks)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExportDetectsConfigDrift pins F3: after a wire-affecting config edit (a prompt_version bump) since
|
||||
// the run, Export flags ConfigDrift — the gate/stage re-derivation may not match what translate did.
|
||||
func TestExportDetectsConfigDrift(t *testing.T) {
|
||||
srv := newJSONProvider(&reqRec{}, draftEdit)
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r1.Close()
|
||||
|
||||
// Unchanged config → no drift.
|
||||
r2 := newRunner(t, bookPath)
|
||||
exp, err := r2.Export()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exp.ConfigDrift {
|
||||
t.Fatalf("an unchanged config must not drift (current=%s)", exp.CurrentSnapshot)
|
||||
}
|
||||
r2.Close()
|
||||
|
||||
// Bump a wire-affecting field → the current config renders a new snapshot → drift.
|
||||
dir := filepath.Dir(bookPath)
|
||||
body, err := os.ReadFile(filepath.Join(dir, "pipeline.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changed := strings.Replace(string(body), "prompt_version: v-test", "prompt_version: v-test2", 1)
|
||||
if changed == string(body) {
|
||||
t.Fatal("setup: prompt_version token not found")
|
||||
}
|
||||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), changed)
|
||||
|
||||
r3 := newRunner(t, bookPath)
|
||||
defer r3.Close()
|
||||
exp3, err := r3.Export()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exp3.ConfigDrift || exp3.CurrentSnapshot == "" {
|
||||
t.Errorf("a prompt_version bump since the run must surface as ConfigDrift: %+v", exp3)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExportFailLoudOnInconsistentStore pins F9: a DispOK final row with an EMPTY final_hash is an
|
||||
// inconsistent store; Export fails LOUD rather than silently exporting "".
|
||||
func TestExportFailLoudOnInconsistentStore(t *testing.T) {
|
||||
srv := newJSONProvider(&reqRec{}, draftEdit)
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 0})
|
||||
ctx := obs.WithReqInfo(context.Background(), obs.ReqInfo{TraceID: obs.NewTraceID()})
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
res, err := r.TranslateBook(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Corrupt the final-stage row: DispOK but no final_hash (simulating store inconsistency).
|
||||
lastStage := r.Pipeline.Stages[len(r.Pipeline.Stages)-1].Name
|
||||
ch := res.Chunks[0]
|
||||
cs, err := r.Store.GetChunkStatus(r.Book.BookID, ch.Chapter, ch.ChunkIdx, lastStage)
|
||||
if err != nil || cs == nil {
|
||||
t.Fatalf("read final row: %v cs=%v", err, cs)
|
||||
}
|
||||
cs.FinalHash = ""
|
||||
if err := r.Store.UpsertChunkStatus(*cs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Export(); err == nil {
|
||||
t.Fatal("Export must fail loud on a DispOK row with an empty final_hash")
|
||||
} else if !strings.Contains(err.Error(), "empty final_hash") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -36,17 +36,19 @@ type QualityReport struct {
|
|||
MeanSentPerNarrPara float64 `json:"mean_sentences_per_narrative_paragraph"`
|
||||
|
||||
// Deterministic signal aggregates (all observability, never a disposition).
|
||||
DialogueDashFlags int `json:"dialogue_dash_flags"` // Rosenthal dialogue-dash inconsistencies
|
||||
GlossaryMisses int `json:"glossary_misses"` // CONFIRMED post-check misses (D10 consistency)
|
||||
NumberDriftFlags int `json:"number_drift_flags"` // reflow number drift + 万/億 magnitude drift
|
||||
TrustGated int `json:"trust_gated"` // lower-trust suppressions refused (seed hygiene)
|
||||
CJKLeakChunks int `json:"cjk_leak_chunks"` // chunks the sanitizer stripped a CJK leak from
|
||||
EchoChunks int `json:"echo_chunks"` // chunks flagged cjk_artifact (untranslated echo)
|
||||
DialogueDashFlags int `json:"dialogue_dash_flags"` // Rosenthal dialogue-dash inconsistencies
|
||||
GlossaryMisses int `json:"glossary_misses"` // CONFIRMED post-check misses (D10 consistency)
|
||||
NumberDriftFlags int `json:"number_drift_flags"` // reflow number drift + 万/億 magnitude drift
|
||||
TrustGated int `json:"trust_gated"` // lower-trust suppressions refused (seed hygiene)
|
||||
CosmeticStripChunks int `json:"cosmetic_strip_chunks"` // chunks the sanitizer auto-stripped (markdown header OR CJK leak — F6: was mislabeled cjk_leak)
|
||||
EchoChunks int `json:"echo_chunks"` // chunks flagged cjk_artifact (untranslated echo)
|
||||
|
||||
// Rates over ProcessedChunks (0..1) for the two "instant unreadability" families, so the numbers
|
||||
// read as a bounded fraction of processed chunks rather than a bare count.
|
||||
CJKLeakRate float64 `json:"cjk_leak_rate"`
|
||||
EchoRate float64 `json:"echo_rate"`
|
||||
// read as a bounded fraction of processed chunks rather than a bare count. CosmeticStripRate covers
|
||||
// BOTH strip classes (a markdown-only strip is NOT a CJK leak — F6, D39.4: the old cjk_leak_rate
|
||||
// counted every sanitizer_stripped chunk and read as false CJK leakage).
|
||||
CosmeticStripRate float64 `json:"cosmetic_strip_rate"`
|
||||
EchoRate float64 `json:"echo_rate"`
|
||||
|
||||
Chunks []ChunkQuality `json:"chunks,omitempty"`
|
||||
}
|
||||
|
|
@ -114,6 +116,13 @@ func (r *Runner) QualityReport() (*QualityReport, error) {
|
|||
return q
|
||||
}
|
||||
|
||||
// The glossary post-check GATE flips a chunk to withheld (flagged glossary_miss, empty export) at
|
||||
// the CHUNK level without a chunk_status row (like Export/Status re-derive it). F5 (D39.4): the
|
||||
// structural KPI must EXCLUDE these — their export is "", so counting their (withheld) text in
|
||||
// TextChunks/KPI diverges from what `tmctl export` and `translate` actually ship.
|
||||
gateOn := r.Pipeline.Gates.Glossary.PostcheckGate
|
||||
withheld := map[chunkKey]bool{}
|
||||
|
||||
// Aggregate the stored retrieval-state signals (glossary consistency, style breakdown, trust-gated).
|
||||
for _, rs := range states {
|
||||
q := chunkOf(chunkKey{rs.Chapter, rs.ChunkIdx})
|
||||
|
|
@ -121,6 +130,9 @@ func (r *Runner) QualityReport() (*QualityReport, error) {
|
|||
q.TrustGated += rs.NTrustGatedSuppress
|
||||
rep.GlossaryMisses += rs.NPostcheckMiss
|
||||
rep.TrustGated += rs.NTrustGatedSuppress
|
||||
if gateOn && rs.NPostcheckMiss > 0 {
|
||||
withheld[chunkKey{rs.Chapter, rs.ChunkIdx}] = true
|
||||
}
|
||||
if rs.NStyleFlags > 0 && rs.StyleDetail != "" {
|
||||
var cg cheapGateResult
|
||||
if json.Unmarshal([]byte(rs.StyleDetail), &cg) == nil {
|
||||
|
|
@ -159,7 +171,7 @@ func (r *Runner) QualityReport() (*QualityReport, error) {
|
|||
chunkOf(k) // ensure the chunk row exists even if it has no retrieval-state signals
|
||||
}
|
||||
if cs.FlagReason == string(FlagSanitizerStripped) {
|
||||
rep.CJKLeakChunks++ // a stripped chunk carried a markdown/CJK cosmetic leak (detail says which)
|
||||
rep.CosmeticStripChunks++ // a stripped chunk carried a markdown OR CJK cosmetic leak (F6)
|
||||
}
|
||||
// Structural KPI: recompute over the exported final text (ok, or the cosmetic-stripped export).
|
||||
if cs.FinalHash == "" {
|
||||
|
|
@ -168,6 +180,9 @@ func (r *Runner) QualityReport() (*QualityReport, error) {
|
|||
if cs.Disposition != string(DispOK) && cs.FlagReason != string(FlagSanitizerStripped) {
|
||||
continue // a dropped chunk exported nothing
|
||||
}
|
||||
if withheld[k] {
|
||||
continue // F5: the glossary gate withheld this chunk's text — it exports nothing
|
||||
}
|
||||
cp, cperr := r.Store.GetCheckpoint(cs.FinalHash)
|
||||
if cperr != nil {
|
||||
return nil, cperr
|
||||
|
|
@ -187,7 +202,7 @@ func (r *Runner) QualityReport() (*QualityReport, error) {
|
|||
rep.MeanSentPerNarrPara = float64(rep.NarrativeSentences) / float64(rep.NarrativeParagraphs)
|
||||
}
|
||||
if rep.ProcessedChunks > 0 {
|
||||
rep.CJKLeakRate = float64(rep.CJKLeakChunks) / float64(rep.ProcessedChunks)
|
||||
rep.CosmeticStripRate = float64(rep.CosmeticStripChunks) / float64(rep.ProcessedChunks)
|
||||
rep.EchoRate = float64(rep.EchoChunks) / float64(rep.ProcessedChunks)
|
||||
}
|
||||
for _, k := range order {
|
||||
|
|
|
|||
|
|
@ -66,8 +66,8 @@ func TestQualityReportAggregates(t *testing.T) {
|
|||
t.Fatalf("chunk counts: total=%d processed=%d text=%d, want 1/1/1", q.TotalChunks, q.ProcessedChunks, q.TextChunks)
|
||||
}
|
||||
// Rates are bounded [0,1] over ProcessedChunks (the disjoint-set bug fix).
|
||||
if q.EchoRate < 0 || q.EchoRate > 1 || q.CJKLeakRate < 0 || q.CJKLeakRate > 1 {
|
||||
t.Errorf("rates must be within [0,1], got echo=%.3f cjk=%.3f", q.EchoRate, q.CJKLeakRate)
|
||||
if q.EchoRate < 0 || q.EchoRate > 1 || q.CosmeticStripRate < 0 || q.CosmeticStripRate > 1 {
|
||||
t.Errorf("rates must be within [0,1], got echo=%.3f cosmetic=%.3f", q.EchoRate, q.CosmeticStripRate)
|
||||
}
|
||||
if q.NarrativeSentences != 3 || q.NarrativeParagraphs != 2 {
|
||||
t.Errorf("structural KPI = %d sent / %d para, want 3/2", q.NarrativeSentences, q.NarrativeParagraphs)
|
||||
|
|
@ -76,10 +76,46 @@ func TestQualityReportAggregates(t *testing.T) {
|
|||
t.Errorf("mean sentences/narrative-paragraph = %.3f, want 1.5", q.MeanSentPerNarrPara)
|
||||
}
|
||||
// A clean run: no glossary misses, no echo, no CJK strip, no trust-gated, no dialogue-dash issue.
|
||||
if q.GlossaryMisses != 0 || q.EchoChunks != 0 || q.CJKLeakChunks != 0 || q.TrustGated != 0 || q.DialogueDashFlags != 0 {
|
||||
if q.GlossaryMisses != 0 || q.EchoChunks != 0 || q.CosmeticStripChunks != 0 || q.TrustGated != 0 || q.DialogueDashFlags != 0 {
|
||||
t.Errorf("clean run must have zero defect signals, got %+v", q)
|
||||
}
|
||||
if len(q.Chunks) != 1 || q.Chunks[0].NarrativeParagraphs != 2 {
|
||||
t.Errorf("per-chunk row missing/wrong: %+v", q.Chunks)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQualityReportExcludesGateWithheld pins F5 (D39.4): a chunk the glossary post-check GATE withheld
|
||||
// (flagged glossary_miss, empty export) must NOT be counted in TextChunks / the structural KPI — its
|
||||
// text never ships, so counting it would diverge from `tmctl export`/`translate`.
|
||||
func TestQualityReportExcludesGateWithheld(t *testing.T) {
|
||||
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
return "ОТРЕДАКТИРОВАННЫЙ.", "stop" // final drops the approved name → confirmed miss
|
||||
}
|
||||
return "Некто пошёл в библиотеку.", "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{regenerate: 1, source: suzukiSource, glossarySeed: suzukiSeed, postcheckGate: true})
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := r.QualityReport()
|
||||
if err != nil {
|
||||
t.Fatalf("QualityReport: %v", err)
|
||||
}
|
||||
// The single chunk is gate-withheld → processed but NOT text-bearing.
|
||||
if q.ProcessedChunks != 1 {
|
||||
t.Fatalf("the withheld chunk still reached the final stage: processed=%d", q.ProcessedChunks)
|
||||
}
|
||||
if q.TextChunks != 0 || q.NarrativeSentences != 0 || q.NarrativeParagraphs != 0 {
|
||||
t.Errorf("gate-withheld chunk must be excluded from the KPI, got text=%d sent=%d para=%d",
|
||||
q.TextChunks, q.NarrativeSentences, q.NarrativeParagraphs)
|
||||
}
|
||||
// It IS visible as a glossary miss (the observability signal is kept).
|
||||
if q.GlossaryMisses == 0 {
|
||||
t.Errorf("the confirmed miss must still be reported: %+v", q)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,6 +70,10 @@ func (r *Runner) resumeFromChunkStatus(ctx context.Context, st config.Stage, ch
|
|||
// checkpoint, so report ITS finish_reason ("sanitized_export") rather than a bare "" that
|
||||
// contradicts the referenced row (adversarial review NIT; telemetry-only).
|
||||
sr.FinishReason = cp.FinishReason
|
||||
// F7 (D39.4): mirror the ok branch's model attribution — the derived checkpoint carries the
|
||||
// ACTUAL model that produced the flagged output; without this the resumed telemetry row is
|
||||
// attributed to the config slug (st.Model) instead of the model that actually answered.
|
||||
sr.Model = cp.ModelActual
|
||||
}
|
||||
rl := r.baseRequestLog(st, ch, st.Model, cs.FinalHash)
|
||||
rl.ModelActual = sr.Model
|
||||
|
|
|
|||
|
|
@ -78,7 +78,16 @@ import (
|
|||
// delegates that fold too. This shifts which chunks flag and a stripped chunk's export → a loud
|
||||
// --resnapshot + golden re-capture (invariant №8). [exportNormalize itself is an EPHEMERAL export
|
||||
// projection, not a persisted input; only stripCosmetic's output rides a checkpoint, versioned here.]
|
||||
const sanitizerVersion = "sanitizer-v5"
|
||||
// v6 (D39.2 T2, owner decision 16.07: KEEP glosses; + D39.4 adversarial fix-list F1/F2): (a) a
|
||||
// whitelisted CJK term gloss — a balanced Han/kana run inside parens DIRECTLY after a Cyrillic/Latin
|
||||
// head word, «Кулак Ло Ханя (羅漢拳)» — is NOT a leak: detectCJKLeak no longer fires on it and
|
||||
// stripCosmetic/exportNormalize preserve its bytes; a BARE Han run stays a leak. (b) F1: ALL six classes
|
||||
// now DETECT over the export-normalised (folded) text, closing the fullwidth-signature gate bypass
|
||||
// (a «###»/«:» variant used to read clean while export manufactured the ASCII defect). (c) F2: a gloss
|
||||
// bounds its non-CJK content (inner length, proportional pinyin-token count, per-token length,
|
||||
// lowercase), so a leaked English clause after a token Han char is not whitelisted. All shift which
|
||||
// chunks flag and a stripped chunk's export → a loud --resnapshot + golden re-capture (invariant №8).
|
||||
const sanitizerVersion = "sanitizer-v6"
|
||||
|
||||
// sanitizer tuning constants (versioned by sanitizerVersion).
|
||||
const (
|
||||
|
|
@ -137,29 +146,46 @@ func (s sanitizerResult) summary() string {
|
|||
}
|
||||
|
||||
// sanitizeOutput runs all six classes over one chunk's FINAL translated text.
|
||||
//
|
||||
// FOLD-FIRST (F1, D39.4 — adversarial-found gate bypass): ALL six classes DETECT over the
|
||||
// export-normalised (recoverableFold-ed) text, not the raw text. Otherwise a FULLWIDTH variation of a
|
||||
// signature slips past the ASCII regexes while exportNormalize still MANUFACTURES the ASCII defect in
|
||||
// the shipped bytes: «Вот отредактированный перевод:»(U+FF1A fullwidth colon), «### Глава 5»
|
||||
// (fullwidth hash), a trailing «Примечание:…», fullwidth-Latin — all read total=0 on the raw text yet
|
||||
// export a real ASCII preamble/header/note/Latin leak (their ASCII twins ARE flagged, so it was a pure
|
||||
// bypass; the pre-D39 v4 folded the whole fullwidth block and its re-check caught this — a regression).
|
||||
// Only DETECTION folds; the strip/export paths (stripCosmetic/exportNormalize) still operate on the
|
||||
// ORIGINAL text so they preserve the original/gloss bytes. Gloss spans are re-anchored consistently:
|
||||
// the mask and detectCJKLeak both re-derive glossSegments over the SAME folded text they scan.
|
||||
// recoverableFold is idempotent, so detectCJKLeak's own internal fold is a no-op on the folded input.
|
||||
func sanitizeOutput(text string) sanitizerResult {
|
||||
var r sanitizerResult
|
||||
if det := detectLeadingPreamble(text); det != "" {
|
||||
folded := recoverableFold(text)
|
||||
// The two SCRIPT-based substantive classes (Latin insertion, broken word) additionally run with
|
||||
// whitelisted CJK glosses masked out (D39.2 T2): a gloss's own pinyin/Han must not read as a leaked
|
||||
// Latin phrase or a homoglyph token and drop the chunk. Masking is over the FOLDED text too.
|
||||
glossMasked := maskGlossSpans(folded)
|
||||
if det := detectLeadingPreamble(folded); det != "" {
|
||||
r.Preamble++
|
||||
r.Detail = append(r.Detail, det)
|
||||
}
|
||||
if det := detectTrailingNote(text); det != "" {
|
||||
if det := detectTrailingNote(folded); det != "" {
|
||||
r.TrailingNote++
|
||||
r.Detail = append(r.Detail, det)
|
||||
}
|
||||
if n, det := detectMarkdownHeaders(text); n > 0 {
|
||||
if n, det := detectMarkdownHeaders(folded); n > 0 {
|
||||
r.MarkdownHeader = n
|
||||
r.Detail = append(r.Detail, det...)
|
||||
}
|
||||
if det := detectLatinInsertion(text); det != "" {
|
||||
if det := detectLatinInsertion(glossMasked); det != "" {
|
||||
r.LatinInsert++
|
||||
r.Detail = append(r.Detail, det)
|
||||
}
|
||||
if n, det := detectBrokenWords(text); n > 0 {
|
||||
if n, det := detectBrokenWords(glossMasked); n > 0 {
|
||||
r.BrokenWord = n
|
||||
r.Detail = append(r.Detail, det...)
|
||||
}
|
||||
if n, det := detectCJKLeak(text); n > 0 {
|
||||
if n, det := detectCJKLeak(folded); n > 0 {
|
||||
r.CJKLeak = n
|
||||
r.Detail = append(r.Detail, det...)
|
||||
}
|
||||
|
|
@ -394,33 +420,196 @@ func isCJKLeakRune(r rune) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// --- CJK-gloss whitelist (D39.2 T2, owner decision 16.07: keep glosses) ---------
|
||||
|
||||
// glossParenRE finds a candidate parenthesised CJK gloss: a Cyrillic/Latin HEAD letter, an optional
|
||||
// short whitespace gap, then a single-level (NO nested paren, NO newline) «(...)» in ASCII OR fullwidth
|
||||
// parens. The gap class covers a plain space, a NON-BREAKING space, and the CJK-native IDEOGRAPHIC space
|
||||
// U+3000 (all bind a term to its gloss). U+3000 is load-bearing for detection↔strip CONSISTENCY (D39.4
|
||||
// review finding 1): the FOLD-FIRST detector sees U+3000 folded to a plain space and recognises the
|
||||
// gloss, but stripCosmetic/exportNormalize detect glosses on the ORIGINAL bytes — so without U+3000 in
|
||||
// the gap class a U+3000-bound gloss would read CLEAN in detection yet be DESTROYED by the strip. The
|
||||
// head requirement is load-bearing — a paren block with no letter head (line start, after punctuation, a
|
||||
// bare «(羅漢拳)») does NOT match, so its Han stays a leak. Group 1 is the whole paren block (the
|
||||
// protected span); isValidGloss then validates its inner content. RE2 \p classes are Unicode.
|
||||
var glossParenRE = regexp.MustCompile(`[\p{Cyrillic}\p{Latin}][ \x{00A0}\x{3000}]{0,2}(\([^()\n]*\)|([^()\n]*))`)
|
||||
|
||||
// Gloss bounds (D39.2 T2 + D39.4 F2): a whitelisted gloss is a short TERM with at most a proportional
|
||||
// pinyin romanisation, NOT a leaked clause smuggled in parens. maxGlossCJKRunes bounds the CJK term
|
||||
// (羅漢拳, 五行八卦掌); the F2 bounds keep the non-CJK content pinyin-plausible so «拳 the iron fist that
|
||||
// breaks the sky» is NOT whitelisted. [Tunable: flagged to the orchestrator; raise if a real term is
|
||||
// dropped in practice.]
|
||||
const (
|
||||
maxGlossCJKRunes = 6 // CJK chars in the term (九阳真经=4, 五行八卦掌=5)
|
||||
maxGlossInnerRunes = 40 // whole inner-content backstop (a clause is long)
|
||||
maxGlossLatinTokenRunes = 24 // one romanisation word ≈ 4×maxGlossCJKRunes: the JOINED pinyin of a ≤6-char term («jiǔyángzhēnjīng»=15, «wǔxíngbāguàzhǎng»=16) must fit, while a long English word («antidisestablishmentarianism»=28) exceeds it (D39.4 review finding 2)
|
||||
)
|
||||
|
||||
// isValidGloss reports whether a matched paren block «(inner)» is a whitelisted CJK TERM gloss: inner
|
||||
// carries 1..maxGlossCJKRunes CJK letters (Han/kana — the original term) plus at most a PROPORTIONAL
|
||||
// pinyin romanisation (Latin tokens ≤ cjk+1, each lowercase and ≤maxGlossLatinTokenRunes), and NOTHING
|
||||
// outside the gloss charset (CJK letters + Latin/digits/combining tone marks + the separators space «·»
|
||||
// «'» «-»), within maxGlossInnerRunes total. A parenthetical with Cyrillic prose, CJK PUNCTUATION
|
||||
// («、」»), a nested fullwidth paren, a too-long CJK run (a clause, not a term), or a non-proportional /
|
||||
// uppercase / over-long Latin run (a leaked English phrase, F2) fails — so any CJK inside stays a leak
|
||||
// (precision over recall: a false whitelist would smuggle a real leak past the detector to export).
|
||||
func isValidGloss(block string) bool {
|
||||
rs := []rune(block)
|
||||
if len(rs) < 3 { // «(x)» is the minimum
|
||||
return false
|
||||
}
|
||||
inner := rs[1 : len(rs)-1] // inner content, without the opening/closing paren
|
||||
if len(inner) > maxGlossInnerRunes {
|
||||
return false // a term gloss is short; a long run is a clause (F2)
|
||||
}
|
||||
cjk, latinToks, tokLen := 0, 0, 0
|
||||
for _, r := range inner {
|
||||
switch {
|
||||
case unicode.Is(unicode.Han, r), unicode.Is(unicode.Hiragana, r), unicode.Is(unicode.Katakana, r):
|
||||
cjk++
|
||||
if tokLen > 0 { // a CJK char ends the current Latin token
|
||||
latinToks++
|
||||
tokLen = 0
|
||||
}
|
||||
case unicode.Is(unicode.Latin, r):
|
||||
if unicode.IsUpper(r) {
|
||||
return false // pinyin romanisation is lowercase; an uppercase Latin run reads as prose (F2)
|
||||
}
|
||||
tokLen++
|
||||
if tokLen > maxGlossLatinTokenRunes {
|
||||
return false // a token longer than any romanisation word → not pinyin (F2)
|
||||
}
|
||||
case unicode.IsDigit(r), unicode.Is(unicode.Mn, r):
|
||||
// a pinyin tone digit / combining tone mark stays within the current Latin token
|
||||
if tokLen > 0 {
|
||||
tokLen++
|
||||
}
|
||||
case r == ' ' || r == '·' || r == '\'' || r == '-':
|
||||
if tokLen > 0 { // an intra-gloss separator ends the current Latin token
|
||||
latinToks++
|
||||
tokLen = 0
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
if tokLen > 0 {
|
||||
latinToks++
|
||||
}
|
||||
// The Latin tokens must be PROPORTIONAL to the term — pinyin transliterates the Han roughly
|
||||
// one token per character (+1 slack) — so a 1-CJK head with a 7-word English clause is rejected (F2).
|
||||
return cjk >= 1 && cjk <= maxGlossCJKRunes && latinToks <= cjk+1
|
||||
}
|
||||
|
||||
// maskGlossSpans replaces every whitelisted gloss block with a single space, so the SUBSTANTIVE
|
||||
// whole-text sanitizer classes that would otherwise mis-fire on a gloss's own bytes run gloss-blind: the
|
||||
// Latin-insertion class must not read a space-separated PINYIN romanisation «(五行八卦掌 wu xing ba gua
|
||||
// zhang)» as a leaked Latin phrase, and the broken-word class must not read a Han-glued pinyin token as a
|
||||
// homoglyph — both would drop the whole chunk to a placeholder, defeating the "keep glosses" intent. The
|
||||
// CJK-leak/strip/normalize paths use glossSegments directly (they must preserve the gloss bytes); these
|
||||
// substantive classes only need the gloss GONE from their view, so a plain mask suffices. Pure.
|
||||
func maskGlossSpans(text string) string {
|
||||
spans := glossSpans(text)
|
||||
if len(spans) == 0 {
|
||||
return text
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(text))
|
||||
pos := 0
|
||||
for _, sp := range spans {
|
||||
b.WriteString(text[pos:sp[0]])
|
||||
b.WriteByte(' ') // a single space breaks token adjacency and drops the gloss's pinyin/Han from view
|
||||
pos = sp[1]
|
||||
}
|
||||
b.WriteString(text[pos:])
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// glossSpans returns the byte ranges of the whitelisted CJK glosses in text (the paren blocks,
|
||||
// including their parens), left-to-right and non-overlapping (FindAll semantics). Detected on the
|
||||
// ORIGINAL text so the span bytes can be preserved verbatim (a fullwidth char that is PART of a gloss
|
||||
// is not folded — owner decision 16.07). Two adjacent glosses «(拳) и (腿)» yield two spans.
|
||||
func glossSpans(text string) [][2]int {
|
||||
m := glossParenRE.FindAllStringSubmatchIndex(text, -1)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
var spans [][2]int
|
||||
for _, idx := range m {
|
||||
s, e := idx[2], idx[3] // group 1 = the paren block
|
||||
if s < 0 {
|
||||
continue
|
||||
}
|
||||
if isValidGloss(text[s:e]) {
|
||||
spans = append(spans, [2]int{s, e})
|
||||
}
|
||||
}
|
||||
return spans
|
||||
}
|
||||
|
||||
// textSegment is one slice of a chunk split by glossSegments: a normal segment (folded/stripped/
|
||||
// detected) or a gloss segment (preserved verbatim).
|
||||
type textSegment struct {
|
||||
text string
|
||||
gloss bool
|
||||
}
|
||||
|
||||
// glossSegments splits text into alternating normal / gloss segments on the whitelisted gloss spans, so
|
||||
// the three export-contract functions treat a gloss the SAME way: normal segments are folded/detected/
|
||||
// stripped, gloss segments are passed through untouched. The whole text is one normal segment when no
|
||||
// gloss is present.
|
||||
func glossSegments(text string) []textSegment {
|
||||
spans := glossSpans(text)
|
||||
if len(spans) == 0 {
|
||||
return []textSegment{{text: text}}
|
||||
}
|
||||
var segs []textSegment
|
||||
pos := 0
|
||||
for _, sp := range spans {
|
||||
if sp[0] > pos {
|
||||
segs = append(segs, textSegment{text: text[pos:sp[0]]})
|
||||
}
|
||||
segs = append(segs, textSegment{text: text[sp[0]:sp[1]], gloss: true})
|
||||
pos = sp[1]
|
||||
}
|
||||
if pos < len(text) {
|
||||
segs = append(segs, textSegment{text: text[pos:]})
|
||||
}
|
||||
return segs
|
||||
}
|
||||
|
||||
// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail per
|
||||
// distinct run. It runs over the RECOVERABLE-FOLD of the text (D39 слой 6): a fullwidth «123» or a
|
||||
// «、» is a wrong-glyph rendering the export contract silently fixes, so it is NOT a leak — only a
|
||||
// genuinely contentless Han/kana/CJK-bracket run is. A single run («却有») is enough to fire — the
|
||||
// class is COSMETIC, so the whole chunk is not lost: stripCosmetic removes the run and the remainder
|
||||
// exports (flagged for review).
|
||||
// distinct run. It runs over the RECOVERABLE-FOLD of each NON-gloss segment (D39 слой 6): a fullwidth
|
||||
// «123» or a «、» is a wrong-glyph rendering the export contract silently fixes, so it is NOT a leak —
|
||||
// only a genuinely contentless Han/kana/CJK-bracket run is. A whitelisted CJK term gloss «(羅漢拳)» is
|
||||
// skipped entirely (D39.2 T2 — owner keeps glosses). A single run («却有») is enough to fire — the class
|
||||
// is COSMETIC, so the whole chunk is not lost: stripCosmetic removes the run and the remainder exports.
|
||||
func detectCJKLeak(text string) (int, []string) {
|
||||
var det []string
|
||||
seen := map[string]bool{}
|
||||
n := 0
|
||||
rs := []rune(recoverableFold(text))
|
||||
for i := 0; i < len(rs); {
|
||||
if !isCJKLeakRune(rs[i]) {
|
||||
i++
|
||||
continue
|
||||
for _, seg := range glossSegments(text) {
|
||||
if seg.gloss {
|
||||
continue // a whitelisted gloss is legitimate original-term content, not a leak
|
||||
}
|
||||
j := i
|
||||
for j < len(rs) && isCJKLeakRune(rs[j]) {
|
||||
j++
|
||||
rs := []rune(recoverableFold(seg.text))
|
||||
for i := 0; i < len(rs); {
|
||||
if !isCJKLeakRune(rs[i]) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
j := i
|
||||
for j < len(rs) && isCJKLeakRune(rs[j]) {
|
||||
j++
|
||||
}
|
||||
n++
|
||||
run := string(rs[i:j])
|
||||
if !seen[run] {
|
||||
seen[run] = true
|
||||
det = append(det, "CJK-утечка в ru-выходе: "+preview(run))
|
||||
}
|
||||
i = j
|
||||
}
|
||||
n++
|
||||
run := string(rs[i:j])
|
||||
if !seen[run] {
|
||||
seen[run] = true
|
||||
det = append(det, "CJK-утечка в ru-выходе: "+preview(run))
|
||||
}
|
||||
i = j
|
||||
}
|
||||
return n, det
|
||||
}
|
||||
|
|
@ -466,14 +655,25 @@ var ideographicPunctFold = strings.NewReplacer("、", ",", "。", ".")
|
|||
// free, no re-bill). Deliberately conservative on whitespace: it does NOT collapse internal doubled
|
||||
// spaces (a clean chunk's spacing is left as the model wrote it).
|
||||
func exportNormalize(text string) string {
|
||||
out := recoverableFold(text)
|
||||
// NFC composes a canonically-decomposed base+mark (a «й» stored as и+U+0306, a «ё» as е+U+0308)
|
||||
// BEFORE the combining-strip below, so the strip only removes the NON-composable stress accents
|
||||
// (U+0301 on a vowel), never corrupting «й»/«ё» into «и»/«е». NFC is canonical-only, so it leaves
|
||||
// «…» and other compatibility characters intact (unlike NFKC).
|
||||
out = norm.NFC.String(out)
|
||||
out = stripCombiningOnCyrillic(out)
|
||||
return trailingHSpaceRE.ReplaceAllString(out, "")
|
||||
var b strings.Builder
|
||||
b.Grow(len(text))
|
||||
for _, seg := range glossSegments(text) {
|
||||
if seg.gloss {
|
||||
// A whitelisted CJK term gloss is preserved VERBATIM — including any fullwidth char that is
|
||||
// part of the gloss, which is NOT folded (owner decision 16.07, D39.2 T2).
|
||||
b.WriteString(seg.text)
|
||||
continue
|
||||
}
|
||||
out := recoverableFold(seg.text)
|
||||
// NFC composes a canonically-decomposed base+mark (a «й» stored as и+U+0306, a «ё» as е+U+0308)
|
||||
// BEFORE the combining-strip below, so the strip only removes the NON-composable stress accents
|
||||
// (U+0301 on a vowel), never corrupting «й»/«ё» into «и»/«е». NFC is canonical-only, so it leaves
|
||||
// «…» and other compatibility characters intact (unlike NFKC).
|
||||
out = norm.NFC.String(out)
|
||||
out = stripCombiningOnCyrillic(out)
|
||||
b.WriteString(out)
|
||||
}
|
||||
return trailingHSpaceRE.ReplaceAllString(b.String(), "")
|
||||
}
|
||||
|
||||
// stripCombiningOnCyrillic drops a combining mark (unicode.Mn) that sits directly on a CYRILLIC base
|
||||
|
|
@ -506,22 +706,32 @@ func stripCombiningOnCyrillic(s string) string {
|
|||
// non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
|
||||
// sanitizer_stripped). It never crosses newlines when tidying, so paragraph structure is kept.
|
||||
func stripCosmetic(text string) string {
|
||||
// 1) markdown header prefixes → keep the heading text, drop the «#### » artifact.
|
||||
// 1) markdown header prefixes → keep the heading text, drop the «#### » artifact. Line-leading, so
|
||||
// it never touches a mid-line gloss span computed below.
|
||||
out := leadingMarkdownStripRE.ReplaceAllString(text, "$1")
|
||||
// 2) fold the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «123»→«123», U+3000→space,
|
||||
// «、»→«,» «。»→«.») via the shared export-contract fold — no longer a hand-rolled NFKC
|
||||
// (L5-stripcosmetic-duplicates-nfkc). After this only genuinely contentless CJK remains to drop.
|
||||
out = recoverableFold(out)
|
||||
// 3) drop the contentless CJK-leak runes (Han/kana/CJK brackets & symbols) the fold cannot
|
||||
// recover, each leaving a SINGLE SPACE so adjacent Russian words do not merge («нашёл特产древний»
|
||||
// → «нашёл древний», not «нашёлдревний»).
|
||||
// 2+3) per segment: fold the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «123»→«123»,
|
||||
// U+3000→space, «、»→«,» «。»→«.») via the shared export-contract fold (L5-stripcosmetic-duplicates-nfkc)
|
||||
// and drop the contentless CJK-leak runes the fold cannot recover, each leaving a SINGLE SPACE so
|
||||
// adjacent Russian words do not merge («нашёл特产древний»→«нашёл древний»). A whitelisted CJK term
|
||||
// gloss «(羅漢拳)» has its TERM BYTES preserved (D39.2 T2 — owner keeps glosses), so its parens no
|
||||
// longer empty out and its term is not dropped. [The step-4 whitespace tidy below runs over the
|
||||
// reassembled string, so it may normalise STRAY whitespace INSIDE a malformed gloss («(羅漢拳 )» →
|
||||
// «(羅漢拳)»); this is benign and does NOT diverge from what `tmctl translate`/`tmctl export` ship,
|
||||
// since translate's stripped-export path is exportNormalize(stripCosmetic(x)) and Export reads that
|
||||
// same derived checkpoint — both go through THIS function, so the shipped bytes agree.]
|
||||
var b strings.Builder
|
||||
b.Grow(len(out))
|
||||
for _, r := range out {
|
||||
if isCJKLeakRune(r) {
|
||||
b.WriteByte(' ')
|
||||
} else {
|
||||
b.WriteRune(r)
|
||||
for _, seg := range glossSegments(out) {
|
||||
if seg.gloss {
|
||||
b.WriteString(seg.text)
|
||||
continue
|
||||
}
|
||||
for _, r := range recoverableFold(seg.text) {
|
||||
if isCJKLeakRune(r) {
|
||||
b.WriteByte(' ')
|
||||
} else {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
out = b.String()
|
||||
|
|
|
|||
|
|
@ -291,8 +291,9 @@ func TestStripCosmetic(t *testing.T) {
|
|||
{"Индекс ABC найден.", "Индекс ABC найден."},
|
||||
// ideographic comma/period fold to plain — the clause boundary is kept.
|
||||
{"Он вошёл в дом、и дверь закрылась。", "Он вошёл в дом,и дверь закрылась."},
|
||||
// a parenthetical Han gloss is stripped and the emptied «()» removed (adversarial review).
|
||||
{"Кулак Ло Хань (羅漢拳) был силён.", "Кулак Ло Хань был силён."},
|
||||
// a whitelisted CJK term gloss is now PRESERVED (D39.2 T2 — owner keeps glosses); a co-occurring
|
||||
// BARE Han run (no head, not in parens) is still stripped, and the gloss parens do not empty out.
|
||||
{"Кулак Ло Хань (羅漢拳) ударил 特产 врага.", "Кулак Ло Хань (羅漢拳) ударил врага."},
|
||||
// a legit ratio «2 : 1» is NOT reformatted (colon dropped from the tidy set).
|
||||
{"### Итог\n\nСчёт был 2 : 1 却有 в пользу гостей.", "Итог\n\nСчёт был 2 : 1 в пользу гостей."},
|
||||
// U+3000 ideographic space normalised to a plain space (the indent artifact).
|
||||
|
|
@ -348,6 +349,133 @@ func TestExportNormalize(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestSanitizerCJKGlossWhitelist pins the CJK-gloss whitelist (D39.2 T2, owner decision 16.07: KEEP
|
||||
// glosses): a balanced CJK run inside parens directly after a Cyrillic/Latin head word is NOT a leak —
|
||||
// detectCJKLeak does not flag it and stripCosmetic/exportNormalize preserve its bytes; a bare Han run
|
||||
// (no head, line start, after punctuation, CJK punctuation inside, nested paren) is STILL a leak.
|
||||
func TestSanitizerCJKGlossWhitelist(t *testing.T) {
|
||||
// Whitelisted glosses: NOT flagged, PRESERVED verbatim by both strip and normalize.
|
||||
preserved := []string{
|
||||
"Кулак Ло Хань (羅漢拳) был силён.", // the owner's canonical case (ASCII parens)
|
||||
"Он изучил дар (資質) древнего клана.", // gloss with no space before paren-adjacent head
|
||||
"Приём Ло Хань(羅漢拳) сработал.", // head directly adjacent to «(» (no space)
|
||||
"Два приёма: Кулак (拳) и Нога (腿) в бою.", // two glosses in a row — both preserved
|
||||
"Стиль Тайцзи (太極拳 tàijíquán) он знал.", // Han + pinyin with tone marks and a space
|
||||
"Техника Zhang (張三豐) древних мастеров.", // Latin head word, Han gloss
|
||||
"Название клана(羅漢)на воротах.", // FULLWIDTH parens — preserved verbatim, not folded
|
||||
"Кулак (羅漢拳) был силён.", // NBSP between head and paren (finding B) — term kept
|
||||
"Приём (五行八卦掌 wu xing ba gua zhang) стар.", // 5 space-separated PINYIN syllables (finding A)
|
||||
"Он изучил приём (九阳真经 jiǔyángzhēnjīng) там.", // JOINED pinyin of a 4-char term (D39.4 finding 2)
|
||||
}
|
||||
for _, s := range preserved {
|
||||
if got := sanitizeOutput(s); got.CJKLeak != 0 {
|
||||
t.Errorf("gloss FALSE-flagged as leak in %q: %v", preview(s), got.Detail)
|
||||
}
|
||||
if got := stripCosmetic(s); got != s {
|
||||
t.Errorf("stripCosmetic altered a preserved gloss:\n in %q\n got %q", s, got)
|
||||
}
|
||||
if got := exportNormalize(s); got != s {
|
||||
t.Errorf("exportNormalize altered a preserved gloss:\n in %q\n got %q", s, got)
|
||||
}
|
||||
}
|
||||
// FINDING A: a space-separated pinyin gloss must NOT trip the Latin-insertion class and DROP the whole
|
||||
// chunk (detectLatinInsertion runs gloss-masked). The chunk must be fully clean (nothing fired).
|
||||
if got := sanitizeOutput("Приём (五行八卦掌 wu xing ba gua zhang) стар."); got.total() != 0 {
|
||||
t.Errorf("pinyin gloss must not flag the chunk (finding A), got %+v", got)
|
||||
}
|
||||
// FINDING B: a DOUBLE space between head and paren — the gloss TERM is kept and the chunk not flagged
|
||||
// (the tidy normalises the double space to one, so it is not byte-verbatim, hence a separate check).
|
||||
if got := sanitizeOutput("Кулак (羅漢拳) был силён."); got.CJKLeak != 0 {
|
||||
t.Errorf("double-space gloss FALSE-flagged (finding B): %+v", got)
|
||||
}
|
||||
if got := stripCosmetic("Кулак (羅漢拳) был силён."); !strings.Contains(got, "羅漢拳") {
|
||||
t.Errorf("double-space gloss term dropped (finding B): %q", got)
|
||||
}
|
||||
// D39.4 finding 1: a U+3000 (ideographic-space) gap binds the term to the gloss — the term is kept
|
||||
// and the chunk not flagged (the U+3000 in the head normalises to a plain space, so not byte-verbatim).
|
||||
if got := sanitizeOutput("Кулак (羅漢拳) был силён."); got.CJKLeak != 0 {
|
||||
t.Errorf("U+3000-bound gloss FALSE-flagged (finding 1): %+v", got)
|
||||
}
|
||||
if got := stripCosmetic("Кулак (羅漢拳) был силён."); !strings.Contains(got, "羅漢拳") {
|
||||
t.Errorf("U+3000-bound gloss term dropped (finding 1): %q", got)
|
||||
}
|
||||
// NOT a whitelisted gloss → the CJK stays a leak (bare run / no letter head / CJK punctuation /
|
||||
// nesting / a CLAUSE too long to be a term — finding D).
|
||||
leak := []string{
|
||||
"Судзуки нашёл 特产 древнего клана.", // bare Han, no parens at all
|
||||
"(羅漢拳) — так назвали приём.", // paren gloss at LINE START, no letter head → leak
|
||||
"Он сказал: (太極) и ушёл.", // after «:» + space, no letter head → leak
|
||||
"Приём 5 (羅漢拳) в списке.", // digit head, not a letter → not a gloss
|
||||
"Клан (羅、漢) распался.", // CJK PUNCTUATION «、» inside the parens → not a term gloss
|
||||
"Он ушёл (却有很多事情要做), но вернулся.", // 8-CJK CLAUSE, not a term gloss (finding D) → leak
|
||||
"Техника (九字真言破邪法) была забыта тогда.", // 7 CJK > maxGlossCJKRunes → leak (finding D bound)
|
||||
"Кулак (拳 the iron fist that breaks the sky) взлетел.", // F2: English clause after a token Han → leak
|
||||
"Приём (拳 antidisestablishmentarianism) стар.", // F2: single over-long non-pinyin Latin word → leak
|
||||
"Стиль (拳 iron fist of sky) забыт.", // F2: non-proportional Latin (3 toks vs 1 CJK) → leak
|
||||
}
|
||||
for _, s := range leak {
|
||||
if got := sanitizeOutput(s); got.CJKLeak == 0 {
|
||||
t.Errorf("expected a CJK leak (not a valid gloss) in %q: %+v", preview(s), got)
|
||||
}
|
||||
}
|
||||
// The emptied-bracket regression must NOT reappear: a preserved gloss keeps its parens and term, so
|
||||
// stripCosmetic never yields «()» — and a real bare leak next to a gloss strips cleanly.
|
||||
if got := stripCosmetic("Кулак (羅漢拳) и 特产 рядом."); got != "Кулак (羅漢拳) и рядом." {
|
||||
t.Errorf("mixed gloss+leak strip = %q, want the gloss kept and the bare leak dropped", got)
|
||||
}
|
||||
// D39.4 finding 1: a U+3000-BOUND gloss must survive the strip of a CO-OCCURRING bare leak — the
|
||||
// fold-first detector and the original-bytes strip must agree the U+3000 gloss is protected.
|
||||
if got := stripCosmetic("Кулак (羅漢拳) и 特产 рядом."); !strings.Contains(got, "羅漢拳") {
|
||||
t.Errorf("finding 1: U+3000-bound gloss destroyed alongside a bare leak: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSanitizerFoldFirstBypass pins F1 (D39.4): a FULLWIDTH variation of a signature must be detected,
|
||||
// not slip past the ASCII regexes while exportNormalize manufactures the ASCII defect in the export.
|
||||
// All six classes detect over the export-normalised (folded) text.
|
||||
func TestSanitizerFoldFirstBypass(t *testing.T) {
|
||||
fire := map[string]string{
|
||||
"fullwidth-colon-preamble": "Вот отредактированный перевод:\n\nСудзуки шёл.", // U+FF1A colon
|
||||
"fullwidth-markdown": "### Глава 5\n\nСудзуки открыл дверь.", // U+FF03 hash ×3
|
||||
"fullwidth-trailing-note": "Судзуки ушёл.\n\nПримечание:имя оставлено как есть.", // U+FF1A trailing
|
||||
"fullwidth-latin-phrase": "Он прочитал: the quick brown fox today.", // fullwidth latin
|
||||
}
|
||||
for name, s := range fire {
|
||||
if got := sanitizeOutput(s); got.total() == 0 {
|
||||
t.Errorf("F1 %s: a fullwidth signature slipped past the detector (bypass): %q → %+v", name, preview(s), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSanitizerExportNormalizeInvariant pins the F1 gate/export-contract invariant: for any text the
|
||||
// sanitizer calls CLEAN, its export-normalised form is ALSO clean — the detector and exportNormalize
|
||||
// agree, so no fullwidth defect is manufactured in the export of a "clean" chunk. Runs over the whole
|
||||
// legitimate-prose corpus the class tests use.
|
||||
func TestSanitizerExportNormalizeInvariant(t *testing.T) {
|
||||
corpus := []string{
|
||||
"Судзуки шёл по коридорам Академии магии, вспоминая вчерашнюю долгую лекцию.",
|
||||
"— Сегодня я открою дверь книгохранилища, — прошептал он, сжимая ключ.",
|
||||
"Цена была 123 золотых, и он расплатился не торгуясь.",
|
||||
" Судзуки медленно шёл по длинному коридору к башне.",
|
||||
"Цена была 123 золотых.",
|
||||
"Он вошёл в дом、и дверь закрылась.",
|
||||
"Он источа́ло тепло за́мка.",
|
||||
"Он ждал… но никто не пришёл. № 5, ½ часа.",
|
||||
"Кулак Ло Хань (羅漢拳) был силён.",
|
||||
"Стиль Тайцзи (太極拳 tàijíquán) он знал.",
|
||||
"Он купил iPhone, iPad, MacBook, Apple Watch и Google Pixel вчера.",
|
||||
"На гербе была выбита надпись: sic transit gloria mundi.",
|
||||
}
|
||||
for _, s := range corpus {
|
||||
if sanitizeOutput(s).total() != 0 {
|
||||
continue // not a clean input — the invariant is stated for clean verdicts
|
||||
}
|
||||
if got := sanitizeOutput(exportNormalize(s)); got.total() != 0 {
|
||||
t.Errorf("F1 invariant: clean %q became a defect after exportNormalize: %+v", preview(s), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSanitizerDeterministic asserts the verdict is a pure function of the text (resume-safe).
|
||||
func TestSanitizerDeterministic(t *testing.T) {
|
||||
s := "### Заголовок\nВот отредактированный перевод: текст. Стольь странно.\nПримечание: правка."
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
==== run 1 (fresh) ====
|
||||
snapshot_id: 79309d253aeaa641272a3fb53eca471567cecf1d2e8339a7fba2e2f1d4c37708
|
||||
snapshot_payload: {"brief_hash":"c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df","chunker_version":"chunker-v4-utf8-quotedepth","estimator_version":"estimator-v0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v1-refusal+echo015+loop","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"stm_depth":0,"overlap_tokens":0,"cache_ttl":""},"memory_version":"636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v2","sanitizer":{"enabled":true,"version":"sanitizer-v5"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"26d0245a56d180eafd200ebe5311aab2488142c4a98dcddee21ea457814aa21a","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}},{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]}
|
||||
snapshot_id: 325d971e2b6ec79a129891f969d16a3ddea1410f345f52d9d47e3f50c70726b2
|
||||
snapshot_payload: {"brief_hash":"c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df","chunker_version":"chunker-v4-utf8-quotedepth","estimator_version":"estimator-v0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v1-refusal+echo015+loop","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"stm_depth":0,"overlap_tokens":0,"cache_ttl":""},"memory_version":"636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v2","sanitizer":{"enabled":true,"version":"sanitizer-v6"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"26d0245a56d180eafd200ebe5311aab2488142c4a98dcddee21ea457814aa21a","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}},{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]}
|
||||
brief_hash: c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df
|
||||
memory_version: 636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53
|
||||
-- book result --
|
||||
|
|
@ -51,44 +51,44 @@ chunk ch8/0 disposition=flagged flag="sanitizer_stripped" final_text="Судзу
|
|||
stage=edit role=editor model=fake-model resume=false disp=flagged flag="sanitizer_stripped" attempts=1 escalated=false esc_model="" finish="stop" cum_usd=0.00182 detail="CJK-утечка в ru-выходе: 特产"
|
||||
stage_text="" recovered="Судзуки нашёл древний камень на каменном алтаре."
|
||||
-- chunk_status --
|
||||
ch1/0 draft snap_match=true content_hash=331c6f32c7f3ed15447e2639a2621d24a180e561a2aee92115324f492db5de3e disp=ok flag="" attempts=1 final_hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/0 edit snap_match=true content_hash=a559165b4d1fc3f9e75472ca654f4d039ae1e0cad1541c043e7aa68fbea5415a disp=ok flag="" attempts=1 final_hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/1 draft snap_match=true content_hash=12fe3ccdf062f43e011363a33ff85c9b6842a8f23d790991fe9afab51a8becc8 disp=ok flag="" attempts=1 final_hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/1 edit snap_match=true content_hash=b11a1740e5af98af20e207672555da8d411311e6c1e26595b456b656e8de59ef disp=ok flag="" attempts=1 final_hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch2/0 draft snap_match=true content_hash=2693325bc747687acf090bc263fe8c17783726b949c6d8a2d072da3401be6827 disp=ok flag="" attempts=1 final_hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a cost=0.00728 escalated=true esc_model="fake-fallback" detail=""
|
||||
ch2/0 edit snap_match=true content_hash=b467337f05775296985a6d79454b518302812edb57d398001b8852d0663c5bbb disp=ok flag="" attempts=1 final_hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/0 draft snap_match=true content_hash=331c6f32c7f3ed15447e2639a2621d24a180e561a2aee92115324f492db5de3e disp=ok flag="" attempts=1 final_hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/0 edit snap_match=true content_hash=a559165b4d1fc3f9e75472ca654f4d039ae1e0cad1541c043e7aa68fbea5415a disp=ok flag="" attempts=1 final_hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/1 draft snap_match=true content_hash=12fe3ccdf062f43e011363a33ff85c9b6842a8f23d790991fe9afab51a8becc8 disp=ok flag="" attempts=1 final_hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/1 edit snap_match=true content_hash=b11a1740e5af98af20e207672555da8d411311e6c1e26595b456b656e8de59ef disp=ok flag="" attempts=1 final_hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch2/0 draft snap_match=true content_hash=2693325bc747687acf090bc263fe8c17783726b949c6d8a2d072da3401be6827 disp=ok flag="" attempts=1 final_hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 cost=0.00728 escalated=true esc_model="fake-fallback" detail=""
|
||||
ch2/0 edit snap_match=true content_hash=b467337f05775296985a6d79454b518302812edb57d398001b8852d0663c5bbb disp=ok flag="" attempts=1 final_hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch3/0 draft snap_match=true content_hash=733fc6a89215efe112cfadd1d1fb361bb8a7ab1013994248a49da0dbd137548e disp=flagged flag="hard_refusal" attempts=1 final_hash= cost=0.00728 escalated=true esc_model="fake-fallback" detail="provider finish_reason=refusal"
|
||||
ch3/0 edit snap_match=true content_hash= disp=skipped flag="hard_refusal" attempts=0 final_hash= cost=0 escalated=false esc_model="" detail="skipped: an upstream stage was flagged (hard_refusal)"
|
||||
ch4/0 draft snap_match=true content_hash=7aabdcb89dea365c95363b2662144ca1a10b307d377f47a8e7cc19b5b872d287 disp=ok flag="" attempts=1 final_hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch4/0 edit snap_match=true content_hash=2854dfe1648b0cf0cbb683cb7ce94c5eedc9223b2ce20c99dc008c7add6317ef disp=ok flag="" attempts=1 final_hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch5/0 draft snap_match=true content_hash=d8aba42cabb41853a81875ebc725bb516dc2412e29de3a03bc1a68e90f8cf157 disp=ok flag="" attempts=1 final_hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch5/0 edit snap_match=true content_hash=99ebb10d754a2f3ae3a00d62734ad1c39160f083350217eda5bda6e91373f4de disp=ok flag="" attempts=1 final_hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch6/0 draft snap_match=true content_hash=9375b05780724cc354eba15a595da9a5f964219c0887042484256eed11fb566f disp=ok flag="" attempts=1 final_hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch4/0 draft snap_match=true content_hash=7aabdcb89dea365c95363b2662144ca1a10b307d377f47a8e7cc19b5b872d287 disp=ok flag="" attempts=1 final_hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch4/0 edit snap_match=true content_hash=2854dfe1648b0cf0cbb683cb7ce94c5eedc9223b2ce20c99dc008c7add6317ef disp=ok flag="" attempts=1 final_hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch5/0 draft snap_match=true content_hash=d8aba42cabb41853a81875ebc725bb516dc2412e29de3a03bc1a68e90f8cf157 disp=ok flag="" attempts=1 final_hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch5/0 edit snap_match=true content_hash=99ebb10d754a2f3ae3a00d62734ad1c39160f083350217eda5bda6e91373f4de disp=ok flag="" attempts=1 final_hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch6/0 draft snap_match=true content_hash=9375b05780724cc354eba15a595da9a5f964219c0887042484256eed11fb566f disp=ok flag="" attempts=1 final_hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch6/0 edit snap_match=true content_hash=f9a70ce8a43db4a89eda5e53a6f29ba5f0c2a35559a9d35b73f771220bb262ee disp=flagged flag="sanitizer_defect" attempts=1 final_hash= cost=0.00182 escalated=false esc_model="" detail="ведущая служебная преамбула: Вот перевод фрагмента:"
|
||||
ch7/0 draft snap_match=true content_hash=d90c335cba3d1547a22bb3f656443cb432f84c12a577d43509cfe472cf86a2ac disp=ok flag="" attempts=1 final_hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch7/0 edit snap_match=true content_hash=f003d8ad25546238e63b98d28a08f942c8e3cf57251df694de81bea293a6dd26 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:3f4f20574caef4ee6153e60f6acc5f41e9ed5afe63b2cafd69396bafb18f9051 cost=0.00182 escalated=false esc_model="" detail="markdown-заголовок в выходе: ### Глава 7"
|
||||
ch8/0 draft snap_match=true content_hash=fbd27af11da2275cd1d47a0224970a5f8d073c6164f614efb6522ea96fa1e021 disp=ok flag="" attempts=1 final_hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch8/0 edit snap_match=true content_hash=40c2dd5b7b713800e8453930779bf2c88e2a296056f594bc1a4f5c495956915a disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:31dcd0e8a631cbce9ef7873f139bc415902858a708ea16602c8fc38f77cf3b66 cost=0.00182 escalated=false esc_model="" detail="CJK-утечка в ru-выходе: 特产"
|
||||
ch7/0 draft snap_match=true content_hash=d90c335cba3d1547a22bb3f656443cb432f84c12a577d43509cfe472cf86a2ac disp=ok flag="" attempts=1 final_hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch7/0 edit snap_match=true content_hash=f003d8ad25546238e63b98d28a08f942c8e3cf57251df694de81bea293a6dd26 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:e1f2765b9f6d3cc4c461471cd7c143476614c602b6b08843bd239c77beb3b102 cost=0.00182 escalated=false esc_model="" detail="markdown-заголовок в выходе: ### Глава 7"
|
||||
ch8/0 draft snap_match=true content_hash=fbd27af11da2275cd1d47a0224970a5f8d073c6164f614efb6522ea96fa1e021 disp=ok flag="" attempts=1 final_hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch8/0 edit snap_match=true content_hash=40c2dd5b7b713800e8453930779bf2c88e2a296056f594bc1a4f5c495956915a disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:067cfd0a649bfcd3db7fd04e8beff2a610dba06e292664e0160f06d16eef5209 cost=0.00182 escalated=false esc_model="" detail="CJK-утечка в ru-выходе: 特产"
|
||||
-- request_log (insertion order) --
|
||||
ch1/0 draft role=translator req=fake-model actual=fake-model hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/0 edit role=editor req=fake-model actual=fake-model hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/1 draft role=translator req=fake-model actual=fake-model hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/1 edit role=editor req=fake-model actual=fake-model hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 draft role=translator req=fake-model actual=fake-model hash=afeec2c4175c036a72332a9d6e27ab48a5584bb3819ca5a3c5b95ad01160ed74 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 edit role=editor req=fake-model actual=fake-model hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch3/0 draft role=translator req=fake-model actual=fake-model hash=cb19e60076fb427f7294b8a0b80e2d810af3630d16353892f83a49b947affdd0 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=bc3af2566b80f34217bcbf5898895ab00887355f3f1e8a427d5d504683d41902 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err=""
|
||||
ch4/0 draft role=translator req=fake-model actual=fake-model hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch4/0 edit role=editor req=fake-model actual=fake-model hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch5/0 draft role=translator req=fake-model actual=fake-model hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch5/0 edit role=editor req=fake-model actual=fake-model hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch6/0 draft role=translator req=fake-model actual=fake-model hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch6/0 edit role=editor req=fake-model actual=fake-model hash=b76db9cd65fd0cde6bcc4d2d8cc77ae9628e23ebeff2653cb59740b458f1960c tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch7/0 draft role=translator req=fake-model actual=fake-model hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch7/0 edit role=editor req=fake-model actual=fake-model hash=327f47fe5e882624cb3c01c1e0dcfde566e8b695cbe482de8e4916eb90d6f45b tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch8/0 draft role=translator req=fake-model actual=fake-model hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch8/0 edit role=editor req=fake-model actual=fake-model hash=5b836ac514f218026f8f61f4d8ef6e1a68b7625ec2dde3cb95e196b6ac55864d tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/0 draft role=translator req=fake-model actual=fake-model hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/0 edit role=editor req=fake-model actual=fake-model hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/1 draft role=translator req=fake-model actual=fake-model hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/1 edit role=editor req=fake-model actual=fake-model hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 draft role=translator req=fake-model actual=fake-model hash=5f4763e07c67f541958817ce6e7a65c8277e1d806558d3e218e92362ba858866 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 edit role=editor req=fake-model actual=fake-model hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch3/0 draft role=translator req=fake-model actual=fake-model hash=be200a513ccbfbb410ffc7788967610e2323aeda610c4d25296c8f06ddf3aead tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=4e5ad1c52475c43b5f98f0a369187a11367e4c739b67d5d9269b9949b718de33 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err=""
|
||||
ch4/0 draft role=translator req=fake-model actual=fake-model hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch4/0 edit role=editor req=fake-model actual=fake-model hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch5/0 draft role=translator req=fake-model actual=fake-model hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch5/0 edit role=editor req=fake-model actual=fake-model hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch6/0 draft role=translator req=fake-model actual=fake-model hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch6/0 edit role=editor req=fake-model actual=fake-model hash=ae7cb45838ddec818f5e6c8f71410db7f00365dc8d1a41ec1a94af1d490e2178 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch7/0 draft role=translator req=fake-model actual=fake-model hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch7/0 edit role=editor req=fake-model actual=fake-model hash=9394388fc09349b7e4014e12ae991f041c5912307964d83ae165a8c99a3aac2e tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch8/0 draft role=translator req=fake-model actual=fake-model hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch8/0 edit role=editor req=fake-model actual=fake-model hash=fd0a61bd15ec394e35f291a3f114469a0fca591b3bc529ef8bd79686fe3bef93 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
-- retrieval_state --
|
||||
ch1/0 snap_match=true exact=3 sticky=0 ambiguous=0 spoiler=1 evicted=0 postcheck_miss=1 style_flags=0 trust_gated=0
|
||||
injected_ids=["紋章\u001f\u001f0\u001f0","鈴木\u001f\u001f0\u001f0","魔法学院\u001f\u001f0\u001f0"] postcheck_detail=[{"src":"紋章","dst":"герб","disp":"confirmed"}] style_detail= trust_gate_detail=
|
||||
|
|
@ -129,8 +129,8 @@ ch8/0 snap_match=true exact=1 sticky=0 ambiguous=0 spoiler=0 evicted=0 postcheck
|
|||
[17] {"max_tokens":4000,"messages":[{"role":"system","content":"Переводи художественный текст с ja на ru. Жанр: ранобэ. Аудитория: взрослые. Баланс Венути: 0.50. Хонорифики: keep. Транскрипция: polivanov."},{"role":"system","content":"ГЛОССАРИЙ (используй эти утверждённые переводы имён и терминов последовательно; строки с пометкой ⟨проверить⟩ — неподтверждённые кандидаты):\n鈴木 → Судзуки"},{"role":"user","content":"第八章 漢字漏洩\n\n鈴木は石板の文字をなぞり、古い漢字の意味を思い出そうとした。だが記憶は霧のように薄れ、指先だけが淡く光っていた。"}],"model":"fake-model","stream":false,"temperature":0.3}
|
||||
[18] {"max_tokens":4000,"messages":[{"role":"system","content":"Ты — монолингвальный редактор русского текста. Правь стиль, не меняя смысла. Книга: Золотая книга."},{"role":"system","content":"КАНОНИЧЕСКИЕ ПЕРЕВОДЫ имён и терминов (в черновике эти сущности должны быть переданы ИМЕННО этими формами — приводи к ним любые расхождения, склоняя по контексту; не вводи иных вариантов и не меняй ничего другого):\n- «Судзуки»"},{"role":"user","content":"Черновик перевода для редактуры: ЧЕРНОВИК ПЕРЕВОДА 0bc6a4817acb. ИЕРОГЛИФ-СЦЕНА Судзуки нашёл камень."}],"model":"fake-model","stream":false,"temperature":0.4}
|
||||
==== run 2 (resume) ====
|
||||
snapshot_id: 79309d253aeaa641272a3fb53eca471567cecf1d2e8339a7fba2e2f1d4c37708
|
||||
snapshot_payload: {"brief_hash":"c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df","chunker_version":"chunker-v4-utf8-quotedepth","estimator_version":"estimator-v0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v1-refusal+echo015+loop","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"stm_depth":0,"overlap_tokens":0,"cache_ttl":""},"memory_version":"636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v2","sanitizer":{"enabled":true,"version":"sanitizer-v5"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"26d0245a56d180eafd200ebe5311aab2488142c4a98dcddee21ea457814aa21a","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}},{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]}
|
||||
snapshot_id: 325d971e2b6ec79a129891f969d16a3ddea1410f345f52d9d47e3f50c70726b2
|
||||
snapshot_payload: {"brief_hash":"c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df","chunker_version":"chunker-v4-utf8-quotedepth","estimator_version":"estimator-v0","max_tokens_policy":"maxtok-v1-double-per-attempt","classifier_version":"classify-v1-refusal+echo015+loop","pipeline_core":"C1","max_output_ratio":2,"min_max_tokens":512,"context_assembly":{"glossary_injection":"selective","glossary_token_budget":800,"stm_depth":0,"overlap_tokens":0,"cache_ttl":""},"memory_version":"636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53","postcheck_gate":false,"coverage":{"enabled":false},"style_check_version":"cheapgate-v2","sanitizer":{"enabled":true,"version":"sanitizer-v6"},"stages":[{"name":"draft","role":"translator","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"26d0245a56d180eafd200ebe5311aab2488142c4a98dcddee21ea457814aa21a","temperature":0.3,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000},"escalate_to":"fake-fallback","escalate_capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":6000}},{"name":"edit","role":"editor","model":"fake-model","prompt_version":"v-golden","prompt_sha256":"6a8e7f71139d6bc09a95159d1cb895e05b56d2f2f7417c5e20a7073c517c79f6","temperature":0.4,"reasoning":"off","capability":{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}}]}
|
||||
brief_hash: c2021b1b5e6c20f6084e425b9a70bdc5f682801be347a57eb1c591deb8cab0df
|
||||
memory_version: 636d2a6993285e433ce25bad2f5da01c1bcc19a4682dc9fe1efb15e5feb92f53
|
||||
-- book result --
|
||||
|
|
@ -181,61 +181,61 @@ chunk ch8/0 disposition=flagged flag="sanitizer_stripped" final_text="Судзу
|
|||
stage=edit role=editor model=fake-model resume=true disp=flagged flag="sanitizer_stripped" attempts=1 escalated=false esc_model="" finish="sanitized_export" cum_usd=0.00182 detail="CJK-утечка в ru-выходе: 特产"
|
||||
stage_text="" recovered="Судзуки нашёл древний камень на каменном алтаре."
|
||||
-- chunk_status --
|
||||
ch1/0 draft snap_match=true content_hash=331c6f32c7f3ed15447e2639a2621d24a180e561a2aee92115324f492db5de3e disp=ok flag="" attempts=1 final_hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/0 edit snap_match=true content_hash=a559165b4d1fc3f9e75472ca654f4d039ae1e0cad1541c043e7aa68fbea5415a disp=ok flag="" attempts=1 final_hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/1 draft snap_match=true content_hash=12fe3ccdf062f43e011363a33ff85c9b6842a8f23d790991fe9afab51a8becc8 disp=ok flag="" attempts=1 final_hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/1 edit snap_match=true content_hash=b11a1740e5af98af20e207672555da8d411311e6c1e26595b456b656e8de59ef disp=ok flag="" attempts=1 final_hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch2/0 draft snap_match=true content_hash=2693325bc747687acf090bc263fe8c17783726b949c6d8a2d072da3401be6827 disp=ok flag="" attempts=1 final_hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a cost=0.00728 escalated=true esc_model="fake-fallback" detail=""
|
||||
ch2/0 edit snap_match=true content_hash=b467337f05775296985a6d79454b518302812edb57d398001b8852d0663c5bbb disp=ok flag="" attempts=1 final_hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/0 draft snap_match=true content_hash=331c6f32c7f3ed15447e2639a2621d24a180e561a2aee92115324f492db5de3e disp=ok flag="" attempts=1 final_hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/0 edit snap_match=true content_hash=a559165b4d1fc3f9e75472ca654f4d039ae1e0cad1541c043e7aa68fbea5415a disp=ok flag="" attempts=1 final_hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/1 draft snap_match=true content_hash=12fe3ccdf062f43e011363a33ff85c9b6842a8f23d790991fe9afab51a8becc8 disp=ok flag="" attempts=1 final_hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch1/1 edit snap_match=true content_hash=b11a1740e5af98af20e207672555da8d411311e6c1e26595b456b656e8de59ef disp=ok flag="" attempts=1 final_hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch2/0 draft snap_match=true content_hash=2693325bc747687acf090bc263fe8c17783726b949c6d8a2d072da3401be6827 disp=ok flag="" attempts=1 final_hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 cost=0.00728 escalated=true esc_model="fake-fallback" detail=""
|
||||
ch2/0 edit snap_match=true content_hash=b467337f05775296985a6d79454b518302812edb57d398001b8852d0663c5bbb disp=ok flag="" attempts=1 final_hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch3/0 draft snap_match=true content_hash=733fc6a89215efe112cfadd1d1fb361bb8a7ab1013994248a49da0dbd137548e disp=flagged flag="hard_refusal" attempts=1 final_hash= cost=0.00728 escalated=true esc_model="fake-fallback" detail="provider finish_reason=refusal"
|
||||
ch3/0 edit snap_match=true content_hash= disp=skipped flag="hard_refusal" attempts=0 final_hash= cost=0 escalated=false esc_model="" detail="skipped: an upstream stage was flagged (hard_refusal)"
|
||||
ch4/0 draft snap_match=true content_hash=7aabdcb89dea365c95363b2662144ca1a10b307d377f47a8e7cc19b5b872d287 disp=ok flag="" attempts=1 final_hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch4/0 edit snap_match=true content_hash=2854dfe1648b0cf0cbb683cb7ce94c5eedc9223b2ce20c99dc008c7add6317ef disp=ok flag="" attempts=1 final_hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch5/0 draft snap_match=true content_hash=d8aba42cabb41853a81875ebc725bb516dc2412e29de3a03bc1a68e90f8cf157 disp=ok flag="" attempts=1 final_hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch5/0 edit snap_match=true content_hash=99ebb10d754a2f3ae3a00d62734ad1c39160f083350217eda5bda6e91373f4de disp=ok flag="" attempts=1 final_hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch6/0 draft snap_match=true content_hash=9375b05780724cc354eba15a595da9a5f964219c0887042484256eed11fb566f disp=ok flag="" attempts=1 final_hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch4/0 draft snap_match=true content_hash=7aabdcb89dea365c95363b2662144ca1a10b307d377f47a8e7cc19b5b872d287 disp=ok flag="" attempts=1 final_hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch4/0 edit snap_match=true content_hash=2854dfe1648b0cf0cbb683cb7ce94c5eedc9223b2ce20c99dc008c7add6317ef disp=ok flag="" attempts=1 final_hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch5/0 draft snap_match=true content_hash=d8aba42cabb41853a81875ebc725bb516dc2412e29de3a03bc1a68e90f8cf157 disp=ok flag="" attempts=1 final_hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch5/0 edit snap_match=true content_hash=99ebb10d754a2f3ae3a00d62734ad1c39160f083350217eda5bda6e91373f4de disp=ok flag="" attempts=1 final_hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch6/0 draft snap_match=true content_hash=9375b05780724cc354eba15a595da9a5f964219c0887042484256eed11fb566f disp=ok flag="" attempts=1 final_hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch6/0 edit snap_match=true content_hash=f9a70ce8a43db4a89eda5e53a6f29ba5f0c2a35559a9d35b73f771220bb262ee disp=flagged flag="sanitizer_defect" attempts=1 final_hash= cost=0.00182 escalated=false esc_model="" detail="ведущая служебная преамбула: Вот перевод фрагмента:"
|
||||
ch7/0 draft snap_match=true content_hash=d90c335cba3d1547a22bb3f656443cb432f84c12a577d43509cfe472cf86a2ac disp=ok flag="" attempts=1 final_hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch7/0 edit snap_match=true content_hash=f003d8ad25546238e63b98d28a08f942c8e3cf57251df694de81bea293a6dd26 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:3f4f20574caef4ee6153e60f6acc5f41e9ed5afe63b2cafd69396bafb18f9051 cost=0.00182 escalated=false esc_model="" detail="markdown-заголовок в выходе: ### Глава 7"
|
||||
ch8/0 draft snap_match=true content_hash=fbd27af11da2275cd1d47a0224970a5f8d073c6164f614efb6522ea96fa1e021 disp=ok flag="" attempts=1 final_hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch8/0 edit snap_match=true content_hash=40c2dd5b7b713800e8453930779bf2c88e2a296056f594bc1a4f5c495956915a disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:31dcd0e8a631cbce9ef7873f139bc415902858a708ea16602c8fc38f77cf3b66 cost=0.00182 escalated=false esc_model="" detail="CJK-утечка в ru-выходе: 特产"
|
||||
ch7/0 draft snap_match=true content_hash=d90c335cba3d1547a22bb3f656443cb432f84c12a577d43509cfe472cf86a2ac disp=ok flag="" attempts=1 final_hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch7/0 edit snap_match=true content_hash=f003d8ad25546238e63b98d28a08f942c8e3cf57251df694de81bea293a6dd26 disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:e1f2765b9f6d3cc4c461471cd7c143476614c602b6b08843bd239c77beb3b102 cost=0.00182 escalated=false esc_model="" detail="markdown-заголовок в выходе: ### Глава 7"
|
||||
ch8/0 draft snap_match=true content_hash=fbd27af11da2275cd1d47a0224970a5f8d073c6164f614efb6522ea96fa1e021 disp=ok flag="" attempts=1 final_hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 cost=0.00182 escalated=false esc_model="" detail=""
|
||||
ch8/0 edit snap_match=true content_hash=40c2dd5b7b713800e8453930779bf2c88e2a296056f594bc1a4f5c495956915a disp=flagged flag="sanitizer_stripped" attempts=1 final_hash=tm-sanitized-v1:067cfd0a649bfcd3db7fd04e8beff2a610dba06e292664e0160f06d16eef5209 cost=0.00182 escalated=false esc_model="" detail="CJK-утечка в ru-выходе: 特产"
|
||||
-- request_log (insertion order) --
|
||||
ch1/0 draft role=translator req=fake-model actual=fake-model hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/0 edit role=editor req=fake-model actual=fake-model hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/1 draft role=translator req=fake-model actual=fake-model hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/1 edit role=editor req=fake-model actual=fake-model hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 draft role=translator req=fake-model actual=fake-model hash=afeec2c4175c036a72332a9d6e27ab48a5584bb3819ca5a3c5b95ad01160ed74 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 edit role=editor req=fake-model actual=fake-model hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch3/0 draft role=translator req=fake-model actual=fake-model hash=cb19e60076fb427f7294b8a0b80e2d810af3630d16353892f83a49b947affdd0 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=bc3af2566b80f34217bcbf5898895ab00887355f3f1e8a427d5d504683d41902 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err=""
|
||||
ch4/0 draft role=translator req=fake-model actual=fake-model hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch4/0 edit role=editor req=fake-model actual=fake-model hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch5/0 draft role=translator req=fake-model actual=fake-model hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch5/0 edit role=editor req=fake-model actual=fake-model hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch6/0 draft role=translator req=fake-model actual=fake-model hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch6/0 edit role=editor req=fake-model actual=fake-model hash=b76db9cd65fd0cde6bcc4d2d8cc77ae9628e23ebeff2653cb59740b458f1960c tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch7/0 draft role=translator req=fake-model actual=fake-model hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch7/0 edit role=editor req=fake-model actual=fake-model hash=327f47fe5e882624cb3c01c1e0dcfde566e8b695cbe482de8e4916eb90d6f45b tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch8/0 draft role=translator req=fake-model actual=fake-model hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch8/0 edit role=editor req=fake-model actual=fake-model hash=5b836ac514f218026f8f61f4d8ef6e1a68b7625ec2dde3cb95e196b6ac55864d tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/0 draft role=translator req=fake-model actual=fake-model hash=eae32ce66442ad5b2626b2c948a8d56ffbde5fcb16edfd357114a4a9b7616c17 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch1/0 edit role=editor req=fake-model actual=fake-model hash=6cc9e0252932889579db113e46fe2c85d50a77d73e259a4ef82f313a525f549f tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch1/1 draft role=translator req=fake-model actual=fake-model hash=c61e712612914041e429a968fca79948cc54b31121ad8c4040c26d1f4d5a6a20 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch1/1 edit role=editor req=fake-model actual=fake-model hash=23f92dbd00f65ec09d798b2510168957dac4bc0df8f0efbca3b9590cbe6e9678 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch2/0 draft role=translator req=fake-model actual=fake-fallback hash=e6361a84b9b065664351b8106d678ee4cf649c5308c7ea5094c94feff501064a tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch2/0 edit role=editor req=fake-model actual=fake-model hash=5d6c443102025c034222e3c37753a38204fab80d0108b4a833b301ed65b6a0d4 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch1/0 draft role=translator req=fake-model actual=fake-model hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/0 edit role=editor req=fake-model actual=fake-model hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/1 draft role=translator req=fake-model actual=fake-model hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/1 edit role=editor req=fake-model actual=fake-model hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 draft role=translator req=fake-model actual=fake-model hash=5f4763e07c67f541958817ce6e7a65c8277e1d806558d3e218e92362ba858866 tm_hit=0 ok=0 finish="stop" degraded="cjk_artifact" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 draft role=translator req=fake-fallback actual=fake-fallback hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err=""
|
||||
ch2/0 edit role=editor req=fake-model actual=fake-model hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch3/0 draft role=translator req=fake-model actual=fake-model hash=be200a513ccbfbb410ffc7788967610e2323aeda610c4d25296c8f06ddf3aead tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch3/0 draft role=translator req=fake-fallback actual=fake-fallback hash=4e5ad1c52475c43b5f98f0a369187a11367e4c739b67d5d9269b9949b718de33 tm_hit=0 ok=0 finish="refusal" degraded="hard_refusal" cost=0.0054600000000000004 tokens=1000/200/0/500/0 err=""
|
||||
ch4/0 draft role=translator req=fake-model actual=fake-model hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch4/0 edit role=editor req=fake-model actual=fake-model hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch5/0 draft role=translator req=fake-model actual=fake-model hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch5/0 edit role=editor req=fake-model actual=fake-model hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch6/0 draft role=translator req=fake-model actual=fake-model hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch6/0 edit role=editor req=fake-model actual=fake-model hash=ae7cb45838ddec818f5e6c8f71410db7f00365dc8d1a41ec1a94af1d490e2178 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_defect" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch7/0 draft role=translator req=fake-model actual=fake-model hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch7/0 edit role=editor req=fake-model actual=fake-model hash=9394388fc09349b7e4014e12ae991f041c5912307964d83ae165a8c99a3aac2e tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch8/0 draft role=translator req=fake-model actual=fake-model hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 tm_hit=0 ok=1 finish="stop" degraded="" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch8/0 edit role=editor req=fake-model actual=fake-model hash=fd0a61bd15ec394e35f291a3f114469a0fca591b3bc529ef8bd79686fe3bef93 tm_hit=0 ok=0 finish="stop" degraded="sanitizer_stripped" cost=0.00182 tokens=1000/200/0/500/0 err=""
|
||||
ch1/0 draft role=translator req=fake-model actual=fake-model hash=f2a775cd8f27741efb197298fcdf33cc5efa575dd91a490aa5519a15151f84d7 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch1/0 edit role=editor req=fake-model actual=fake-model hash=7d051e500c543f54a80dc0d36c8b03aac32225a51e9a015cc94ac03491629fb7 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch1/1 draft role=translator req=fake-model actual=fake-model hash=b27adec57257108c0c14c7a8e1f4b05c1dc66bf616a8a72d85dc7140bdcd8d81 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch1/1 edit role=editor req=fake-model actual=fake-model hash=c625e57b0778843942c78c5a8de5168e20857c9851e872b9a6e924edcbdc7059 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch2/0 draft role=translator req=fake-model actual=fake-fallback hash=cf1b1146260ceeccc2ee7c87c9e828c8332af1a93193f6fd4939c58ee6b9b0f2 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch2/0 edit role=editor req=fake-model actual=fake-model hash=13ab5112ef1bf90ca6371bb81c73eb86fb31ed55c54bd60954ea092b354891b7 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch3/0 draft role=translator req=fake-model actual=fake-model hash= tm_hit=1 ok=0 finish="" degraded="hard_refusal" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch4/0 draft role=translator req=fake-model actual=fake-model hash=ffca9be51da4c6343883b032161fff9e6e697fe7e8aefdd343748f189eb7dece tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch4/0 edit role=editor req=fake-model actual=fake-model hash=b41b05f4ea1bcac3c12f3ccac14f835580adbc54dc682ae04c3a1668e10b3711 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch5/0 draft role=translator req=fake-model actual=fake-model hash=610d92887eb72a678b63cba01dc6cb75f971dc51cc800ef73de9f74cc22422b6 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch5/0 edit role=editor req=fake-model actual=fake-model hash=0a45029f562a60cc9097a8e557b61d549189e9763f2969db34ff64f3638904cb tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch6/0 draft role=translator req=fake-model actual=fake-model hash=af78fdb90758c3b68d267eebb2b9af310124dddea2141e717ba0dd3079e6572f tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch4/0 draft role=translator req=fake-model actual=fake-model hash=3fedb549effcc911491a22a2db4a07e8385d329fe5b3f17c65c4bed8a72b8a3c tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch4/0 edit role=editor req=fake-model actual=fake-model hash=d74baa087896d28bbd5588f9d85a22261255551db74c4921fdb7983ba2fc9f70 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch5/0 draft role=translator req=fake-model actual=fake-model hash=839873c92b11274628cfb0d8f03384df6549e4bd5529fcdef6a5c5ea46beaed9 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch5/0 edit role=editor req=fake-model actual=fake-model hash=22214c6cc4a4b04fc48d6a8a0e1ff94a7a9745df3b16e6d4ea8dbde4e4b7b308 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch6/0 draft role=translator req=fake-model actual=fake-model hash=88cb389b6ed249076c333a88c3b76a33661d2976ade55bd5a81cd6014b1e4604 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch6/0 edit role=editor req=fake-model actual=fake-model hash= tm_hit=1 ok=0 finish="" degraded="sanitizer_defect" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch7/0 draft role=translator req=fake-model actual=fake-model hash=7678dd3f733a80403de5de0900318246b3e0f45244e168d9cfdfc373a8fa6c72 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch7/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:3f4f20574caef4ee6153e60f6acc5f41e9ed5afe63b2cafd69396bafb18f9051 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch8/0 draft role=translator req=fake-model actual=fake-model hash=a2015ce7a9d9c349b4433649a0d864513363ba5099fa0c413409449687eebadb tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch8/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:31dcd0e8a631cbce9ef7873f139bc415902858a708ea16602c8fc38f77cf3b66 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch7/0 draft role=translator req=fake-model actual=fake-model hash=23ccf5b0150e650e923d06c6e92f633665ea36ee75089e8947dff26748be2c7a tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch7/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:e1f2765b9f6d3cc4c461471cd7c143476614c602b6b08843bd239c77beb3b102 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch8/0 draft role=translator req=fake-model actual=fake-model hash=cc5dbfbcdcd3070e6471266fc9b6ce3d5abf88b8c75f3bb619522f9146cc6e31 tm_hit=1 ok=1 finish="stop" degraded="" cost=0 tokens=0/0/0/0/0 err=""
|
||||
ch8/0 edit role=editor req=fake-model actual=fake-model hash=tm-sanitized-v1:067cfd0a649bfcd3db7fd04e8beff2a610dba06e292664e0160f06d16eef5209 tm_hit=1 ok=0 finish="sanitized_export" degraded="sanitizer_stripped" cost=0 tokens=0/0/0/0/0 err=""
|
||||
-- retrieval_state --
|
||||
ch1/0 snap_match=true exact=3 sticky=0 ambiguous=0 spoiler=1 evicted=0 postcheck_miss=1 style_flags=0 trust_gated=0
|
||||
injected_ids=["紋章\u001f\u001f0\u001f0","鈴木\u001f\u001f0\u001f0","魔法学院\u001f\u001f0\u001f0"] postcheck_detail=[{"src":"紋章","dst":"герб","disp":"confirmed"}] style_detail= trust_gate_detail=
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue