Land the readability infra-pack (D38): cosmetic-class strip-and-export for flagged chunks, a CJK-leak sanitizer class, and an opt-in post-reflow number/omission regression guard
This commit is contained in:
Claude (backend session)
parent
b14ba6f20c
commit
d5beefc8f4
19 changed files with 929 additions and 139 deletions
|
|
@ -26,9 +26,16 @@ import (
|
|||
func renderTranslate(w io.Writer, res *pipeline.BookResult, ledger func() (committed, reserved float64, err error)) error {
|
||||
for _, ch := range res.Chunks {
|
||||
fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — %s%s ===\n", ch.Chapter, ch.ChunkIdx, ch.Disposition, flagSuffix(ch.FlagReason))
|
||||
if ch.Disposition == pipeline.DispOK {
|
||||
switch {
|
||||
case ch.Disposition == pipeline.DispOK:
|
||||
fmt.Fprintln(w, ch.FinalText)
|
||||
} else {
|
||||
case ch.FinalText != "":
|
||||
// Cosmetic sanitizer strip (D35.4a): the leak was removed and the remainder exported,
|
||||
// but the chunk stays flagged for a human to verify the auto-clean — not lost to an
|
||||
// empty placeholder (ch5/ch20 chapter openers used to drop whole for a leading «###»).
|
||||
fmt.Fprintf(w, "[ФЛАГ %s — утечка вычищена, экспортирован очищенным, проверь] ↓\n", ch.FlagReason)
|
||||
fmt.Fprintln(w, ch.FinalText)
|
||||
default:
|
||||
fmt.Fprintf(w, "[ФЛАГ %s] чанк не переведён — черновик/редактура непригодны, помечен для человека\n", ch.FlagReason)
|
||||
}
|
||||
for _, st := range ch.Stages {
|
||||
|
|
@ -159,7 +166,7 @@ func renderReport(w io.Writer,
|
|||
fmt.Fprintf(w, "%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NPostcheckMiss, rs.PostcheckDetail)
|
||||
}
|
||||
// Cheap style/number flaggers (observability, not gates): total + per-chunk detail.
|
||||
fmt.Fprintf(w, "\n=== СТИЛЬ-ГЕЙТЫ (наблюдаемость: тире-диалоги, ё, транслит-междометия, разряды 万/億) — всего %d ===\n", style)
|
||||
fmt.Fprintf(w, "\n=== СТИЛЬ-ГЕЙТЫ (наблюдаемость: тире-диалоги, ё, транслит-междометия, разряды 万/億, reflow-регрессия) — всего %d ===\n", style)
|
||||
stylePrinted := false
|
||||
for _, rs := range states {
|
||||
if rs.NStyleFlags == 0 {
|
||||
|
|
|
|||
|
|
@ -86,9 +86,22 @@ type Stage struct {
|
|||
|
||||
// Gates is the QA-gate config skeleton (пороги полигона; исполнение — Фаза 1).
|
||||
type Gates struct {
|
||||
Coverage CoverageGate `yaml:"coverage"`
|
||||
Glossary GlossaryGate `yaml:"glossary"`
|
||||
Sanitizer SanitizerGate `yaml:"sanitizer"`
|
||||
Coverage CoverageGate `yaml:"coverage"`
|
||||
Glossary GlossaryGate `yaml:"glossary"`
|
||||
Sanitizer SanitizerGate `yaml:"sanitizer"`
|
||||
RegressionGuard RegressionGuardGate `yaml:"regression_guard"`
|
||||
}
|
||||
|
||||
// RegressionGuardGate controls the post-reflow regression guard (D38 infra-pack,
|
||||
// research/18 §C#5): two deterministic OBSERVABILITY flaggers over the draft→final transform —
|
||||
// a length collapse and a numeric drift — that surface a reflow that dropped content or drifted a
|
||||
// number (四成四=44%→«четыре десятых»). Opt-in (default false), and by design NEVER a disposition
|
||||
// change: the reflow editor legitimately restructures/merges, so a hard skip would false-flag a
|
||||
// good edit — a hit is recorded in the cheap-gate observability channel (retrieval-state
|
||||
// n_style_flags) and surfaced in the report, never dropping the chunk. Its thresholds are code
|
||||
// consts (versioned with the cheap gates), so the gate carries only an on/off switch.
|
||||
type RegressionGuardGate struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// SanitizerGate controls the output-sanitizer (D30.3): a deterministic verdict-axis
|
||||
|
|
|
|||
|
|
@ -22,10 +22,16 @@ type StageResult struct {
|
|||
LatencyMS int
|
||||
FinishReason string
|
||||
Text string // the usable output (only on an ok disposition)
|
||||
Disposition Disposition
|
||||
FlagReason FlagReason // "" when ok
|
||||
Detail string
|
||||
Attempts int
|
||||
// RecoveredText is the cosmetic-stripped export text (D35.4a): non-empty ONLY for a
|
||||
// FlagSanitizerStripped final stage, where the sanitizer removed a leading markdown header /
|
||||
// CJK-leak and committed the cleaned remainder. It carries the export forward on BOTH the
|
||||
// fresh path (runStage) and resume (resumeFromChunkStatus reads the derived checkpoint), so
|
||||
// translateChunk assembles the identical FinalText either way. "" for every other disposition.
|
||||
RecoveredText string
|
||||
Disposition Disposition
|
||||
FlagReason FlagReason // "" when ok
|
||||
Detail string
|
||||
Attempts int
|
||||
// Escalated — a single-hop fallback draft was tried this stage (D12); when the
|
||||
// fallback passed the re-gate, Model above is the fallback (it answered).
|
||||
Escalated bool
|
||||
|
|
@ -34,10 +40,13 @@ type StageResult struct {
|
|||
|
||||
// ChunkOutcome is one chunk's result across the stage list.
|
||||
type ChunkOutcome struct {
|
||||
Chapter int
|
||||
ChunkIdx int
|
||||
Stages []StageResult
|
||||
FinalText string // the last stage's output; "" when the chunk is flagged
|
||||
Chapter int
|
||||
ChunkIdx int
|
||||
Stages []StageResult
|
||||
// FinalText is the exported text: the last stage's output on ok; the cosmetic-stripped
|
||||
// remainder on a FlagSanitizerStripped flag (D35.4a — the chunk is exported flagged, not lost
|
||||
// to an empty placeholder); "" on any other flag (the contaminated output never exports).
|
||||
FinalText string
|
||||
Disposition Disposition // ok | flagged (a chunk has no "skipped" — that is a per-later-stage state)
|
||||
FlagReason FlagReason // the flagging stage's reason ("" when ok)
|
||||
CostUSD float64 // THIS run's spend on this chunk
|
||||
|
|
|
|||
|
|
@ -46,6 +46,10 @@ const cheapGateVersion = "cheapgate-v2"
|
|||
type cheapGateConfig struct {
|
||||
yoPolicy string // "auto" (inconsistency only) | "all-yo" | "all-e"
|
||||
allowlist map[string]bool // lower-cased surfaces exempt from the interjection blocklist
|
||||
// regressionEnabled turns on the post-reflow regression guard (D38, regressionguard.go): an
|
||||
// OPT-IN observability flagger (draft→final length collapse + number drift) folded into this
|
||||
// result. Off → the two regression fields stay 0 and the output is byte-identical to before.
|
||||
regressionEnabled bool
|
||||
}
|
||||
|
||||
// cheapGateResult is the per-chunk outcome: a count per flagger plus human-readable detail lines
|
||||
|
|
@ -55,15 +59,22 @@ type cheapGateResult struct {
|
|||
YoInconsistent int `json:"yo,omitempty"`
|
||||
TranslitInterj int `json:"translit_interj,omitempty"`
|
||||
NumberMagnitude int `json:"number_magnitude,omitempty"`
|
||||
Detail []string `json:"detail,omitempty"`
|
||||
// LengthCollapse / NumberDrift are the opt-in post-reflow regression guard (D38,
|
||||
// regressionguard.go), folded into this observability result. They stay 0 unless
|
||||
// cfg.regressionEnabled, so a book that does not enable the guard serialises identically.
|
||||
LengthCollapse int `json:"length_collapse,omitempty"`
|
||||
NumberDrift int `json:"number_drift,omitempty"`
|
||||
Detail []string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
func (c cheapGateResult) total() int {
|
||||
return c.DialogueDash + c.YoInconsistent + c.TranslitInterj + c.NumberMagnitude
|
||||
return c.DialogueDash + c.YoInconsistent + c.TranslitInterj + c.NumberMagnitude + c.LengthCollapse + c.NumberDrift
|
||||
}
|
||||
|
||||
// runCheapGates runs all four flaggers over one chunk's source and FINAL translated text.
|
||||
func runCheapGates(source, final string, cfg cheapGateConfig) cheapGateResult {
|
||||
// runCheapGates runs the four always-on style flaggers over one chunk's source and FINAL text, plus
|
||||
// (opt-in) the draft→final regression guard. `draft` is the first-stage translator output (== final
|
||||
// when there is no distinct reflow stage, so the guard then trivially never fires).
|
||||
func runCheapGates(source, draft, final string, cfg cheapGateConfig) cheapGateResult {
|
||||
var r cheapGateResult
|
||||
n, det := lintDialogueDash(final)
|
||||
r.DialogueDash, r.Detail = n, append(r.Detail, det...)
|
||||
|
|
@ -76,6 +87,12 @@ func runCheapGates(source, final string, cfg cheapGateConfig) cheapGateResult {
|
|||
n, det = lintNumberMagnitude(source, final)
|
||||
r.NumberMagnitude = n
|
||||
r.Detail = append(r.Detail, det...)
|
||||
if cfg.regressionEnabled {
|
||||
rg := runRegressionGuard(draft, final)
|
||||
r.LengthCollapse = rg.LengthCollapse
|
||||
r.NumberDrift = rg.NumberDrift
|
||||
r.Detail = append(r.Detail, rg.Detail...)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ func TestRunCheapGatesCombined(t *testing.T) {
|
|||
src := "他有三万石粮食。"
|
||||
final := "- Ара-ара, — у Пётр было три миллиона мешков. Потом Петр ушёл."
|
||||
cfg := cheapGateConfig{yoPolicy: "auto", allowlist: map[string]bool{}}
|
||||
r := runCheapGates(src, final, cfg)
|
||||
r := runCheapGates(src, final, final, cfg)
|
||||
if r.DialogueDash == 0 {
|
||||
t.Error("expected a dialogue-dash flag (hyphen-led line)")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"textmachine/backend/internal/store"
|
||||
)
|
||||
|
||||
|
|
@ -39,6 +40,17 @@ func (r *Runner) classifyOutput(role, source, output, finish string, isFinal boo
|
|||
// its rule version folds into the snapshot only when enabled (sanitizerSnapshot).
|
||||
if isFinal && r.Pipeline.Gates.Sanitizer.Enabled {
|
||||
if san := sanitizeOutput(output); san.total() > 0 {
|
||||
// Cosmetic-only leak (leading markdown header and/or CJK-leak) → strip it and export
|
||||
// the remainder flagged, not drop the whole chunk (D35.4a). The guard is load-bearing:
|
||||
// we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that is now
|
||||
// clean, so runStage/resume can trust that the derived export text is usable; anything
|
||||
// else falls through to the substantive skip. Both are pure over `output`, so a resumed
|
||||
// checkpoint re-derives the identical verdict.
|
||||
if san.cosmeticOnly() {
|
||||
if stripped := stripCosmetic(output); strings.TrimSpace(stripped) != "" && sanitizeOutput(stripped).total() == 0 {
|
||||
return classification{FlagSanitizerStripped, san.summary()}
|
||||
}
|
||||
}
|
||||
return classification{FlagSanitizerDefect, san.summary()}
|
||||
}
|
||||
}
|
||||
|
|
@ -73,6 +85,14 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
|||
prev := ""
|
||||
flagged := false
|
||||
var flagReason FlagReason
|
||||
// recovered carries a cosmetic sanitizer strip's cleaned export forward from the flagging
|
||||
// stage (D35.4a): non-empty only when the FINAL stage flagged FlagSanitizerStripped, "" for a
|
||||
// dropped substantive flag. Captured from the FIRST flagging stage (the sanitizer flags only
|
||||
// the final stage, so an upstream substantive flag leaves it "" and the chunk exports empty).
|
||||
recovered := ""
|
||||
// draftText is the FIRST (translator) stage's output — the pre-reflow baseline for the opt-in
|
||||
// regression guard's draft→final comparison (== final in a single-stage pipeline).
|
||||
draftText := ""
|
||||
|
||||
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and
|
||||
// serialize the translator injection block. Recomputed on every run (resumed chunks
|
||||
|
|
@ -127,9 +147,13 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
|||
if sr.Disposition == DispFlagged {
|
||||
flagged = true
|
||||
flagReason = sr.FlagReason
|
||||
recovered = sr.RecoveredText
|
||||
continue
|
||||
}
|
||||
prev = sr.Text
|
||||
if stageIdx == 0 {
|
||||
draftText = sr.Text // the translator draft, before any reflow (regression-guard baseline)
|
||||
}
|
||||
}
|
||||
|
||||
// Post-check the FINAL, exported output (E1): the reader sees the last stage's text
|
||||
|
|
@ -161,7 +185,7 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
|||
// (the 万/億 magnitude gate compares source↔output).
|
||||
var cheap cheapGateResult
|
||||
if !flagged && prev != "" {
|
||||
cheap = runCheapGates(ch.Text, prev, r.cheapGateConfig())
|
||||
cheap = runCheapGates(ch.Text, draftText, prev, r.cheapGateConfig())
|
||||
if cheap.total() > 0 {
|
||||
r.Log.InfoContext(ctx, "cheap style gates flagged the chunk (observability, not a gate)",
|
||||
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "style_flags", cheap.total(),
|
||||
|
|
@ -183,7 +207,10 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
|||
if flagged {
|
||||
out.Disposition = DispFlagged
|
||||
out.FlagReason = flagReason
|
||||
out.FinalText = "" // garbage/refusal/glossary-miss never propagates to the next chunk or export
|
||||
// A cosmetic sanitizer strip exports its cleaned remainder (recovered != ""); every other
|
||||
// flag (garbage/refusal/glossary-miss/substantive-sanitizer) exports "" — the contaminated
|
||||
// output never reaches TM/export (D2 / D35.4a).
|
||||
out.FinalText = recovered
|
||||
} else {
|
||||
out.FinalText = prev
|
||||
}
|
||||
|
|
@ -247,5 +274,9 @@ func (r *Runner) cheapGateConfig() cheapGateConfig {
|
|||
for _, s := range r.Book.StyleAllowlist {
|
||||
allow[s] = true
|
||||
}
|
||||
return cheapGateConfig{yoPolicy: r.Book.YoPolicy, allowlist: allow}
|
||||
return cheapGateConfig{
|
||||
yoPolicy: r.Book.YoPolicy,
|
||||
allowlist: allow,
|
||||
regressionEnabled: r.Pipeline.Gates.RegressionGuard.Enabled,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,14 +82,24 @@ const (
|
|||
// escalatable in v1 (the L3 targeted re-ask is a Phase-2 remedy, research/14 §9).
|
||||
FlagGlossaryMiss FlagReason = "glossary_miss"
|
||||
|
||||
// FlagSanitizerDefect is the output-sanitizer verdict (D30.3, sanitizer.go): the stage
|
||||
// output carries an "instant unreadability" defect no other gate catches — a leaked
|
||||
// service preamble, a trailing note/edit block, a markdown ### header, a Latin-script
|
||||
// insertion, or a broken word form. Emitted ONLY when the opt-in Gates.Sanitizer is
|
||||
// enabled; the contaminated output is flagged (D2 flag+skip) so it never commits to
|
||||
// TM/export. NOT retryable (a same-model retry re-produces the artifact) and NOT
|
||||
// escalatable in v1 (the editor is pinned; a redrive surfaces the defect to a human).
|
||||
// FlagSanitizerDefect is the output-sanitizer verdict for a SUBSTANTIVE defect (D30.3,
|
||||
// sanitizer.go): the stage output carries an "instant unreadability" defect with no reliable
|
||||
// removable boundary — a leaked service preamble, a trailing note/edit block, a Latin-script
|
||||
// insertion, or a broken word form. Emitted ONLY when the opt-in Gates.Sanitizer is enabled;
|
||||
// the contaminated output is flagged AND skipped (D2 flag+skip) so it never commits to
|
||||
// TM/export (FinalText stays ""). NOT retryable (a same-model retry re-produces the artifact)
|
||||
// and NOT escalatable in v1 (the editor is pinned; a redrive surfaces the defect to a human).
|
||||
FlagSanitizerDefect FlagReason = "sanitizer_defect"
|
||||
|
||||
// FlagSanitizerStripped is the output-sanitizer verdict for a COSMETIC-ONLY defect that was
|
||||
// STRIPPED and the remainder EXPORTED (D38 infra-pack, D35.4a): a leading markdown «### Глава»
|
||||
// header and/or a stray CJK-leak run, removed deterministically by stripCosmetic. Unlike
|
||||
// FlagSanitizerDefect the chunk is NOT lost — the cleaned text is committed as the export
|
||||
// (final_hash points at a derived sanitized checkpoint, so the standard
|
||||
// final_hash→checkpoint.response_text export contract yields the cleaned text) — but it STAYS
|
||||
// flagged so a human verifies the auto-clean ("не терять чанк ... флаг для человека"). NOT
|
||||
// retryable / NOT escalatable (deterministic; a redrive would re-produce the same leak).
|
||||
FlagSanitizerStripped FlagReason = "sanitizer_stripped"
|
||||
)
|
||||
|
||||
// classifierVersion versions the INTRINSIC classify() verdict logic — the refusal
|
||||
|
|
|
|||
|
|
@ -41,8 +41,15 @@ import (
|
|||
// • UNION: ch5 fires an OBLIQUE-ONLY decl term (老人→старик) whose FINAL text carries the
|
||||
// NOMINATIVE «старик» → postcheck_miss=0 only because the base-dst∪decl union checks the
|
||||
// base (revert the union → this chunk false-flags a miss);
|
||||
// • SANITIZER: ch6 draft leaks a service preamble → the output-sanitizer gate (ON in the
|
||||
// fixture pipeline) flags it FlagSanitizerDefect and the edit is skipped.
|
||||
// • SANITIZER (SUBSTANTIVE): ch6 edit leaks a service preamble → the output-sanitizer gate (ON
|
||||
// in the fixture pipeline) flags it FlagSanitizerDefect and the edit is DROPPED (final_text "").
|
||||
// D38 infra-pack cells (pin the cosmetic strip-and-export path, D35.4a — final_hash points at a
|
||||
// derived $0 sanitized checkpoint so the export contract yields the cleaned text; resume re-serves
|
||||
// it identically at $0):
|
||||
// • SANITIZER (COSMETIC markdown): ch7 edit leaks a leading «### Глава 7» → flagged
|
||||
// sanitizer_stripped, final_text = «Глава 7\n\n…» (the «### » stripped), a derived checkpoint holds it;
|
||||
// • SANITIZER (COSMETIC CJK): ch8 edit leaks a sparse «特产» → flagged sanitizer_stripped, final_text
|
||||
// = the run removed and the seam tidied.
|
||||
// A cell with AMBIGUOUS>0 (an auto/draft candidate forcing a post-check) remains an OPTIONAL
|
||||
// future extension, deliberately NOT added here.
|
||||
//
|
||||
|
|
@ -57,7 +64,9 @@ const (
|
|||
goldenEchoMarker = "響動計画"
|
||||
goldenRefusalMarker = "拒絶計画"
|
||||
goldenUnionMarker = "老人の秘密" // ch5 — oblique-only decl union case
|
||||
goldenSanitizerMarker = "序文漏洩" // ch6 — sanitizer preamble-leak case
|
||||
goldenSanitizerMarker = "序文漏洩" // ch6 — sanitizer preamble-leak case (SUBSTANTIVE → dropped)
|
||||
goldenMarkdownMarker = "見出漏洩" // ch7 — cosmetic markdown «### Глава» leak (STRIPPED + exported)
|
||||
goldenCJKMarker = "漢字漏洩" // ch8 — cosmetic CJK-leak «特产» (STRIPPED + exported)
|
||||
)
|
||||
|
||||
// goldenRespond is the deterministic mock provider brain. The response text is a pure
|
||||
|
|
@ -85,6 +94,12 @@ func goldenRespond(body string) (text, finish string) {
|
|||
case strings.Contains(body, goldenUnionMarker) && !isEditBody(body):
|
||||
// ch5 draft carries a keying token («СТАРИК-СЦЕНА») the edit branch below keys on.
|
||||
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". СТАРИК-СЦЕНА Судзуки увидел старика.", "stop"
|
||||
case strings.Contains(body, goldenMarkdownMarker) && !isEditBody(body):
|
||||
// ch7 draft carries a keying token («МАРКДАУН-СЦЕНА») the cosmetic edit branch keys on.
|
||||
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". МАРКДАУН-СЦЕНА Судзуки открыл седьмую дверь.", "stop"
|
||||
case strings.Contains(body, goldenCJKMarker) && !isEditBody(body):
|
||||
// ch8 draft carries a keying token («ИЕРОГЛИФ-СЦЕНА») the cosmetic edit branch keys on.
|
||||
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". ИЕРОГЛИФ-СЦЕНА Судзуки нашёл камень.", "stop"
|
||||
case strings.Contains(body, goldenEchoMarker) && !isEditBody(body):
|
||||
if req.Model == "fake-fallback" {
|
||||
// The escalation hop returns a clean translation → the re-gate passes it.
|
||||
|
|
@ -100,6 +115,16 @@ func goldenRespond(body string) (text, finish string) {
|
|||
// ch5 edit — the FINAL text carries the NOMINATIVE «старик» (not an oblique decl
|
||||
// form of 老人→старик), exercising the base-dst∪decl union → postcheck_miss=0.
|
||||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки увидел, как в тени ждал старик.", "stop"
|
||||
case isEditBody(body) && strings.Contains(body, "МАРКДАУН-СЦЕНА"):
|
||||
// ch7 edit leaks a leading markdown «### Глава» header — a COSMETIC class: the sanitizer
|
||||
// STRIPS it and exports the remainder flagged sanitizer_stripped (D35.4a). This PINS the
|
||||
// strip-and-export path: final_text = «Глава 7\n\n…» (no «### »), disposition=flagged.
|
||||
return "### Глава 7\n\nСудзуки открыл седьмую дверь книгохранилища.", "stop"
|
||||
case isEditBody(body) && strings.Contains(body, "ИЕРОГЛИФ-СЦЕНА"):
|
||||
// ch8 edit leaks a sparse CJK run «特产» in otherwise-Russian prose (below the 15% echo
|
||||
// threshold classify() catches) — a COSMETIC class: stripped + exported flagged. PINS the
|
||||
// CJK-leak strip: «Судзуки нашёл 特产 древний камень.» → «Судзуки нашёл древний камень.»
|
||||
return "Судзуки нашёл 特产 древний камень на каменном алтаре.", "stop"
|
||||
case isEditBody(body):
|
||||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки шёл по коридорам Академии магии.", "stop"
|
||||
default:
|
||||
|
|
@ -171,7 +196,7 @@ func captureGolden(t *testing.T, label string, r *Runner, res *BookResult, wireB
|
|||
w(" stage=%s role=%s model=%s resume=%t disp=%s flag=%q attempts=%d escalated=%t esc_model=%q finish=%q cum_usd=%s detail=%q",
|
||||
st.Stage, st.Role, st.Model, st.FromResume, st.Disposition, st.FlagReason,
|
||||
st.Attempts, st.Escalated, st.EscalationModel, st.FinishReason, fl(st.CumCostUSD), st.Detail)
|
||||
w(" stage_text=%q", st.Text)
|
||||
w(" stage_text=%q recovered=%q", st.Text, st.RecoveredText)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
159
backend/internal/pipeline/regressionguard.go
Normal file
159
backend/internal/pipeline/regressionguard.go
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// regressionguard.go: the post-reflow regression guard (D38 infra-pack, research/18 §C#5) — two
|
||||
// deterministic OBSERVABILITY flaggers over the DRAFT→FINAL transform, opt-in like a QA gate
|
||||
// (Gates.RegressionGuard) but NEVER a disposition change. The reflow editor legitimately
|
||||
// restructures and merges sentences, so a HARD skip would false-flag a correct edit; instead a hit
|
||||
// is recorded in the cheap-gate observability channel (retrieval-state n_style_flags) and surfaced
|
||||
// in the report for a human. Both flaggers are tuned PRECISION over recall.
|
||||
//
|
||||
// (a) length collapse — the final is drastically SHORTER than the draft (>regressionMaxShrinkPct
|
||||
// of non-space chars), catching a reflow that dropped whole sentences/paragraphs (draft
|
||||
// 5000 → final 600). Reflow SUMMARISES structure into paragraphs; it must not delete content.
|
||||
// (b) number drift — a MULTI-DIGIT arabic number in the draft is ABSENT from the final, or a
|
||||
// new one APPEARS: 四成四=44%→«четыре десятых» (the 44 vanished), or a hallucinated figure.
|
||||
// Single digits are EXCLUDED (they are routinely spelled out — «5»→«пять» — a legit reflow),
|
||||
// and grouped triples are stitched (10 000 ≡ 10000), so the signal is high precision.
|
||||
//
|
||||
// HONEST recall gap: a number rendered as WORDS on BOTH sides (черновик «сорок четыре» → финал
|
||||
// «четыре десятых») is invisible here — offline word-number alignment is a Ф2 concern. And a
|
||||
// legit reflow that converts a multi-digit figure to words («44%»→«сорок четыре процента») will
|
||||
// show as a disappeared number: a FALSE observability flag, tolerable ONLY because this never
|
||||
// drops the chunk (a human glance dismisses it). Precision over recall, observability over gating.
|
||||
|
||||
const (
|
||||
// regressionMaxShrinkPct: a final shorter than the draft by MORE than this percent of non-space
|
||||
// characters is a collapse signal. 40% — a reflow that keeps content while merging lines shrinks
|
||||
// only slightly (Russian is ~fertility-neutral draft→final); losing ~half the characters is a
|
||||
// strong omission signal, not a merge. Tuned precision-over-recall; a code const (a change is a
|
||||
// loud cheapGateVersion bump), not config, so the gate carries only an on/off switch.
|
||||
regressionMaxShrinkPct = 40
|
||||
// regressionMinChars: below this the draft is too short for a percentage ratio to be meaningful
|
||||
// (a 3-word draft legitimately halving is noise), so the length flagger stays silent.
|
||||
regressionMinChars = 200
|
||||
)
|
||||
|
||||
// regressionGuardResult is the guard's per-chunk outcome (folded into the cheap-gate observability
|
||||
// result). LengthCollapse/NumberDrift are 0/1 counts; Detail carries the human-readable lines.
|
||||
type regressionGuardResult struct {
|
||||
LengthCollapse int
|
||||
NumberDrift int
|
||||
Detail []string
|
||||
}
|
||||
|
||||
// runRegressionGuard compares the DRAFT (first-stage translator output) with the FINAL (last-stage
|
||||
// reflow output). Pure and deterministic (sorted detail), so a resume re-derives identical counts.
|
||||
func runRegressionGuard(draft, final string) regressionGuardResult {
|
||||
var r regressionGuardResult
|
||||
|
||||
// (a) length collapse — non-space chars, mirroring the coverage gate's length metric.
|
||||
dn, fn := nonSpaceRuneCount(draft), nonSpaceRuneCount(final)
|
||||
if dn >= regressionMinChars && fn*100 < dn*(100-regressionMaxShrinkPct) {
|
||||
r.LengthCollapse = 1
|
||||
r.Detail = append(r.Detail, fmt.Sprintf(
|
||||
"reflow-регрессия: коллапс длины черновик→финал %d→%d непробельных символов (−%d%%, порог −%d%%) — возможен пропуск",
|
||||
dn, fn, 100-fn*100/dn, regressionMaxShrinkPct))
|
||||
}
|
||||
|
||||
// (b) number drift — multi-digit arabic tokens present on one side only.
|
||||
dNums, fNums := arabicNumberTokens(draft), arabicNumberTokens(final)
|
||||
disappeared := numberSetDiff(dNums, fNums)
|
||||
appeared := numberSetDiff(fNums, dNums)
|
||||
if len(disappeared) > 0 || len(appeared) > 0 {
|
||||
r.NumberDrift = len(disappeared) + len(appeared)
|
||||
if len(disappeared) > 0 {
|
||||
r.Detail = append(r.Detail, "reflow-регрессия: числа черновика отсутствуют в финале (возможен пропуск/дрейф разряда): "+joinPreview(disappeared))
|
||||
}
|
||||
if len(appeared) > 0 {
|
||||
r.Detail = append(r.Detail, "reflow-регрессия: в финале появились числа, которых не было в черновике (возможна добавка): "+joinPreview(appeared))
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// nonSpaceRuneCount counts non-whitespace runes — the length metric for the collapse flagger.
|
||||
func nonSpaceRuneCount(s string) int {
|
||||
n := 0
|
||||
for _, r := range s {
|
||||
if !unicode.IsSpace(r) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// arabicNumberTokens returns the MULTI-DIGIT (≥2) arabic integers in text, stitching grouping
|
||||
// separators (space / NBSP / comma) between runs of exactly three digits so "10 000" reads as one
|
||||
// "10000", not "10" and "000". Single-digit numbers are dropped (routinely spelled out in prose).
|
||||
func arabicNumberTokens(text string) []string {
|
||||
rs := []rune(text)
|
||||
var out []string
|
||||
for i := 0; i < len(rs); {
|
||||
if !isASCIIDigit(rs[i]) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
j := i
|
||||
for j < len(rs) && isASCIIDigit(rs[j]) {
|
||||
j++
|
||||
}
|
||||
digits := string(rs[i:j])
|
||||
// Stitch " ddd" / " ddd" / ",ddd" grouped triples.
|
||||
for j < len(rs) {
|
||||
if rs[j] == ' ' || rs[j] == ' ' || rs[j] == ',' {
|
||||
k := j + 1
|
||||
g := 0
|
||||
for k < len(rs) && isASCIIDigit(rs[k]) {
|
||||
k++
|
||||
g++
|
||||
}
|
||||
if g == 3 {
|
||||
digits += string(rs[j+1 : k])
|
||||
j = k
|
||||
continue
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
if len(digits) >= 2 {
|
||||
out = append(out, digits)
|
||||
}
|
||||
i = j
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// numberSetDiff returns the distinct members of a that do not appear in b, sorted. A SET diff (not
|
||||
// a multiset): a number present on both sides is covered even if its multiplicity differs, keeping
|
||||
// the precision-over-recall bias (a repeated number is not a drift signal on its own).
|
||||
func numberSetDiff(a, b []string) []string {
|
||||
inB := make(map[string]bool, len(b))
|
||||
for _, x := range b {
|
||||
inB[x] = true
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, x := range a {
|
||||
if !inB[x] && !seen[x] {
|
||||
seen[x] = true
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// joinPreview renders a sorted number list compactly for the detail line (bounded).
|
||||
func joinPreview(nums []string) string {
|
||||
const max = 8
|
||||
if len(nums) > max {
|
||||
return preview(fmt.Sprintf("%v …(+%d)", nums[:max], len(nums)-max))
|
||||
}
|
||||
return fmt.Sprintf("%v", nums)
|
||||
}
|
||||
82
backend/internal/pipeline/regressionguard_test.go
Normal file
82
backend/internal/pipeline/regressionguard_test.go
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// regressionguard_test.go — the post-reflow regression guard (D38). Precision over recall: each
|
||||
// flagger asserts BOTH firing on a real regression AND non-firing on a legitimate reflow.
|
||||
|
||||
func TestRegressionGuardLengthCollapse(t *testing.T) {
|
||||
// A draft well over the min-chars floor, halved by the "reflow" → collapse fires.
|
||||
draft := strings.Repeat("Судзуки медленно шёл по коридору академии магии. ", 12) // ~560 non-space chars
|
||||
shortFinal := "Судзуки шёл."
|
||||
if r := runRegressionGuard(draft, shortFinal); r.LengthCollapse == 0 {
|
||||
t.Errorf("expected a length-collapse flag (draft≫final)")
|
||||
}
|
||||
// A legitimate reflow that merges lines keeps ~all the content → must NOT fire.
|
||||
sameFinal := strings.Repeat("Судзуки медленно шёл по коридору академии магии. ", 11)
|
||||
if r := runRegressionGuard(draft, sameFinal); r.LengthCollapse != 0 {
|
||||
t.Errorf("length-collapse FALSE POSITIVE on a near-equal reflow: %v", r.Detail)
|
||||
}
|
||||
// A short draft below the floor must never fire (a % ratio is meaningless there).
|
||||
if r := runRegressionGuard("Он ушёл.", "Он."); r.LengthCollapse != 0 {
|
||||
t.Errorf("length-collapse fired below the min-chars floor: %v", r.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegressionGuardNumberDrift(t *testing.T) {
|
||||
// 四成四=44% rendered as «44» in the draft, drifted to a word form in the final → 44 disappears.
|
||||
if r := runRegressionGuard("У него было 44 процента запаса.", "У него было четыре десятых запаса."); r.NumberDrift == 0 {
|
||||
t.Errorf("expected a number-drift flag (44 vanished)")
|
||||
}
|
||||
// A number APPEARING in the final that was not in the draft (hallucinated figure).
|
||||
if r := runRegressionGuard("Он собрал войско.", "Он собрал войско из 3000 солдат."); r.NumberDrift == 0 {
|
||||
t.Errorf("expected a number-drift flag (3000 appeared)")
|
||||
}
|
||||
clean := []struct{ draft, final string }{
|
||||
// The multi-digit number is preserved across the reflow (44 on both sides).
|
||||
{"У него было 44 процента.", "У него имелось 44 процента запаса."},
|
||||
// Grouped-triple vs joined form of the SAME number must not drift (10 000 ≡ 10000).
|
||||
{"Армия в 10 000 воинов.", "Армия насчитывала 10000 воинов."},
|
||||
// SINGLE digits are excluded (routinely spelled out), so «5»→«пять» is not a drift.
|
||||
{"У него было 5 мечей.", "У него было пять мечей."},
|
||||
// No numbers at all — never fires.
|
||||
{"Судзуки шёл по коридору академии магии.", "Судзуки медленно шёл по длинному коридору."},
|
||||
}
|
||||
for _, c := range clean {
|
||||
if r := runRegressionGuard(c.draft, c.final); r.NumberDrift != 0 {
|
||||
t.Errorf("number-drift FALSE POSITIVE on %q→%q: %v", c.draft, c.final, r.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunCheapGatesRegressionOptIn pins that the guard is OFF by default and folds into the
|
||||
// observability total only when enabled (a book that does not opt in serialises identically).
|
||||
func TestRunCheapGatesRegressionOptIn(t *testing.T) {
|
||||
draft := strings.Repeat("Судзуки медленно шёл по коридору академии магии. ", 12)
|
||||
final := "Судзуки шёл."
|
||||
off := runCheapGates("", draft, final, cheapGateConfig{yoPolicy: "auto", allowlist: map[string]bool{}})
|
||||
if off.LengthCollapse != 0 || off.NumberDrift != 0 {
|
||||
t.Errorf("regression guard fired while disabled: %+v", off)
|
||||
}
|
||||
on := runCheapGates("", draft, final, cheapGateConfig{yoPolicy: "auto", allowlist: map[string]bool{}, regressionEnabled: true})
|
||||
if on.LengthCollapse == 0 {
|
||||
t.Errorf("regression guard did not fire while enabled: %+v", on)
|
||||
}
|
||||
if on.total() <= off.total() {
|
||||
t.Errorf("enabled guard must raise the observability total: off=%d on=%d", off.total(), on.total())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegressionGuardDeterministic — a pure function of (draft, final), resume-safe.
|
||||
func TestRegressionGuardDeterministic(t *testing.T) {
|
||||
d := strings.Repeat("текст с числом 44 и 128. ", 20)
|
||||
f := "короткий финал без чисел"
|
||||
a, b := runRegressionGuard(d, f), runRegressionGuard(d, f)
|
||||
if a.LengthCollapse != b.LengthCollapse || a.NumberDrift != b.NumberDrift ||
|
||||
strings.Join(a.Detail, "|") != strings.Join(b.Detail, "|") {
|
||||
t.Fatalf("runRegressionGuard not deterministic: %+v vs %+v", a, b)
|
||||
}
|
||||
}
|
||||
|
|
@ -50,6 +50,26 @@ func (r *Runner) resumeFromChunkStatus(ctx context.Context, st config.Stage, ch
|
|||
sr.Model = cp.ModelActual
|
||||
sr.FinishReason = cp.FinishReason
|
||||
sr.Text = cp.ResponseText
|
||||
} else if cs.FlagReason == string(FlagSanitizerStripped) && cs.FinalHash != "" {
|
||||
// A cosmetic sanitizer strip (D35.4a): the cleaned export text lives in the derived
|
||||
// checkpoint final_hash points at (written durably before this chunk_status row), so resume
|
||||
// re-serves the identical FinalText WITHOUT re-stripping — deterministic and $0. A missing
|
||||
// derived checkpoint is a torn store (final_hash is written after it), so fail loud like the
|
||||
// ok branch rather than silently export empty.
|
||||
cp, err := r.Store.GetCheckpoint(cs.FinalHash)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pipeline: read sanitized-export checkpoint %.12s for %s/ch%d/chunk%d/%s: %w",
|
||||
cs.FinalHash, r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, err)
|
||||
}
|
||||
if cp == nil || cp.ResponseText == "" {
|
||||
return nil, fmt.Errorf("pipeline: chunk_status sanitizer_stripped for %s/ch%d/chunk%d/%s references derived checkpoint %.12s which is missing/empty — inconsistent store",
|
||||
r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, cs.FinalHash)
|
||||
}
|
||||
sr.RecoveredText = cp.ResponseText
|
||||
// Mirror the ok branch: the resume request_log row's request_hash points at this derived
|
||||
// checkpoint, so report ITS finish_reason ("sanitized_export") rather than a bare "" that
|
||||
// contradicts the referenced row (adversarial review NIT; telemetry-only).
|
||||
sr.FinishReason = cp.FinishReason
|
||||
}
|
||||
rl := r.baseRequestLog(st, ch, st.Model, cs.FinalHash)
|
||||
rl.ModelActual = sr.Model
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
// (sanitizerVersion, mirroring coverageSnapshot), and moves to verdictSnapshotID once
|
||||
// content-addressed resume lands (D15.2).
|
||||
//
|
||||
// Five classes, each tuned PRECISION over recall (fire only on a high-confidence signal;
|
||||
// Six classes, each tuned PRECISION over recall (fire only on a high-confidence signal;
|
||||
// the ambiguous middle stays silent — a false flag turns a good chunk into an export
|
||||
// placeholder, worse than a missed defect for an opt-in readability gate):
|
||||
//
|
||||
|
|
@ -27,6 +27,21 @@ import (
|
|||
// 3. markdown ### header — «### Глава 1» (8/54 finals of stage A — D29 finding)
|
||||
// 4. Latin-script insertion — an untranslated Latin phrase/heavy Latin in the ru output
|
||||
// 5. broken word form — homoglyph-mixed token or an invalid Cyrillic sign bigram
|
||||
// 6. CJK-leak in the final — a stray Han ideograph / kana / fullwidth glyph in the ru output
|
||||
// («…охотники,却有 деньги!» arms/C/17.0 — D38.1; below the ≥15%
|
||||
// echo threshold classify() already catches, so a sparse leak)
|
||||
//
|
||||
// DISPOSITION-BY-CLASS (D38 infra-pack): the classes split into two tiers so a chunk is not
|
||||
// silently lost to an export placeholder when the leak is a REMOVABLE cosmetic span (D35.4a —
|
||||
// ch5/ch20 chapter openers dropped whole for a leading «### Глава N»):
|
||||
// - COSMETIC (strippable, exported + flagged for a human, cosmeticOnly()==true): the markdown
|
||||
// header (3) and the CJK-leak (6). The leak sits in a bounded span stripCosmetic() removes
|
||||
// deterministically, leaving readable prose; the chunk is flagged sanitizer_stripped (a
|
||||
// "auto-cleaned, verify" signal), NOT dropped.
|
||||
// - SUBSTANTIVE (dropped to a placeholder, current behaviour): the preamble (1), trailing
|
||||
// note (2), Latin insertion (4) and broken word (5). Their contamination has no reliable
|
||||
// removable boundary (a preamble may swallow real narration; a Latin clause / broken token is
|
||||
// lost content), so the output is not trusted — flagged sanitizer_defect + skipped.
|
||||
//
|
||||
// HONEST recall gap (documented, precision over recall): the broken-word class catches only
|
||||
// the two ZERO-false-positive signatures (invalid Cyrillic sign bigrams, mixed Cyrillic/Latin
|
||||
|
|
@ -43,7 +58,15 @@ import (
|
|||
// v2 (D33.1/33.2): tail edit-summary «Основные правки для справки:» now flags; the
|
||||
// junction-duplication split detector was removed (false-flagged prose); markdown/heavy-Latin
|
||||
// narrowed. A verdict-axis change → loud --resnapshot + golden re-capture (invariant №8).
|
||||
const sanitizerVersion = "sanitizer-v2"
|
||||
// v3 (D38 infra-pack): added the CJK-leak class (Han/kana/fullwidth in the ru final) and the
|
||||
// cosmetic-vs-substantive disposition split (the markdown header and CJK-leak are now STRIPPED
|
||||
// and exported flagged, not dropped to an empty placeholder — D35.4a). Both shift the resolved
|
||||
// disposition/export of a flagged chunk → a loud --resnapshot + golden re-capture (invariant №8).
|
||||
// v4 (D38 infra-pack, adversarial review): stripCosmetic now FOLDS content-bearing fullwidth ASCII
|
||||
// («１２３»→«123», not deleted) and ideographic comma/period, space-replaces dropped ideographs (no
|
||||
// word-merge), removes an emptied bracket pair, and no longer reformats a legit «2 : 1» — all shift
|
||||
// a stripped chunk's EXPORT text → a loud --resnapshot + golden re-capture (invariant №8).
|
||||
const sanitizerVersion = "sanitizer-v4"
|
||||
|
||||
// sanitizer tuning constants (versioned by sanitizerVersion).
|
||||
const (
|
||||
|
|
@ -72,11 +95,21 @@ type sanitizerResult struct {
|
|||
MarkdownHeader int `json:"markdown_header,omitempty"`
|
||||
LatinInsert int `json:"latin_insert,omitempty"`
|
||||
BrokenWord int `json:"broken_word,omitempty"`
|
||||
CJKLeak int `json:"cjk_leak,omitempty"`
|
||||
Detail []string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
func (s sanitizerResult) total() int {
|
||||
return s.Preamble + s.TrailingNote + s.MarkdownHeader + s.LatinInsert + s.BrokenWord
|
||||
return s.Preamble + s.TrailingNote + s.MarkdownHeader + s.LatinInsert + s.BrokenWord + s.CJKLeak
|
||||
}
|
||||
|
||||
// cosmeticOnly reports whether the ONLY classes that fired are the STRIPPABLE cosmetic ones
|
||||
// (markdown header and/or CJK-leak) — the strip-and-export tier (D35.4a). A single substantive
|
||||
// class (preamble/trailing-note/latin/broken-word) pulls the whole chunk into the skip tier: its
|
||||
// contamination has no reliable removable boundary, so the output is not trusted even if a
|
||||
// cosmetic leak also happens to be present. False when nothing fired.
|
||||
func (s sanitizerResult) cosmeticOnly() bool {
|
||||
return s.total() > 0 && s.Preamble == 0 && s.TrailingNote == 0 && s.LatinInsert == 0 && s.BrokenWord == 0
|
||||
}
|
||||
|
||||
// summary is the human-readable disposition detail for a sanitizer flag (deterministic —
|
||||
|
|
@ -91,7 +124,7 @@ func (s sanitizerResult) summary() string {
|
|||
return strings.Join(s.Detail, "; ")
|
||||
}
|
||||
|
||||
// sanitizeOutput runs all five classes over one chunk's FINAL translated text.
|
||||
// sanitizeOutput runs all six classes over one chunk's FINAL translated text.
|
||||
func sanitizeOutput(text string) sanitizerResult {
|
||||
var r sanitizerResult
|
||||
if det := detectLeadingPreamble(text); det != "" {
|
||||
|
|
@ -114,6 +147,10 @@ func sanitizeOutput(text string) sanitizerResult {
|
|||
r.BrokenWord = n
|
||||
r.Detail = append(r.Detail, det...)
|
||||
}
|
||||
if n, det := detectCJKLeak(text); n > 0 {
|
||||
r.CJKLeak = n
|
||||
r.Detail = append(r.Detail, det...)
|
||||
}
|
||||
sort.Strings(r.Detail)
|
||||
return r
|
||||
}
|
||||
|
|
@ -316,6 +353,125 @@ func detectBrokenWords(text string) (int, []string) {
|
|||
return len(det), det
|
||||
}
|
||||
|
||||
// --- 6. CJK-leak in the final --------------------------------------------------
|
||||
|
||||
// isCJKLeakRune reports whether r is a stray CJK glyph that has NO place in a Russian final: a
|
||||
// Han ideograph, kana, a fullwidth/halfwidth form (U+FF00–FFEF — «，（）！» etc.), or a CJK
|
||||
// symbol/punctuation (U+3001–303F — «、。「」»). The ideographic SPACE U+3000 is deliberately
|
||||
// EXCLUDED (it is an indent artifact, not a content leak — research/18 §C#4 "draft-21 includes a
|
||||
// U+3000 indent → 9 real leaks"): stripCosmetic normalises it to a plain space, but it does not
|
||||
// alone FIRE the class, so a legitimately-indented chunk is not flag-stormed. In clean Russian
|
||||
// prose every one of these is count-0, so the class is high precision. The intrinsic classify()
|
||||
// already flags a ≥15%-CJK output as an ECHO (cjk_artifact, substantive) BEFORE the sanitizer
|
||||
// runs, so this class only ever sees a SPARSE leak in otherwise-Russian text.
|
||||
func isCJKLeakRune(r rune) bool {
|
||||
switch {
|
||||
case unicode.Is(unicode.Han, r):
|
||||
return true
|
||||
case unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r):
|
||||
return true
|
||||
case r >= 0xFF00 && r <= 0xFFEF: // fullwidth & halfwidth forms
|
||||
return true
|
||||
case r >= 0x3001 && r <= 0x303F: // CJK symbols & punctuation, EXCLUDING U+3000 (ideographic space)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail
|
||||
// per distinct run. A single run («却有») is enough to fire — the class is COSMETIC, so the whole
|
||||
// chunk is not lost: stripCosmetic removes the run and the remainder exports (flagged for review).
|
||||
func detectCJKLeak(text string) (int, []string) {
|
||||
var det []string
|
||||
seen := map[string]bool{}
|
||||
n := 0
|
||||
rs := []rune(text)
|
||||
for i := 0; i < len(rs); {
|
||||
if !isCJKLeakRune(rs[i]) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
j := i
|
||||
for j < len(rs) && isCJKLeakRune(rs[j]) {
|
||||
j++
|
||||
}
|
||||
n++
|
||||
run := string(rs[i:j])
|
||||
if !seen[run] {
|
||||
seen[run] = true
|
||||
det = append(det, "CJK-утечка в ru-выходе: "+preview(run))
|
||||
}
|
||||
i = j
|
||||
}
|
||||
return n, det
|
||||
}
|
||||
|
||||
// --- deterministic strip of the COSMETIC classes (D35.4a export fix) -------------
|
||||
|
||||
// leadingMarkdownStripRE matches the leading markdown ATX prefix on a line (optional indent, 1–6
|
||||
// hashes, blank(s)) directly before a header WORD/number — the same shape markdownHeaderRE fires
|
||||
// on. It captures the first header rune so ReplaceAll keeps the heading text, dropping only the
|
||||
// «### » artifact («### Глава 5» → «Глава 5»). A decorative «### ---» scene break (punctuation
|
||||
// after the hashes) does NOT match, so it is left intact (and it never fired the class either).
|
||||
var leadingMarkdownStripRE = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+([\p{L}\p{Nd}])`)
|
||||
|
||||
// stripCosmetic deterministically removes the two STRIPPABLE cosmetic leak classes — leading
|
||||
// markdown headers and CJK-leak runs — and tidies the whitespace the removal leaves, so the
|
||||
// remainder is readable. It is a PURE function (resume re-derives the identical export), invoked
|
||||
// ONLY on output the sanitizer classified as cosmeticOnly (classifyOutput guarantees the result
|
||||
// is non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
|
||||
// sanitizer_stripped). It never crosses newlines when tidying, so paragraph structure is kept.
|
||||
func stripCosmetic(text string) string {
|
||||
// 1) markdown header prefixes → keep the heading text, drop the «#### » artifact.
|
||||
out := leadingMarkdownStripRE.ReplaceAllString(text, "$1")
|
||||
// 2) neutralise CJK-leak runes. CONTENT-BEARING glyphs are FOLDED, not dropped (adversarial
|
||||
// review MAJOR): a fullwidth ASCII form is a wrong-glyph rendering of a plain char, so «１２３»
|
||||
// folds to «123» (the number is kept, not deleted) and «，」→«,» — the same NFKC fold the
|
||||
// memory normaliser uses. Only genuinely contentless runes (Han/kana/CJK brackets/symbols) are
|
||||
// removed, and a removed run leaves a SINGLE SPACE so adjacent Russian words do not merge
|
||||
// («нашёл特产древний» → «нашёл древний», not «нашёлдревний»). U+3000 → a plain space (indent).
|
||||
var b strings.Builder
|
||||
b.Grow(len(out))
|
||||
for _, r := range out {
|
||||
switch {
|
||||
case r == ' ': // U+3000 ideographic space → plain space
|
||||
b.WriteByte(' ')
|
||||
case r >= 0xFF01 && r <= 0xFF5E: // fullwidth ASCII form → fold to plain ASCII (１→1, Ａ→A, ，→,)
|
||||
b.WriteRune(r - 0xFEE0)
|
||||
case r == '、': // ideographic comma → plain comma (keep the clause boundary)
|
||||
b.WriteByte(',')
|
||||
case r == '。': // ideographic full stop → plain period
|
||||
b.WriteByte('.')
|
||||
case isCJKLeakRune(r): // Han / kana / CJK brackets / other symbol → drop, leaving a space
|
||||
b.WriteByte(' ')
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
out = b.String()
|
||||
// 3) tidy the seams the removal created, per line (horizontal whitespace only — never a
|
||||
// newline): collapse doubled spaces, drop a bracket/quote pair the strip emptied («(羅漢拳)» →
|
||||
// «()» → removed), drop a space now sitting before closing punctuation or after an opening
|
||||
// bracket/quote (never-correct Russian spacing), and trim trailing blanks. ':' is deliberately
|
||||
// NOT in the closing-punct set: a space-before-colon is legitimate in a Russian ratio/score/time
|
||||
// («счёт 2 : 1») and must not be reformatted (adversarial review — false positive).
|
||||
out = multiSpaceRE.ReplaceAllString(out, " ")
|
||||
out = emptyBracketRE.ReplaceAllString(out, "")
|
||||
out = spaceBeforePunctRE.ReplaceAllString(out, "$1")
|
||||
out = spaceAfterOpenRE.ReplaceAllString(out, "$1")
|
||||
out = multiSpaceRE.ReplaceAllString(out, " ") // an emptied bracket may leave a doubled space
|
||||
out = trailingHSpaceRE.ReplaceAllString(out, "")
|
||||
return out
|
||||
}
|
||||
|
||||
var (
|
||||
multiSpaceRE = regexp.MustCompile(`[ \t]{2,}`)
|
||||
emptyBracketRE = regexp.MustCompile(`\([ \t]*\)|«[ \t]*»|\[[ \t]*\]`)
|
||||
spaceBeforePunctRE = regexp.MustCompile(`[ \t]+([,.!?;…»)\]])`)
|
||||
spaceAfterOpenRE = regexp.MustCompile(`([«(\[])[ \t]+`)
|
||||
trailingHSpaceRE = regexp.MustCompile(`(?m)[ \t]+$`)
|
||||
)
|
||||
|
||||
type scriptToken struct {
|
||||
text string
|
||||
cyr bool // Cyrillic-only
|
||||
|
|
|
|||
|
|
@ -211,6 +211,100 @@ func TestSanitizerCleanNarrative(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestSanitizerCJKLeak — the D38.1 class: a stray Han/kana/fullwidth glyph in the ru final.
|
||||
func TestSanitizerCJKLeak(t *testing.T) {
|
||||
fire := []string{
|
||||
// The span the orchestrator verified (arms/C/17.0): a raw Han run mid-Russian-sentence.
|
||||
"«…охотники,却有 на это лишние деньги!»",
|
||||
"Судзуки нашёл 特产 древнего клана.", // single Han run
|
||||
"Разряд资质乙等 остался неясен.", // Han glued to Cyrillic
|
||||
"Он сказал: すごい, и замолчал.", // kana leak (ja book)
|
||||
"Цена была １２３ золотых.", // fullwidth digits
|
||||
"Он вошёл в дом、и дверь закрылась.", // ideographic comma (U+3001)
|
||||
}
|
||||
for _, s := range fire {
|
||||
if got := sanitizeOutput(s); got.CJKLeak == 0 {
|
||||
t.Errorf("CJK-leak not detected in %q: %+v", preview(s), got)
|
||||
}
|
||||
}
|
||||
clean := []string{
|
||||
"Судзуки шёл по коридорам Академии магии, вспоминая вчерашнюю долгую лекцию.",
|
||||
"— Сегодня я открою дверь книгохранилища, — прошептал он, сжимая ключ.",
|
||||
"На гербе была латинская надпись, но он её не разобрал в темноте.",
|
||||
"Цена была 123 золотых, и он расплатился не торгуясь.", // ASCII digits — not fullwidth
|
||||
// U+3000 ideographic space alone is an INDENT artifact, NOT a content leak → must NOT fire
|
||||
// (research/18 §C#4); stripCosmetic still normalises it, but it does not flag the chunk.
|
||||
" Судзуки медленно шёл по длинному коридору к башне.",
|
||||
}
|
||||
for _, s := range clean {
|
||||
if got := sanitizeOutput(s); got.CJKLeak != 0 {
|
||||
t.Errorf("CJK-leak FALSE POSITIVE in %q: %v", preview(s), got.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSanitizerCosmeticOnly pins the strip-vs-skip disposition matrix (D35.4a): only the markdown
|
||||
// header and CJK-leak classes are strippable; any substantive class pulls the chunk into skip.
|
||||
func TestSanitizerCosmeticOnly(t *testing.T) {
|
||||
cosmetic := []string{
|
||||
"### Глава 7\n\nСудзуки открыл седьмую дверь и замер на пороге.", // markdown only
|
||||
"Судзуки нашёл 特产 древнего клана в тёмном углу книгохранилища.", // cjk only
|
||||
"### Пролог\n\nСудзуки увидел 却有 на алтаре забытого клана.", // BOTH cosmetic classes
|
||||
}
|
||||
for _, s := range cosmetic {
|
||||
got := sanitizeOutput(s)
|
||||
if got.total() == 0 || !got.cosmeticOnly() {
|
||||
t.Errorf("expected cosmeticOnly for %q: %+v", preview(s), got)
|
||||
}
|
||||
}
|
||||
substantive := []string{
|
||||
"Вот перевод фрагмента: Судзуки открыл дверь.", // preamble (substantive)
|
||||
// a LEADING preamble co-occurring with a CJK leak → the substantive class wins (skip, not strip).
|
||||
"Вот перевод фрагмента: Судзуки увидел 特产 на алтаре забытого клана.",
|
||||
"Судзуки открыл дверь.\n\nПримечание: имя героя оставлено как есть.", // trailing note
|
||||
"Он прочитал: the quick brown fox jumps over lazy dog today.", // latin phrase
|
||||
}
|
||||
for _, s := range substantive {
|
||||
if got := sanitizeOutput(s); got.total() == 0 || got.cosmeticOnly() {
|
||||
t.Errorf("expected NOT cosmeticOnly (substantive) for %q: %+v", preview(s), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStripCosmetic pins the exact deterministic strip output — the "экспортит текст БЕЗ ведущего
|
||||
// ###" contract — and that the result is itself clean (the classifyOutput guarantee).
|
||||
func TestStripCosmetic(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
// markdown header prefix removed, heading text kept.
|
||||
{"### Глава 7\n\nСудзуки открыл дверь.", "Глава 7\n\nСудзуки открыл дверь."},
|
||||
{"# Пролог\nТекст главы.", "Пролог\nТекст главы."},
|
||||
// Han run removed; the seam is tidied (no double space, no space-before-comma), no word-merge.
|
||||
{"«…охотники,却有 на это лишние деньги!»", "«…охотники, на это лишние деньги!»"},
|
||||
{"Судзуки нашёл 特产 древнего клана.", "Судзуки нашёл древнего клана."},
|
||||
{"нашёл特产древний", "нашёл древний"}, // no spaces in source → strip leaves a space, not a merge
|
||||
// CONTENT-BEARING fullwidth ASCII is FOLDED, not deleted (adversarial review MAJOR): the number survives.
|
||||
{"Цена была １２３ золотых.", "Цена была 123 золотых."},
|
||||
{"Индекс ＡＢＣ найден.", "Индекс ABC найден."},
|
||||
// ideographic comma/period fold to plain — the clause boundary is kept.
|
||||
{"Он вошёл в дом、и дверь закрылась。", "Он вошёл в дом,и дверь закрылась."},
|
||||
// a parenthetical Han gloss is stripped and the emptied «()» removed (adversarial review).
|
||||
{"Кулак Ло Хань (羅漢拳) был силён.", "Кулак Ло Хань был силён."},
|
||||
// a legit ratio «2 : 1» is NOT reformatted (colon dropped from the tidy set).
|
||||
{"### Итог\n\nСчёт был 2 : 1 却有 в пользу гостей.", "Итог\n\nСчёт был 2 : 1 в пользу гостей."},
|
||||
// U+3000 ideographic space normalised to a plain space (the indent artifact).
|
||||
{" Судзуки шёл.", " Судзуки шёл."},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := stripCosmetic(c.in)
|
||||
if got != c.want {
|
||||
t.Errorf("stripCosmetic(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
if san := sanitizeOutput(got); san.total() != 0 {
|
||||
t.Errorf("stripCosmetic(%q) is not clean: %+v", c.in, san.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSanitizerDeterministic asserts the verdict is a pure function of the text (resume-safe).
|
||||
func TestSanitizerDeterministic(t *testing.T) {
|
||||
s := "### Заголовок\nВот отредактированный перевод: текст. Стольь странно.\nПримечание: правка."
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ package pipeline
|
|||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
|
@ -149,8 +151,12 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
|||
runCost += esc.fb.runCost
|
||||
anyFresh = anyFresh || esc.fb.freshCall
|
||||
escalated, escModel = true, st.EscalateTo
|
||||
if esc.fb.cls.ok() {
|
||||
last = esc.fb // the fallback draft passed the re-gate → it is authoritative
|
||||
if esc.fb.cls.ok() || esc.fb.cls.Reason == FlagSanitizerStripped {
|
||||
// The fallback is authoritative when it passed the re-gate OR when it is a cosmetic
|
||||
// sanitizer strip on a FINAL escalatable stage (adversarial review): a stripped fallback
|
||||
// is a usable-but-flagged export, so it must be recovered (final_hash → its cleaned text)
|
||||
// rather than discarded to an empty placeholder — mirroring the non-escalated path.
|
||||
last = esc.fb
|
||||
}
|
||||
// else: the fallback also failed → keep the primary flag (last unchanged);
|
||||
// the fallback call is billed and counted, the chunk stays flagged (1 hop).
|
||||
|
|
@ -158,12 +164,27 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
|||
|
||||
disposition := last.cls.Reason.disposition()
|
||||
finalHash := ""
|
||||
recovered := ""
|
||||
if disposition == DispOK {
|
||||
finalHash = last.reqHash
|
||||
} else if last.cls.Reason == FlagSanitizerStripped {
|
||||
// Cosmetic leak (D35.4a): strip it and COMMIT the cleaned remainder as a $0 derived export
|
||||
// checkpoint, then point final_hash at it — so the standard final_hash→checkpoint.response_text
|
||||
// export (records.json / exp12_extract) yields the cleaned text instead of empty, while the
|
||||
// chunk stays flagged for a human. classifyOutput guaranteed the strip is non-empty and clean.
|
||||
// The derived checkpoint is durably written BEFORE chunk_status references it (the ok path's
|
||||
// checkpoint-before-pointer discipline), so a resume never dangles final_hash at a missing row.
|
||||
recovered = stripCosmetic(last.text)
|
||||
dh, err := r.commitSanitizedExport(st, ch, job, last, attemptsMade-1, recovered)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finalHash = dh
|
||||
}
|
||||
// chunk_status is a RESOLVE over the checkpoints: cost_usd sums every attempt
|
||||
// (F3-honest — retries are counted), final_hash points at the authoritative
|
||||
// checkpoint the ok path serves on resume.
|
||||
// checkpoint the ok path serves on resume (or the derived sanitized-export checkpoint for
|
||||
// a FlagSanitizerStripped chunk, so its cleaned text still exports).
|
||||
if err := r.Store.UpsertChunkStatus(store.ChunkStatus{
|
||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: st.Name,
|
||||
SnapshotID: snapID, ContentHash: contentHash, Disposition: string(disposition), FlagReason: string(last.cls.Reason),
|
||||
|
|
@ -181,6 +202,7 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
|||
CumCostUSD: cumCost,
|
||||
LatencyMS: last.latency,
|
||||
FinishReason: last.finish,
|
||||
RecoveredText: recovered,
|
||||
Disposition: disposition,
|
||||
FlagReason: last.cls.Reason,
|
||||
Detail: last.cls.Detail,
|
||||
|
|
@ -198,6 +220,24 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
|||
return sr, nil
|
||||
}
|
||||
|
||||
// commitSanitizedExport persists the cosmetic-stripped export text as a $0 derived checkpoint and
|
||||
// returns its request_hash (→ chunk_status.final_hash). The hash is CONTENT-ADDRESSED and
|
||||
// namespaced ("tm-sanitized-v1:") so it can never collide with a real attempt's hex request_hash
|
||||
// and a resume/re-run re-derives the identical id for free. Called ONLY for a FlagSanitizerStripped
|
||||
// final stage (D35.4a); `att` is the flagged completion the strip ran over.
|
||||
func (r *Runner) commitSanitizedExport(st config.Stage, ch Chunk, job *store.Job, att stageAttempt, attempt int, stripped string) (string, error) {
|
||||
sum := sha256.Sum256([]byte("tm-sanitized-v1\x00" + att.reqHash + "\x00" + stripped))
|
||||
derivedHash := "tm-sanitized-v1:" + hex.EncodeToString(sum[:])
|
||||
if err := r.Store.PutDerivedCheckpoint(store.Checkpoint{
|
||||
RequestHash: derivedHash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: attempt,
|
||||
Stage: st.Name, Role: st.Role, ModelRequested: att.modelActual, ModelActual: att.modelActual,
|
||||
ResponseText: stripped, UsageJSON: "{}", FinishReason: "sanitized_export",
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("pipeline: commit sanitized export ch%d/chunk%d/%s: %w", ch.Chapter, ch.ChunkIdx, st.Name, err)
|
||||
}
|
||||
return derivedHash, nil
|
||||
}
|
||||
|
||||
// stageAttempt is the result of one attempt (a checkpoint hit or a fresh call).
|
||||
type stageAttempt struct {
|
||||
reqHash string
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ func flagReasonSeverity(reason string) int {
|
|||
case FlagCJKArtifact, FlagExcisionSuspect, FlagCoverageFail:
|
||||
return 1
|
||||
case FlagSanitizerDefect:
|
||||
// A contaminated output (leaked preamble / notes / ###) is unreadable-as-shipped —
|
||||
// A contaminated output (leaked preamble / notes) that was DROPPED is unreadable-as-shipped —
|
||||
// ranked with the deterministic content failures, above a mere budget symptom.
|
||||
return 2
|
||||
case FlagLoopDegenerate:
|
||||
|
|
@ -113,10 +113,15 @@ func flagReasonSeverity(reason string) int {
|
|||
return 4
|
||||
case FlagGlossaryMiss:
|
||||
return 5
|
||||
case FlagSanitizerStripped:
|
||||
// A cosmetic leak the sanitizer STRIPPED and exported (D35.4a): the chunk shipped cleaned,
|
||||
// so it is the least alarming flag — an "auto-cleaned, glance to verify" signal, ranked below
|
||||
// a budget symptom (the chunk is not lost; a human need only spot-check the auto-clean).
|
||||
return 7
|
||||
case FlagLength, FlagEmpty:
|
||||
return 6
|
||||
}
|
||||
return 7
|
||||
return 8
|
||||
}
|
||||
|
||||
// bookChunks re-derives the book's chunk manifest — Ingest + SplitChunks — for the honest N/M
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -41,3 +41,11 @@
|
|||
第六章 序文漏洩
|
||||
|
||||
鈴木は最後の扉の前に立ち、深く息を吸い込んだ。その先に何があるのか、誰も知らなかった。
|
||||
|
||||
第七章 見出漏洩
|
||||
|
||||
七層目の朝、鈴木は最後の書庫の扉を開けた。扉の奥には見出しだけが刻まれた石板が静かに待っていた。誰もその文字を読めなかった。
|
||||
|
||||
第八章 漢字漏洩
|
||||
|
||||
鈴木は石板の文字をなぞり、古い漢字の意味を思い出そうとした。だが記憶は霧のように薄れ、指先だけが淡く光っていた。
|
||||
|
|
|
|||
|
|
@ -170,6 +170,33 @@ func (s *Store) SettleWithCheckpoint(res Reservation, cost float64, cp Checkpoin
|
|||
return tx.Commit()
|
||||
}
|
||||
|
||||
// PutDerivedCheckpoint persists a $0 DERIVED checkpoint — a deterministic post-processing
|
||||
// artifact, NOT a billed provider response (D38 infra-pack): the output-sanitizer's cosmetic
|
||||
// strip commits the cleaned final text here so the standard final_hash→checkpoint.response_text
|
||||
// export contract (records.json / exp12_extract) yields the cleaned text for a flagged chunk that
|
||||
// would otherwise export empty (D35.4a). It touches NO spend/reservation (cost must be 0), keyed
|
||||
// by its own derived request_hash (namespaced, collision-free with real attempt hashes), and is
|
||||
// idempotent (ON CONFLICT DO NOTHING) so a re-run re-derives the identical row for free. Escalation
|
||||
// is deliberately false so it never counts toward escalation.budget_usd. It is deleted with its
|
||||
// stage's real checkpoints on `redrive` (ResetChunkStages joins by job/chunk/stage), never orphaned.
|
||||
func (s *Store) PutDerivedCheckpoint(cp Checkpoint) error {
|
||||
ctx, cancel := opContext()
|
||||
defer cancel()
|
||||
_, err := s.w.ExecContext(ctx, `
|
||||
INSERT INTO checkpoints (
|
||||
request_hash, job_id, chunk_idx, attempt, stage, role,
|
||||
model_requested, model_actual, response_text, usage_json,
|
||||
cost_usd, finish_reason, provider_request_id, escalation
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, '', 0)
|
||||
ON CONFLICT (request_hash) DO NOTHING`,
|
||||
cp.RequestHash, cp.JobID, cp.ChunkIdx, cp.Attempt, cp.Stage, cp.Role,
|
||||
cp.ModelRequested, cp.ModelActual, cp.ResponseText, cp.UsageJSON, cp.FinishReason)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store: derived checkpoint insert: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetCheckpoint returns the persisted response for a request hash, if any —
|
||||
// the resume path: a hit means the call is NOT repeated or re-billed.
|
||||
func (s *Store) GetCheckpoint(requestHash string) (*Checkpoint, error) {
|
||||
|
|
|
|||
|
|
@ -73,6 +73,41 @@ func TestReserveSettleLifecycle(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestPutDerivedCheckpoint pins the $0 sanitized-export checkpoint (D38 infra-pack): it persists a
|
||||
// retrievable response, books NO spend, and is idempotent (re-derive on resume is free).
|
||||
func TestPutDerivedCheckpoint(t *testing.T) {
|
||||
s, _ := openTemp(t)
|
||||
job := mustSnapshotAndJob(t, s)
|
||||
|
||||
cp := Checkpoint{RequestHash: "tm-sanitized-v1:abc", JobID: job.ID, ChunkIdx: 0, Stage: "edit",
|
||||
Role: "editor", ModelRequested: "m", ModelActual: "m", ResponseText: "очищенный текст",
|
||||
UsageJSON: "{}", FinishReason: "sanitized_export"}
|
||||
if err := s.PutDerivedCheckpoint(cp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Retrievable via the standard checkpoint read (the final_hash→checkpoint export contract).
|
||||
got, err := s.GetCheckpoint("tm-sanitized-v1:abc")
|
||||
if err != nil || got == nil || got.ResponseText != "очищенный текст" {
|
||||
t.Fatalf("derived checkpoint: %+v err=%v", got, err)
|
||||
}
|
||||
if got.CostUSD != 0 {
|
||||
t.Fatalf("derived checkpoint must be $0, got %v", got.CostUSD)
|
||||
}
|
||||
// No spend booked (it is a post-processing artifact, not a provider call).
|
||||
committed, reserved, _ := s.SpentUSD("book")
|
||||
if committed != 0 || reserved != 0 {
|
||||
t.Fatalf("derived checkpoint booked spend: committed=%v reserved=%v", committed, reserved)
|
||||
}
|
||||
// Idempotent: a resume re-derives the identical row for free (ON CONFLICT DO NOTHING).
|
||||
if err := s.PutDerivedCheckpoint(cp); err != nil {
|
||||
t.Fatalf("re-put must be idempotent: %v", err)
|
||||
}
|
||||
committed, _, _ = s.SpentUSD("book")
|
||||
if committed != 0 {
|
||||
t.Fatalf("idempotent re-put booked spend: committed=%v", committed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCeilingsDeny(t *testing.T) {
|
||||
s, _ := openTemp(t)
|
||||
mustSnapshotAndJob(t, s)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue