Land the readability infra-pack (D38): cosmetic-class strip-and-export for flagged chunks, a CJK-leak sanitizer class, and an opt-in post-reflow number/omission regression guard
This commit is contained in:
parent
b14ba6f20c
commit
d5beefc8f4
19 changed files with 929 additions and 139 deletions
|
|
@ -26,9 +26,16 @@ import (
|
||||||
func renderTranslate(w io.Writer, res *pipeline.BookResult, ledger func() (committed, reserved float64, err error)) error {
|
func renderTranslate(w io.Writer, res *pipeline.BookResult, ledger func() (committed, reserved float64, err error)) error {
|
||||||
for _, ch := range res.Chunks {
|
for _, ch := range res.Chunks {
|
||||||
fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — %s%s ===\n", ch.Chapter, ch.ChunkIdx, ch.Disposition, flagSuffix(ch.FlagReason))
|
fmt.Fprintf(w, "=== ГЛАВА %d ЧАНК %d — %s%s ===\n", ch.Chapter, ch.ChunkIdx, ch.Disposition, flagSuffix(ch.FlagReason))
|
||||||
if ch.Disposition == pipeline.DispOK {
|
switch {
|
||||||
|
case ch.Disposition == pipeline.DispOK:
|
||||||
fmt.Fprintln(w, ch.FinalText)
|
fmt.Fprintln(w, ch.FinalText)
|
||||||
} else {
|
case ch.FinalText != "":
|
||||||
|
// Cosmetic sanitizer strip (D35.4a): the leak was removed and the remainder exported,
|
||||||
|
// but the chunk stays flagged for a human to verify the auto-clean — not lost to an
|
||||||
|
// empty placeholder (ch5/ch20 chapter openers used to drop whole for a leading «###»).
|
||||||
|
fmt.Fprintf(w, "[ФЛАГ %s — утечка вычищена, экспортирован очищенным, проверь] ↓\n", ch.FlagReason)
|
||||||
|
fmt.Fprintln(w, ch.FinalText)
|
||||||
|
default:
|
||||||
fmt.Fprintf(w, "[ФЛАГ %s] чанк не переведён — черновик/редактура непригодны, помечен для человека\n", ch.FlagReason)
|
fmt.Fprintf(w, "[ФЛАГ %s] чанк не переведён — черновик/редактура непригодны, помечен для человека\n", ch.FlagReason)
|
||||||
}
|
}
|
||||||
for _, st := range ch.Stages {
|
for _, st := range ch.Stages {
|
||||||
|
|
@ -159,7 +166,7 @@ func renderReport(w io.Writer,
|
||||||
fmt.Fprintf(w, "%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NPostcheckMiss, rs.PostcheckDetail)
|
fmt.Fprintf(w, "%-4d %-6d %6d %s\n", rs.Chapter, rs.ChunkIdx, rs.NPostcheckMiss, rs.PostcheckDetail)
|
||||||
}
|
}
|
||||||
// Cheap style/number flaggers (observability, not gates): total + per-chunk detail.
|
// Cheap style/number flaggers (observability, not gates): total + per-chunk detail.
|
||||||
fmt.Fprintf(w, "\n=== СТИЛЬ-ГЕЙТЫ (наблюдаемость: тире-диалоги, ё, транслит-междометия, разряды 万/億) — всего %d ===\n", style)
|
fmt.Fprintf(w, "\n=== СТИЛЬ-ГЕЙТЫ (наблюдаемость: тире-диалоги, ё, транслит-междометия, разряды 万/億, reflow-регрессия) — всего %d ===\n", style)
|
||||||
stylePrinted := false
|
stylePrinted := false
|
||||||
for _, rs := range states {
|
for _, rs := range states {
|
||||||
if rs.NStyleFlags == 0 {
|
if rs.NStyleFlags == 0 {
|
||||||
|
|
|
||||||
|
|
@ -89,6 +89,19 @@ type Gates struct {
|
||||||
Coverage CoverageGate `yaml:"coverage"`
|
Coverage CoverageGate `yaml:"coverage"`
|
||||||
Glossary GlossaryGate `yaml:"glossary"`
|
Glossary GlossaryGate `yaml:"glossary"`
|
||||||
Sanitizer SanitizerGate `yaml:"sanitizer"`
|
Sanitizer SanitizerGate `yaml:"sanitizer"`
|
||||||
|
RegressionGuard RegressionGuardGate `yaml:"regression_guard"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegressionGuardGate controls the post-reflow regression guard (D38 infra-pack,
|
||||||
|
// research/18 §C#5): two deterministic OBSERVABILITY flaggers over the draft→final transform —
|
||||||
|
// a length collapse and a numeric drift — that surface a reflow that dropped content or drifted a
|
||||||
|
// number (四成四=44%→«четыре десятых»). Opt-in (default false), and by design NEVER a disposition
|
||||||
|
// change: the reflow editor legitimately restructures/merges, so a hard skip would false-flag a
|
||||||
|
// good edit — a hit is recorded in the cheap-gate observability channel (retrieval-state
|
||||||
|
// n_style_flags) and surfaced in the report, never dropping the chunk. Its thresholds are code
|
||||||
|
// consts (versioned with the cheap gates), so the gate carries only an on/off switch.
|
||||||
|
type RegressionGuardGate struct {
|
||||||
|
Enabled bool `yaml:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SanitizerGate controls the output-sanitizer (D30.3): a deterministic verdict-axis
|
// SanitizerGate controls the output-sanitizer (D30.3): a deterministic verdict-axis
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,12 @@ type StageResult struct {
|
||||||
LatencyMS int
|
LatencyMS int
|
||||||
FinishReason string
|
FinishReason string
|
||||||
Text string // the usable output (only on an ok disposition)
|
Text string // the usable output (only on an ok disposition)
|
||||||
|
// RecoveredText is the cosmetic-stripped export text (D35.4a): non-empty ONLY for a
|
||||||
|
// FlagSanitizerStripped final stage, where the sanitizer removed a leading markdown header /
|
||||||
|
// CJK-leak and committed the cleaned remainder. It carries the export forward on BOTH the
|
||||||
|
// fresh path (runStage) and resume (resumeFromChunkStatus reads the derived checkpoint), so
|
||||||
|
// translateChunk assembles the identical FinalText either way. "" for every other disposition.
|
||||||
|
RecoveredText string
|
||||||
Disposition Disposition
|
Disposition Disposition
|
||||||
FlagReason FlagReason // "" when ok
|
FlagReason FlagReason // "" when ok
|
||||||
Detail string
|
Detail string
|
||||||
|
|
@ -37,7 +43,10 @@ type ChunkOutcome struct {
|
||||||
Chapter int
|
Chapter int
|
||||||
ChunkIdx int
|
ChunkIdx int
|
||||||
Stages []StageResult
|
Stages []StageResult
|
||||||
FinalText string // the last stage's output; "" when the chunk is flagged
|
// FinalText is the exported text: the last stage's output on ok; the cosmetic-stripped
|
||||||
|
// remainder on a FlagSanitizerStripped flag (D35.4a — the chunk is exported flagged, not lost
|
||||||
|
// to an empty placeholder); "" on any other flag (the contaminated output never exports).
|
||||||
|
FinalText string
|
||||||
Disposition Disposition // ok | flagged (a chunk has no "skipped" — that is a per-later-stage state)
|
Disposition Disposition // ok | flagged (a chunk has no "skipped" — that is a per-later-stage state)
|
||||||
FlagReason FlagReason // the flagging stage's reason ("" when ok)
|
FlagReason FlagReason // the flagging stage's reason ("" when ok)
|
||||||
CostUSD float64 // THIS run's spend on this chunk
|
CostUSD float64 // THIS run's spend on this chunk
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,10 @@ const cheapGateVersion = "cheapgate-v2"
|
||||||
type cheapGateConfig struct {
|
type cheapGateConfig struct {
|
||||||
yoPolicy string // "auto" (inconsistency only) | "all-yo" | "all-e"
|
yoPolicy string // "auto" (inconsistency only) | "all-yo" | "all-e"
|
||||||
allowlist map[string]bool // lower-cased surfaces exempt from the interjection blocklist
|
allowlist map[string]bool // lower-cased surfaces exempt from the interjection blocklist
|
||||||
|
// regressionEnabled turns on the post-reflow regression guard (D38, regressionguard.go): an
|
||||||
|
// OPT-IN observability flagger (draft→final length collapse + number drift) folded into this
|
||||||
|
// result. Off → the two regression fields stay 0 and the output is byte-identical to before.
|
||||||
|
regressionEnabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// cheapGateResult is the per-chunk outcome: a count per flagger plus human-readable detail lines
|
// cheapGateResult is the per-chunk outcome: a count per flagger plus human-readable detail lines
|
||||||
|
|
@ -55,15 +59,22 @@ type cheapGateResult struct {
|
||||||
YoInconsistent int `json:"yo,omitempty"`
|
YoInconsistent int `json:"yo,omitempty"`
|
||||||
TranslitInterj int `json:"translit_interj,omitempty"`
|
TranslitInterj int `json:"translit_interj,omitempty"`
|
||||||
NumberMagnitude int `json:"number_magnitude,omitempty"`
|
NumberMagnitude int `json:"number_magnitude,omitempty"`
|
||||||
|
// LengthCollapse / NumberDrift are the opt-in post-reflow regression guard (D38,
|
||||||
|
// regressionguard.go), folded into this observability result. They stay 0 unless
|
||||||
|
// cfg.regressionEnabled, so a book that does not enable the guard serialises identically.
|
||||||
|
LengthCollapse int `json:"length_collapse,omitempty"`
|
||||||
|
NumberDrift int `json:"number_drift,omitempty"`
|
||||||
Detail []string `json:"detail,omitempty"`
|
Detail []string `json:"detail,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c cheapGateResult) total() int {
|
func (c cheapGateResult) total() int {
|
||||||
return c.DialogueDash + c.YoInconsistent + c.TranslitInterj + c.NumberMagnitude
|
return c.DialogueDash + c.YoInconsistent + c.TranslitInterj + c.NumberMagnitude + c.LengthCollapse + c.NumberDrift
|
||||||
}
|
}
|
||||||
|
|
||||||
// runCheapGates runs all four flaggers over one chunk's source and FINAL translated text.
|
// runCheapGates runs the four always-on style flaggers over one chunk's source and FINAL text, plus
|
||||||
func runCheapGates(source, final string, cfg cheapGateConfig) cheapGateResult {
|
// (opt-in) the draft→final regression guard. `draft` is the first-stage translator output (== final
|
||||||
|
// when there is no distinct reflow stage, so the guard then trivially never fires).
|
||||||
|
func runCheapGates(source, draft, final string, cfg cheapGateConfig) cheapGateResult {
|
||||||
var r cheapGateResult
|
var r cheapGateResult
|
||||||
n, det := lintDialogueDash(final)
|
n, det := lintDialogueDash(final)
|
||||||
r.DialogueDash, r.Detail = n, append(r.Detail, det...)
|
r.DialogueDash, r.Detail = n, append(r.Detail, det...)
|
||||||
|
|
@ -76,6 +87,12 @@ func runCheapGates(source, final string, cfg cheapGateConfig) cheapGateResult {
|
||||||
n, det = lintNumberMagnitude(source, final)
|
n, det = lintNumberMagnitude(source, final)
|
||||||
r.NumberMagnitude = n
|
r.NumberMagnitude = n
|
||||||
r.Detail = append(r.Detail, det...)
|
r.Detail = append(r.Detail, det...)
|
||||||
|
if cfg.regressionEnabled {
|
||||||
|
rg := runRegressionGuard(draft, final)
|
||||||
|
r.LengthCollapse = rg.LengthCollapse
|
||||||
|
r.NumberDrift = rg.NumberDrift
|
||||||
|
r.Detail = append(r.Detail, rg.Detail...)
|
||||||
|
}
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -168,7 +168,7 @@ func TestRunCheapGatesCombined(t *testing.T) {
|
||||||
src := "他有三万石粮食。"
|
src := "他有三万石粮食。"
|
||||||
final := "- Ара-ара, — у Пётр было три миллиона мешков. Потом Петр ушёл."
|
final := "- Ара-ара, — у Пётр было три миллиона мешков. Потом Петр ушёл."
|
||||||
cfg := cheapGateConfig{yoPolicy: "auto", allowlist: map[string]bool{}}
|
cfg := cheapGateConfig{yoPolicy: "auto", allowlist: map[string]bool{}}
|
||||||
r := runCheapGates(src, final, cfg)
|
r := runCheapGates(src, final, final, cfg)
|
||||||
if r.DialogueDash == 0 {
|
if r.DialogueDash == 0 {
|
||||||
t.Error("expected a dialogue-dash flag (hyphen-led line)")
|
t.Error("expected a dialogue-dash flag (hyphen-led line)")
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
"textmachine/backend/internal/store"
|
"textmachine/backend/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -39,6 +40,17 @@ func (r *Runner) classifyOutput(role, source, output, finish string, isFinal boo
|
||||||
// its rule version folds into the snapshot only when enabled (sanitizerSnapshot).
|
// its rule version folds into the snapshot only when enabled (sanitizerSnapshot).
|
||||||
if isFinal && r.Pipeline.Gates.Sanitizer.Enabled {
|
if isFinal && r.Pipeline.Gates.Sanitizer.Enabled {
|
||||||
if san := sanitizeOutput(output); san.total() > 0 {
|
if san := sanitizeOutput(output); san.total() > 0 {
|
||||||
|
// Cosmetic-only leak (leading markdown header and/or CJK-leak) → strip it and export
|
||||||
|
// the remainder flagged, not drop the whole chunk (D35.4a). The guard is load-bearing:
|
||||||
|
// we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that is now
|
||||||
|
// clean, so runStage/resume can trust that the derived export text is usable; anything
|
||||||
|
// else falls through to the substantive skip. Both are pure over `output`, so a resumed
|
||||||
|
// checkpoint re-derives the identical verdict.
|
||||||
|
if san.cosmeticOnly() {
|
||||||
|
if stripped := stripCosmetic(output); strings.TrimSpace(stripped) != "" && sanitizeOutput(stripped).total() == 0 {
|
||||||
|
return classification{FlagSanitizerStripped, san.summary()}
|
||||||
|
}
|
||||||
|
}
|
||||||
return classification{FlagSanitizerDefect, san.summary()}
|
return classification{FlagSanitizerDefect, san.summary()}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -73,6 +85,14 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
||||||
prev := ""
|
prev := ""
|
||||||
flagged := false
|
flagged := false
|
||||||
var flagReason FlagReason
|
var flagReason FlagReason
|
||||||
|
// recovered carries a cosmetic sanitizer strip's cleaned export forward from the flagging
|
||||||
|
// stage (D35.4a): non-empty only when the FINAL stage flagged FlagSanitizerStripped, "" for a
|
||||||
|
// dropped substantive flag. Captured from the FIRST flagging stage (the sanitizer flags only
|
||||||
|
// the final stage, so an upstream substantive flag leaves it "" and the chunk exports empty).
|
||||||
|
recovered := ""
|
||||||
|
// draftText is the FIRST (translator) stage's output — the pre-reflow baseline for the opt-in
|
||||||
|
// regression guard's draft→final comparison (== final in a single-stage pipeline).
|
||||||
|
draftText := ""
|
||||||
|
|
||||||
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and
|
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and
|
||||||
// serialize the translator injection block. Recomputed on every run (resumed chunks
|
// serialize the translator injection block. Recomputed on every run (resumed chunks
|
||||||
|
|
@ -127,9 +147,13 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
||||||
if sr.Disposition == DispFlagged {
|
if sr.Disposition == DispFlagged {
|
||||||
flagged = true
|
flagged = true
|
||||||
flagReason = sr.FlagReason
|
flagReason = sr.FlagReason
|
||||||
|
recovered = sr.RecoveredText
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
prev = sr.Text
|
prev = sr.Text
|
||||||
|
if stageIdx == 0 {
|
||||||
|
draftText = sr.Text // the translator draft, before any reflow (regression-guard baseline)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Post-check the FINAL, exported output (E1): the reader sees the last stage's text
|
// Post-check the FINAL, exported output (E1): the reader sees the last stage's text
|
||||||
|
|
@ -161,7 +185,7 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
||||||
// (the 万/億 magnitude gate compares source↔output).
|
// (the 万/億 magnitude gate compares source↔output).
|
||||||
var cheap cheapGateResult
|
var cheap cheapGateResult
|
||||||
if !flagged && prev != "" {
|
if !flagged && prev != "" {
|
||||||
cheap = runCheapGates(ch.Text, prev, r.cheapGateConfig())
|
cheap = runCheapGates(ch.Text, draftText, prev, r.cheapGateConfig())
|
||||||
if cheap.total() > 0 {
|
if cheap.total() > 0 {
|
||||||
r.Log.InfoContext(ctx, "cheap style gates flagged the chunk (observability, not a gate)",
|
r.Log.InfoContext(ctx, "cheap style gates flagged the chunk (observability, not a gate)",
|
||||||
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "style_flags", cheap.total(),
|
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "style_flags", cheap.total(),
|
||||||
|
|
@ -183,7 +207,10 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
||||||
if flagged {
|
if flagged {
|
||||||
out.Disposition = DispFlagged
|
out.Disposition = DispFlagged
|
||||||
out.FlagReason = flagReason
|
out.FlagReason = flagReason
|
||||||
out.FinalText = "" // garbage/refusal/glossary-miss never propagates to the next chunk or export
|
// A cosmetic sanitizer strip exports its cleaned remainder (recovered != ""); every other
|
||||||
|
// flag (garbage/refusal/glossary-miss/substantive-sanitizer) exports "" — the contaminated
|
||||||
|
// output never reaches TM/export (D2 / D35.4a).
|
||||||
|
out.FinalText = recovered
|
||||||
} else {
|
} else {
|
||||||
out.FinalText = prev
|
out.FinalText = prev
|
||||||
}
|
}
|
||||||
|
|
@ -247,5 +274,9 @@ func (r *Runner) cheapGateConfig() cheapGateConfig {
|
||||||
for _, s := range r.Book.StyleAllowlist {
|
for _, s := range r.Book.StyleAllowlist {
|
||||||
allow[s] = true
|
allow[s] = true
|
||||||
}
|
}
|
||||||
return cheapGateConfig{yoPolicy: r.Book.YoPolicy, allowlist: allow}
|
return cheapGateConfig{
|
||||||
|
yoPolicy: r.Book.YoPolicy,
|
||||||
|
allowlist: allow,
|
||||||
|
regressionEnabled: r.Pipeline.Gates.RegressionGuard.Enabled,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -82,14 +82,24 @@ const (
|
||||||
// escalatable in v1 (the L3 targeted re-ask is a Phase-2 remedy, research/14 §9).
|
// escalatable in v1 (the L3 targeted re-ask is a Phase-2 remedy, research/14 §9).
|
||||||
FlagGlossaryMiss FlagReason = "glossary_miss"
|
FlagGlossaryMiss FlagReason = "glossary_miss"
|
||||||
|
|
||||||
// FlagSanitizerDefect is the output-sanitizer verdict (D30.3, sanitizer.go): the stage
|
// FlagSanitizerDefect is the output-sanitizer verdict for a SUBSTANTIVE defect (D30.3,
|
||||||
// output carries an "instant unreadability" defect no other gate catches — a leaked
|
// sanitizer.go): the stage output carries an "instant unreadability" defect with no reliable
|
||||||
// service preamble, a trailing note/edit block, a markdown ### header, a Latin-script
|
// removable boundary — a leaked service preamble, a trailing note/edit block, a Latin-script
|
||||||
// insertion, or a broken word form. Emitted ONLY when the opt-in Gates.Sanitizer is
|
// insertion, or a broken word form. Emitted ONLY when the opt-in Gates.Sanitizer is enabled;
|
||||||
// enabled; the contaminated output is flagged (D2 flag+skip) so it never commits to
|
// the contaminated output is flagged AND skipped (D2 flag+skip) so it never commits to
|
||||||
// TM/export. NOT retryable (a same-model retry re-produces the artifact) and NOT
|
// TM/export (FinalText stays ""). NOT retryable (a same-model retry re-produces the artifact)
|
||||||
// escalatable in v1 (the editor is pinned; a redrive surfaces the defect to a human).
|
// and NOT escalatable in v1 (the editor is pinned; a redrive surfaces the defect to a human).
|
||||||
FlagSanitizerDefect FlagReason = "sanitizer_defect"
|
FlagSanitizerDefect FlagReason = "sanitizer_defect"
|
||||||
|
|
||||||
|
// FlagSanitizerStripped is the output-sanitizer verdict for a COSMETIC-ONLY defect that was
|
||||||
|
// STRIPPED and the remainder EXPORTED (D38 infra-pack, D35.4a): a leading markdown «### Глава»
|
||||||
|
// header and/or a stray CJK-leak run, removed deterministically by stripCosmetic. Unlike
|
||||||
|
// FlagSanitizerDefect the chunk is NOT lost — the cleaned text is committed as the export
|
||||||
|
// (final_hash points at a derived sanitized checkpoint, so the standard
|
||||||
|
// final_hash→checkpoint.response_text export contract yields the cleaned text) — but it STAYS
|
||||||
|
// flagged so a human verifies the auto-clean ("не терять чанк ... флаг для человека"). NOT
|
||||||
|
// retryable / NOT escalatable (deterministic; a redrive would re-produce the same leak).
|
||||||
|
FlagSanitizerStripped FlagReason = "sanitizer_stripped"
|
||||||
)
|
)
|
||||||
|
|
||||||
// classifierVersion versions the INTRINSIC classify() verdict logic — the refusal
|
// classifierVersion versions the INTRINSIC classify() verdict logic — the refusal
|
||||||
|
|
|
||||||
|
|
@ -41,8 +41,15 @@ import (
|
||||||
// • UNION: ch5 fires an OBLIQUE-ONLY decl term (老人→старик) whose FINAL text carries the
|
// • UNION: ch5 fires an OBLIQUE-ONLY decl term (老人→старик) whose FINAL text carries the
|
||||||
// NOMINATIVE «старик» → postcheck_miss=0 only because the base-dst∪decl union checks the
|
// NOMINATIVE «старик» → postcheck_miss=0 only because the base-dst∪decl union checks the
|
||||||
// base (revert the union → this chunk false-flags a miss);
|
// base (revert the union → this chunk false-flags a miss);
|
||||||
// • SANITIZER: ch6 draft leaks a service preamble → the output-sanitizer gate (ON in the
|
// • SANITIZER (SUBSTANTIVE): ch6 edit leaks a service preamble → the output-sanitizer gate (ON
|
||||||
// fixture pipeline) flags it FlagSanitizerDefect and the edit is skipped.
|
// in the fixture pipeline) flags it FlagSanitizerDefect and the edit is DROPPED (final_text "").
|
||||||
|
// D38 infra-pack cells (pin the cosmetic strip-and-export path, D35.4a — final_hash points at a
|
||||||
|
// derived $0 sanitized checkpoint so the export contract yields the cleaned text; resume re-serves
|
||||||
|
// it identically at $0):
|
||||||
|
// • SANITIZER (COSMETIC markdown): ch7 edit leaks a leading «### Глава 7» → flagged
|
||||||
|
// sanitizer_stripped, final_text = «Глава 7\n\n…» (the «### » stripped), a derived checkpoint holds it;
|
||||||
|
// • SANITIZER (COSMETIC CJK): ch8 edit leaks a sparse «特产» → flagged sanitizer_stripped, final_text
|
||||||
|
// = the run removed and the seam tidied.
|
||||||
// A cell with AMBIGUOUS>0 (an auto/draft candidate forcing a post-check) remains an OPTIONAL
|
// A cell with AMBIGUOUS>0 (an auto/draft candidate forcing a post-check) remains an OPTIONAL
|
||||||
// future extension, deliberately NOT added here.
|
// future extension, deliberately NOT added here.
|
||||||
//
|
//
|
||||||
|
|
@ -57,7 +64,9 @@ const (
|
||||||
goldenEchoMarker = "響動計画"
|
goldenEchoMarker = "響動計画"
|
||||||
goldenRefusalMarker = "拒絶計画"
|
goldenRefusalMarker = "拒絶計画"
|
||||||
goldenUnionMarker = "老人の秘密" // ch5 — oblique-only decl union case
|
goldenUnionMarker = "老人の秘密" // ch5 — oblique-only decl union case
|
||||||
goldenSanitizerMarker = "序文漏洩" // ch6 — sanitizer preamble-leak case
|
goldenSanitizerMarker = "序文漏洩" // ch6 — sanitizer preamble-leak case (SUBSTANTIVE → dropped)
|
||||||
|
goldenMarkdownMarker = "見出漏洩" // ch7 — cosmetic markdown «### Глава» leak (STRIPPED + exported)
|
||||||
|
goldenCJKMarker = "漢字漏洩" // ch8 — cosmetic CJK-leak «特产» (STRIPPED + exported)
|
||||||
)
|
)
|
||||||
|
|
||||||
// goldenRespond is the deterministic mock provider brain. The response text is a pure
|
// goldenRespond is the deterministic mock provider brain. The response text is a pure
|
||||||
|
|
@ -85,6 +94,12 @@ func goldenRespond(body string) (text, finish string) {
|
||||||
case strings.Contains(body, goldenUnionMarker) && !isEditBody(body):
|
case strings.Contains(body, goldenUnionMarker) && !isEditBody(body):
|
||||||
// ch5 draft carries a keying token («СТАРИК-СЦЕНА») the edit branch below keys on.
|
// ch5 draft carries a keying token («СТАРИК-СЦЕНА») the edit branch below keys on.
|
||||||
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". СТАРИК-СЦЕНА Судзуки увидел старика.", "stop"
|
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". СТАРИК-СЦЕНА Судзуки увидел старика.", "stop"
|
||||||
|
case strings.Contains(body, goldenMarkdownMarker) && !isEditBody(body):
|
||||||
|
// ch7 draft carries a keying token («МАРКДАУН-СЦЕНА») the cosmetic edit branch keys on.
|
||||||
|
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". МАРКДАУН-СЦЕНА Судзуки открыл седьмую дверь.", "stop"
|
||||||
|
case strings.Contains(body, goldenCJKMarker) && !isEditBody(body):
|
||||||
|
// ch8 draft carries a keying token («ИЕРОГЛИФ-СЦЕНА») the cosmetic edit branch keys on.
|
||||||
|
return "ЧЕРНОВИК ПЕРЕВОДА " + tag + ". ИЕРОГЛИФ-СЦЕНА Судзуки нашёл камень.", "stop"
|
||||||
case strings.Contains(body, goldenEchoMarker) && !isEditBody(body):
|
case strings.Contains(body, goldenEchoMarker) && !isEditBody(body):
|
||||||
if req.Model == "fake-fallback" {
|
if req.Model == "fake-fallback" {
|
||||||
// The escalation hop returns a clean translation → the re-gate passes it.
|
// The escalation hop returns a clean translation → the re-gate passes it.
|
||||||
|
|
@ -100,6 +115,16 @@ func goldenRespond(body string) (text, finish string) {
|
||||||
// ch5 edit — the FINAL text carries the NOMINATIVE «старик» (not an oblique decl
|
// ch5 edit — the FINAL text carries the NOMINATIVE «старик» (not an oblique decl
|
||||||
// form of 老人→старик), exercising the base-dst∪decl union → postcheck_miss=0.
|
// form of 老人→старик), exercising the base-dst∪decl union → postcheck_miss=0.
|
||||||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки увидел, как в тени ждал старик.", "stop"
|
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки увидел, как в тени ждал старик.", "stop"
|
||||||
|
case isEditBody(body) && strings.Contains(body, "МАРКДАУН-СЦЕНА"):
|
||||||
|
// ch7 edit leaks a leading markdown «### Глава» header — a COSMETIC class: the sanitizer
|
||||||
|
// STRIPS it and exports the remainder flagged sanitizer_stripped (D35.4a). This PINS the
|
||||||
|
// strip-and-export path: final_text = «Глава 7\n\n…» (no «### »), disposition=flagged.
|
||||||
|
return "### Глава 7\n\nСудзуки открыл седьмую дверь книгохранилища.", "stop"
|
||||||
|
case isEditBody(body) && strings.Contains(body, "ИЕРОГЛИФ-СЦЕНА"):
|
||||||
|
// ch8 edit leaks a sparse CJK run «特产» in otherwise-Russian prose (below the 15% echo
|
||||||
|
// threshold classify() catches) — a COSMETIC class: stripped + exported flagged. PINS the
|
||||||
|
// CJK-leak strip: «Судзуки нашёл 特产 древний камень.» → «Судзуки нашёл древний камень.»
|
||||||
|
return "Судзуки нашёл 特产 древний камень на каменном алтаре.", "stop"
|
||||||
case isEditBody(body):
|
case isEditBody(body):
|
||||||
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки шёл по коридорам Академии магии.", "stop"
|
return "ОТРЕДАКТИРОВАННЫЙ ПЕРЕВОД " + tag + ". Судзуки шёл по коридорам Академии магии.", "stop"
|
||||||
default:
|
default:
|
||||||
|
|
@ -171,7 +196,7 @@ func captureGolden(t *testing.T, label string, r *Runner, res *BookResult, wireB
|
||||||
w(" stage=%s role=%s model=%s resume=%t disp=%s flag=%q attempts=%d escalated=%t esc_model=%q finish=%q cum_usd=%s detail=%q",
|
w(" stage=%s role=%s model=%s resume=%t disp=%s flag=%q attempts=%d escalated=%t esc_model=%q finish=%q cum_usd=%s detail=%q",
|
||||||
st.Stage, st.Role, st.Model, st.FromResume, st.Disposition, st.FlagReason,
|
st.Stage, st.Role, st.Model, st.FromResume, st.Disposition, st.FlagReason,
|
||||||
st.Attempts, st.Escalated, st.EscalationModel, st.FinishReason, fl(st.CumCostUSD), st.Detail)
|
st.Attempts, st.Escalated, st.EscalationModel, st.FinishReason, fl(st.CumCostUSD), st.Detail)
|
||||||
w(" stage_text=%q", st.Text)
|
w(" stage_text=%q recovered=%q", st.Text, st.RecoveredText)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
159
backend/internal/pipeline/regressionguard.go
Normal file
159
backend/internal/pipeline/regressionguard.go
Normal file
|
|
@ -0,0 +1,159 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// regressionguard.go: the post-reflow regression guard (D38 infra-pack, research/18 §C#5) — two
|
||||||
|
// deterministic OBSERVABILITY flaggers over the DRAFT→FINAL transform, opt-in like a QA gate
|
||||||
|
// (Gates.RegressionGuard) but NEVER a disposition change. The reflow editor legitimately
|
||||||
|
// restructures and merges sentences, so a HARD skip would false-flag a correct edit; instead a hit
|
||||||
|
// is recorded in the cheap-gate observability channel (retrieval-state n_style_flags) and surfaced
|
||||||
|
// in the report for a human. Both flaggers are tuned PRECISION over recall.
|
||||||
|
//
|
||||||
|
// (a) length collapse — the final is drastically SHORTER than the draft (>regressionMaxShrinkPct
|
||||||
|
// of non-space chars), catching a reflow that dropped whole sentences/paragraphs (draft
|
||||||
|
// 5000 → final 600). Reflow SUMMARISES structure into paragraphs; it must not delete content.
|
||||||
|
// (b) number drift — a MULTI-DIGIT arabic number in the draft is ABSENT from the final, or a
|
||||||
|
// new one APPEARS: 四成四=44%→«четыре десятых» (the 44 vanished), or a hallucinated figure.
|
||||||
|
// Single digits are EXCLUDED (they are routinely spelled out — «5»→«пять» — a legit reflow),
|
||||||
|
// and grouped triples are stitched (10 000 ≡ 10000), so the signal is high precision.
|
||||||
|
//
|
||||||
|
// HONEST recall gap: a number rendered as WORDS on BOTH sides (черновик «сорок четыре» → финал
|
||||||
|
// «четыре десятых») is invisible here — offline word-number alignment is a Ф2 concern. And a
|
||||||
|
// legit reflow that converts a multi-digit figure to words («44%»→«сорок четыре процента») will
|
||||||
|
// show as a disappeared number: a FALSE observability flag, tolerable ONLY because this never
|
||||||
|
// drops the chunk (a human glance dismisses it). Precision over recall, observability over gating.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// regressionMaxShrinkPct: a final shorter than the draft by MORE than this percent of non-space
|
||||||
|
// characters is a collapse signal. 40% — a reflow that keeps content while merging lines shrinks
|
||||||
|
// only slightly (Russian is ~fertility-neutral draft→final); losing ~half the characters is a
|
||||||
|
// strong omission signal, not a merge. Tuned precision-over-recall; a code const (a change is a
|
||||||
|
// loud cheapGateVersion bump), not config, so the gate carries only an on/off switch.
|
||||||
|
regressionMaxShrinkPct = 40
|
||||||
|
// regressionMinChars: below this the draft is too short for a percentage ratio to be meaningful
|
||||||
|
// (a 3-word draft legitimately halving is noise), so the length flagger stays silent.
|
||||||
|
regressionMinChars = 200
|
||||||
|
)
|
||||||
|
|
||||||
|
// regressionGuardResult is the guard's per-chunk outcome (folded into the cheap-gate observability
|
||||||
|
// result). LengthCollapse/NumberDrift are 0/1 counts; Detail carries the human-readable lines.
|
||||||
|
type regressionGuardResult struct {
|
||||||
|
LengthCollapse int
|
||||||
|
NumberDrift int
|
||||||
|
Detail []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// runRegressionGuard compares the DRAFT (first-stage translator output) with the FINAL (last-stage
|
||||||
|
// reflow output). Pure and deterministic (sorted detail), so a resume re-derives identical counts.
|
||||||
|
func runRegressionGuard(draft, final string) regressionGuardResult {
|
||||||
|
var r regressionGuardResult
|
||||||
|
|
||||||
|
// (a) length collapse — non-space chars, mirroring the coverage gate's length metric.
|
||||||
|
dn, fn := nonSpaceRuneCount(draft), nonSpaceRuneCount(final)
|
||||||
|
if dn >= regressionMinChars && fn*100 < dn*(100-regressionMaxShrinkPct) {
|
||||||
|
r.LengthCollapse = 1
|
||||||
|
r.Detail = append(r.Detail, fmt.Sprintf(
|
||||||
|
"reflow-регрессия: коллапс длины черновик→финал %d→%d непробельных символов (−%d%%, порог −%d%%) — возможен пропуск",
|
||||||
|
dn, fn, 100-fn*100/dn, regressionMaxShrinkPct))
|
||||||
|
}
|
||||||
|
|
||||||
|
// (b) number drift — multi-digit arabic tokens present on one side only.
|
||||||
|
dNums, fNums := arabicNumberTokens(draft), arabicNumberTokens(final)
|
||||||
|
disappeared := numberSetDiff(dNums, fNums)
|
||||||
|
appeared := numberSetDiff(fNums, dNums)
|
||||||
|
if len(disappeared) > 0 || len(appeared) > 0 {
|
||||||
|
r.NumberDrift = len(disappeared) + len(appeared)
|
||||||
|
if len(disappeared) > 0 {
|
||||||
|
r.Detail = append(r.Detail, "reflow-регрессия: числа черновика отсутствуют в финале (возможен пропуск/дрейф разряда): "+joinPreview(disappeared))
|
||||||
|
}
|
||||||
|
if len(appeared) > 0 {
|
||||||
|
r.Detail = append(r.Detail, "reflow-регрессия: в финале появились числа, которых не было в черновике (возможна добавка): "+joinPreview(appeared))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// nonSpaceRuneCount counts non-whitespace runes — the length metric for the collapse flagger.
|
||||||
|
func nonSpaceRuneCount(s string) int {
|
||||||
|
n := 0
|
||||||
|
for _, r := range s {
|
||||||
|
if !unicode.IsSpace(r) {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// arabicNumberTokens returns the MULTI-DIGIT (≥2) arabic integers in text, stitching grouping
|
||||||
|
// separators (space / NBSP / comma) between runs of exactly three digits so "10 000" reads as one
|
||||||
|
// "10000", not "10" and "000". Single-digit numbers are dropped (routinely spelled out in prose).
|
||||||
|
func arabicNumberTokens(text string) []string {
|
||||||
|
rs := []rune(text)
|
||||||
|
var out []string
|
||||||
|
for i := 0; i < len(rs); {
|
||||||
|
if !isASCIIDigit(rs[i]) {
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
j := i
|
||||||
|
for j < len(rs) && isASCIIDigit(rs[j]) {
|
||||||
|
j++
|
||||||
|
}
|
||||||
|
digits := string(rs[i:j])
|
||||||
|
// Stitch " ddd" / " ddd" / ",ddd" grouped triples.
|
||||||
|
for j < len(rs) {
|
||||||
|
if rs[j] == ' ' || rs[j] == ' ' || rs[j] == ',' {
|
||||||
|
k := j + 1
|
||||||
|
g := 0
|
||||||
|
for k < len(rs) && isASCIIDigit(rs[k]) {
|
||||||
|
k++
|
||||||
|
g++
|
||||||
|
}
|
||||||
|
if g == 3 {
|
||||||
|
digits += string(rs[j+1 : k])
|
||||||
|
j = k
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if len(digits) >= 2 {
|
||||||
|
out = append(out, digits)
|
||||||
|
}
|
||||||
|
i = j
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// numberSetDiff returns the distinct members of a that do not appear in b, sorted. A SET diff (not
|
||||||
|
// a multiset): a number present on both sides is covered even if its multiplicity differs, keeping
|
||||||
|
// the precision-over-recall bias (a repeated number is not a drift signal on its own).
|
||||||
|
func numberSetDiff(a, b []string) []string {
|
||||||
|
inB := make(map[string]bool, len(b))
|
||||||
|
for _, x := range b {
|
||||||
|
inB[x] = true
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, x := range a {
|
||||||
|
if !inB[x] && !seen[x] {
|
||||||
|
seen[x] = true
|
||||||
|
out = append(out, x)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// joinPreview renders a sorted number list compactly for the detail line (bounded).
|
||||||
|
func joinPreview(nums []string) string {
|
||||||
|
const max = 8
|
||||||
|
if len(nums) > max {
|
||||||
|
return preview(fmt.Sprintf("%v …(+%d)", nums[:max], len(nums)-max))
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%v", nums)
|
||||||
|
}
|
||||||
82
backend/internal/pipeline/regressionguard_test.go
Normal file
82
backend/internal/pipeline/regressionguard_test.go
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// regressionguard_test.go — the post-reflow regression guard (D38). Precision over recall: each
|
||||||
|
// flagger asserts BOTH firing on a real regression AND non-firing on a legitimate reflow.
|
||||||
|
|
||||||
|
func TestRegressionGuardLengthCollapse(t *testing.T) {
|
||||||
|
// A draft well over the min-chars floor, halved by the "reflow" → collapse fires.
|
||||||
|
draft := strings.Repeat("Судзуки медленно шёл по коридору академии магии. ", 12) // ~560 non-space chars
|
||||||
|
shortFinal := "Судзуки шёл."
|
||||||
|
if r := runRegressionGuard(draft, shortFinal); r.LengthCollapse == 0 {
|
||||||
|
t.Errorf("expected a length-collapse flag (draft≫final)")
|
||||||
|
}
|
||||||
|
// A legitimate reflow that merges lines keeps ~all the content → must NOT fire.
|
||||||
|
sameFinal := strings.Repeat("Судзуки медленно шёл по коридору академии магии. ", 11)
|
||||||
|
if r := runRegressionGuard(draft, sameFinal); r.LengthCollapse != 0 {
|
||||||
|
t.Errorf("length-collapse FALSE POSITIVE on a near-equal reflow: %v", r.Detail)
|
||||||
|
}
|
||||||
|
// A short draft below the floor must never fire (a % ratio is meaningless there).
|
||||||
|
if r := runRegressionGuard("Он ушёл.", "Он."); r.LengthCollapse != 0 {
|
||||||
|
t.Errorf("length-collapse fired below the min-chars floor: %v", r.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegressionGuardNumberDrift(t *testing.T) {
|
||||||
|
// 四成四=44% rendered as «44» in the draft, drifted to a word form in the final → 44 disappears.
|
||||||
|
if r := runRegressionGuard("У него было 44 процента запаса.", "У него было четыре десятых запаса."); r.NumberDrift == 0 {
|
||||||
|
t.Errorf("expected a number-drift flag (44 vanished)")
|
||||||
|
}
|
||||||
|
// A number APPEARING in the final that was not in the draft (hallucinated figure).
|
||||||
|
if r := runRegressionGuard("Он собрал войско.", "Он собрал войско из 3000 солдат."); r.NumberDrift == 0 {
|
||||||
|
t.Errorf("expected a number-drift flag (3000 appeared)")
|
||||||
|
}
|
||||||
|
clean := []struct{ draft, final string }{
|
||||||
|
// The multi-digit number is preserved across the reflow (44 on both sides).
|
||||||
|
{"У него было 44 процента.", "У него имелось 44 процента запаса."},
|
||||||
|
// Grouped-triple vs joined form of the SAME number must not drift (10 000 ≡ 10000).
|
||||||
|
{"Армия в 10 000 воинов.", "Армия насчитывала 10000 воинов."},
|
||||||
|
// SINGLE digits are excluded (routinely spelled out), so «5»→«пять» is not a drift.
|
||||||
|
{"У него было 5 мечей.", "У него было пять мечей."},
|
||||||
|
// No numbers at all — never fires.
|
||||||
|
{"Судзуки шёл по коридору академии магии.", "Судзуки медленно шёл по длинному коридору."},
|
||||||
|
}
|
||||||
|
for _, c := range clean {
|
||||||
|
if r := runRegressionGuard(c.draft, c.final); r.NumberDrift != 0 {
|
||||||
|
t.Errorf("number-drift FALSE POSITIVE on %q→%q: %v", c.draft, c.final, r.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunCheapGatesRegressionOptIn pins that the guard is OFF by default and folds into the
|
||||||
|
// observability total only when enabled (a book that does not opt in serialises identically).
|
||||||
|
func TestRunCheapGatesRegressionOptIn(t *testing.T) {
|
||||||
|
draft := strings.Repeat("Судзуки медленно шёл по коридору академии магии. ", 12)
|
||||||
|
final := "Судзуки шёл."
|
||||||
|
off := runCheapGates("", draft, final, cheapGateConfig{yoPolicy: "auto", allowlist: map[string]bool{}})
|
||||||
|
if off.LengthCollapse != 0 || off.NumberDrift != 0 {
|
||||||
|
t.Errorf("regression guard fired while disabled: %+v", off)
|
||||||
|
}
|
||||||
|
on := runCheapGates("", draft, final, cheapGateConfig{yoPolicy: "auto", allowlist: map[string]bool{}, regressionEnabled: true})
|
||||||
|
if on.LengthCollapse == 0 {
|
||||||
|
t.Errorf("regression guard did not fire while enabled: %+v", on)
|
||||||
|
}
|
||||||
|
if on.total() <= off.total() {
|
||||||
|
t.Errorf("enabled guard must raise the observability total: off=%d on=%d", off.total(), on.total())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRegressionGuardDeterministic — a pure function of (draft, final), resume-safe.
|
||||||
|
func TestRegressionGuardDeterministic(t *testing.T) {
|
||||||
|
d := strings.Repeat("текст с числом 44 и 128. ", 20)
|
||||||
|
f := "короткий финал без чисел"
|
||||||
|
a, b := runRegressionGuard(d, f), runRegressionGuard(d, f)
|
||||||
|
if a.LengthCollapse != b.LengthCollapse || a.NumberDrift != b.NumberDrift ||
|
||||||
|
strings.Join(a.Detail, "|") != strings.Join(b.Detail, "|") {
|
||||||
|
t.Fatalf("runRegressionGuard not deterministic: %+v vs %+v", a, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -50,6 +50,26 @@ func (r *Runner) resumeFromChunkStatus(ctx context.Context, st config.Stage, ch
|
||||||
sr.Model = cp.ModelActual
|
sr.Model = cp.ModelActual
|
||||||
sr.FinishReason = cp.FinishReason
|
sr.FinishReason = cp.FinishReason
|
||||||
sr.Text = cp.ResponseText
|
sr.Text = cp.ResponseText
|
||||||
|
} else if cs.FlagReason == string(FlagSanitizerStripped) && cs.FinalHash != "" {
|
||||||
|
// A cosmetic sanitizer strip (D35.4a): the cleaned export text lives in the derived
|
||||||
|
// checkpoint final_hash points at (written durably before this chunk_status row), so resume
|
||||||
|
// re-serves the identical FinalText WITHOUT re-stripping — deterministic and $0. A missing
|
||||||
|
// derived checkpoint is a torn store (final_hash is written after it), so fail loud like the
|
||||||
|
// ok branch rather than silently export empty.
|
||||||
|
cp, err := r.Store.GetCheckpoint(cs.FinalHash)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("pipeline: read sanitized-export checkpoint %.12s for %s/ch%d/chunk%d/%s: %w",
|
||||||
|
cs.FinalHash, r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, err)
|
||||||
|
}
|
||||||
|
if cp == nil || cp.ResponseText == "" {
|
||||||
|
return nil, fmt.Errorf("pipeline: chunk_status sanitizer_stripped for %s/ch%d/chunk%d/%s references derived checkpoint %.12s which is missing/empty — inconsistent store",
|
||||||
|
r.Book.BookID, ch.Chapter, ch.ChunkIdx, st.Name, cs.FinalHash)
|
||||||
|
}
|
||||||
|
sr.RecoveredText = cp.ResponseText
|
||||||
|
// Mirror the ok branch: the resume request_log row's request_hash points at this derived
|
||||||
|
// checkpoint, so report ITS finish_reason ("sanitized_export") rather than a bare "" that
|
||||||
|
// contradicts the referenced row (adversarial review NIT; telemetry-only).
|
||||||
|
sr.FinishReason = cp.FinishReason
|
||||||
}
|
}
|
||||||
rl := r.baseRequestLog(st, ch, st.Model, cs.FinalHash)
|
rl := r.baseRequestLog(st, ch, st.Model, cs.FinalHash)
|
||||||
rl.ModelActual = sr.Model
|
rl.ModelActual = sr.Model
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ import (
|
||||||
// (sanitizerVersion, mirroring coverageSnapshot), and moves to verdictSnapshotID once
|
// (sanitizerVersion, mirroring coverageSnapshot), and moves to verdictSnapshotID once
|
||||||
// content-addressed resume lands (D15.2).
|
// content-addressed resume lands (D15.2).
|
||||||
//
|
//
|
||||||
// Five classes, each tuned PRECISION over recall (fire only on a high-confidence signal;
|
// Six classes, each tuned PRECISION over recall (fire only on a high-confidence signal;
|
||||||
// the ambiguous middle stays silent — a false flag turns a good chunk into an export
|
// the ambiguous middle stays silent — a false flag turns a good chunk into an export
|
||||||
// placeholder, worse than a missed defect for an opt-in readability gate):
|
// placeholder, worse than a missed defect for an opt-in readability gate):
|
||||||
//
|
//
|
||||||
|
|
@ -27,6 +27,21 @@ import (
|
||||||
// 3. markdown ### header — «### Глава 1» (8/54 finals of stage A — D29 finding)
|
// 3. markdown ### header — «### Глава 1» (8/54 finals of stage A — D29 finding)
|
||||||
// 4. Latin-script insertion — an untranslated Latin phrase/heavy Latin in the ru output
|
// 4. Latin-script insertion — an untranslated Latin phrase/heavy Latin in the ru output
|
||||||
// 5. broken word form — homoglyph-mixed token or an invalid Cyrillic sign bigram
|
// 5. broken word form — homoglyph-mixed token or an invalid Cyrillic sign bigram
|
||||||
|
// 6. CJK-leak in the final — a stray Han ideograph / kana / fullwidth glyph in the ru output
|
||||||
|
// («…охотники,却有 деньги!» arms/C/17.0 — D38.1; below the ≥15%
|
||||||
|
// echo threshold classify() already catches, so a sparse leak)
|
||||||
|
//
|
||||||
|
// DISPOSITION-BY-CLASS (D38 infra-pack): the classes split into two tiers so a chunk is not
|
||||||
|
// silently lost to an export placeholder when the leak is a REMOVABLE cosmetic span (D35.4a —
|
||||||
|
// ch5/ch20 chapter openers dropped whole for a leading «### Глава N»):
|
||||||
|
// - COSMETIC (strippable, exported + flagged for a human, cosmeticOnly()==true): the markdown
|
||||||
|
// header (3) and the CJK-leak (6). The leak sits in a bounded span stripCosmetic() removes
|
||||||
|
// deterministically, leaving readable prose; the chunk is flagged sanitizer_stripped (a
|
||||||
|
// "auto-cleaned, verify" signal), NOT dropped.
|
||||||
|
// - SUBSTANTIVE (dropped to a placeholder, current behaviour): the preamble (1), trailing
|
||||||
|
// note (2), Latin insertion (4) and broken word (5). Their contamination has no reliable
|
||||||
|
// removable boundary (a preamble may swallow real narration; a Latin clause / broken token is
|
||||||
|
// lost content), so the output is not trusted — flagged sanitizer_defect + skipped.
|
||||||
//
|
//
|
||||||
// HONEST recall gap (documented, precision over recall): the broken-word class catches only
|
// HONEST recall gap (documented, precision over recall): the broken-word class catches only
|
||||||
// the two ZERO-false-positive signatures (invalid Cyrillic sign bigrams, mixed Cyrillic/Latin
|
// the two ZERO-false-positive signatures (invalid Cyrillic sign bigrams, mixed Cyrillic/Latin
|
||||||
|
|
@ -43,7 +58,15 @@ import (
|
||||||
// v2 (D33.1/33.2): tail edit-summary «Основные правки для справки:» now flags; the
|
// v2 (D33.1/33.2): tail edit-summary «Основные правки для справки:» now flags; the
|
||||||
// junction-duplication split detector was removed (false-flagged prose); markdown/heavy-Latin
|
// junction-duplication split detector was removed (false-flagged prose); markdown/heavy-Latin
|
||||||
// narrowed. A verdict-axis change → loud --resnapshot + golden re-capture (invariant №8).
|
// narrowed. A verdict-axis change → loud --resnapshot + golden re-capture (invariant №8).
|
||||||
const sanitizerVersion = "sanitizer-v2"
|
// v3 (D38 infra-pack): added the CJK-leak class (Han/kana/fullwidth in the ru final) and the
|
||||||
|
// cosmetic-vs-substantive disposition split (the markdown header and CJK-leak are now STRIPPED
|
||||||
|
// and exported flagged, not dropped to an empty placeholder — D35.4a). Both shift the resolved
|
||||||
|
// disposition/export of a flagged chunk → a loud --resnapshot + golden re-capture (invariant №8).
|
||||||
|
// v4 (D38 infra-pack, adversarial review): stripCosmetic now FOLDS content-bearing fullwidth ASCII
|
||||||
|
// («123»→«123», not deleted) and ideographic comma/period, space-replaces dropped ideographs (no
|
||||||
|
// word-merge), removes an emptied bracket pair, and no longer reformats a legit «2 : 1» — all shift
|
||||||
|
// a stripped chunk's EXPORT text → a loud --resnapshot + golden re-capture (invariant №8).
|
||||||
|
const sanitizerVersion = "sanitizer-v4"
|
||||||
|
|
||||||
// sanitizer tuning constants (versioned by sanitizerVersion).
|
// sanitizer tuning constants (versioned by sanitizerVersion).
|
||||||
const (
|
const (
|
||||||
|
|
@ -72,11 +95,21 @@ type sanitizerResult struct {
|
||||||
MarkdownHeader int `json:"markdown_header,omitempty"`
|
MarkdownHeader int `json:"markdown_header,omitempty"`
|
||||||
LatinInsert int `json:"latin_insert,omitempty"`
|
LatinInsert int `json:"latin_insert,omitempty"`
|
||||||
BrokenWord int `json:"broken_word,omitempty"`
|
BrokenWord int `json:"broken_word,omitempty"`
|
||||||
|
CJKLeak int `json:"cjk_leak,omitempty"`
|
||||||
Detail []string `json:"detail,omitempty"`
|
Detail []string `json:"detail,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s sanitizerResult) total() int {
|
func (s sanitizerResult) total() int {
|
||||||
return s.Preamble + s.TrailingNote + s.MarkdownHeader + s.LatinInsert + s.BrokenWord
|
return s.Preamble + s.TrailingNote + s.MarkdownHeader + s.LatinInsert + s.BrokenWord + s.CJKLeak
|
||||||
|
}
|
||||||
|
|
||||||
|
// cosmeticOnly reports whether the ONLY classes that fired are the STRIPPABLE cosmetic ones
|
||||||
|
// (markdown header and/or CJK-leak) — the strip-and-export tier (D35.4a). A single substantive
|
||||||
|
// class (preamble/trailing-note/latin/broken-word) pulls the whole chunk into the skip tier: its
|
||||||
|
// contamination has no reliable removable boundary, so the output is not trusted even if a
|
||||||
|
// cosmetic leak also happens to be present. False when nothing fired.
|
||||||
|
func (s sanitizerResult) cosmeticOnly() bool {
|
||||||
|
return s.total() > 0 && s.Preamble == 0 && s.TrailingNote == 0 && s.LatinInsert == 0 && s.BrokenWord == 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// summary is the human-readable disposition detail for a sanitizer flag (deterministic —
|
// summary is the human-readable disposition detail for a sanitizer flag (deterministic —
|
||||||
|
|
@ -91,7 +124,7 @@ func (s sanitizerResult) summary() string {
|
||||||
return strings.Join(s.Detail, "; ")
|
return strings.Join(s.Detail, "; ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// sanitizeOutput runs all five classes over one chunk's FINAL translated text.
|
// sanitizeOutput runs all six classes over one chunk's FINAL translated text.
|
||||||
func sanitizeOutput(text string) sanitizerResult {
|
func sanitizeOutput(text string) sanitizerResult {
|
||||||
var r sanitizerResult
|
var r sanitizerResult
|
||||||
if det := detectLeadingPreamble(text); det != "" {
|
if det := detectLeadingPreamble(text); det != "" {
|
||||||
|
|
@ -114,6 +147,10 @@ func sanitizeOutput(text string) sanitizerResult {
|
||||||
r.BrokenWord = n
|
r.BrokenWord = n
|
||||||
r.Detail = append(r.Detail, det...)
|
r.Detail = append(r.Detail, det...)
|
||||||
}
|
}
|
||||||
|
if n, det := detectCJKLeak(text); n > 0 {
|
||||||
|
r.CJKLeak = n
|
||||||
|
r.Detail = append(r.Detail, det...)
|
||||||
|
}
|
||||||
sort.Strings(r.Detail)
|
sort.Strings(r.Detail)
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
@ -316,6 +353,125 @@ func detectBrokenWords(text string) (int, []string) {
|
||||||
return len(det), det
|
return len(det), det
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- 6. CJK-leak in the final --------------------------------------------------
|
||||||
|
|
||||||
|
// isCJKLeakRune reports whether r is a stray CJK glyph that has NO place in a Russian final: a
|
||||||
|
// Han ideograph, kana, a fullwidth/halfwidth form (U+FF00–FFEF — «,()!» etc.), or a CJK
|
||||||
|
// symbol/punctuation (U+3001–303F — «、。「」»). The ideographic SPACE U+3000 is deliberately
|
||||||
|
// EXCLUDED (it is an indent artifact, not a content leak — research/18 §C#4 "draft-21 includes a
|
||||||
|
// U+3000 indent → 9 real leaks"): stripCosmetic normalises it to a plain space, but it does not
|
||||||
|
// alone FIRE the class, so a legitimately-indented chunk is not flag-stormed. In clean Russian
|
||||||
|
// prose every one of these is count-0, so the class is high precision. The intrinsic classify()
|
||||||
|
// already flags a ≥15%-CJK output as an ECHO (cjk_artifact, substantive) BEFORE the sanitizer
|
||||||
|
// runs, so this class only ever sees a SPARSE leak in otherwise-Russian text.
|
||||||
|
func isCJKLeakRune(r rune) bool {
|
||||||
|
switch {
|
||||||
|
case unicode.Is(unicode.Han, r):
|
||||||
|
return true
|
||||||
|
case unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r):
|
||||||
|
return true
|
||||||
|
case r >= 0xFF00 && r <= 0xFFEF: // fullwidth & halfwidth forms
|
||||||
|
return true
|
||||||
|
case r >= 0x3001 && r <= 0x303F: // CJK symbols & punctuation, EXCLUDING U+3000 (ideographic space)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail
|
||||||
|
// per distinct run. A single run («却有») is enough to fire — the class is COSMETIC, so the whole
|
||||||
|
// chunk is not lost: stripCosmetic removes the run and the remainder exports (flagged for review).
|
||||||
|
func detectCJKLeak(text string) (int, []string) {
|
||||||
|
var det []string
|
||||||
|
seen := map[string]bool{}
|
||||||
|
n := 0
|
||||||
|
rs := []rune(text)
|
||||||
|
for i := 0; i < len(rs); {
|
||||||
|
if !isCJKLeakRune(rs[i]) {
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
j := i
|
||||||
|
for j < len(rs) && isCJKLeakRune(rs[j]) {
|
||||||
|
j++
|
||||||
|
}
|
||||||
|
n++
|
||||||
|
run := string(rs[i:j])
|
||||||
|
if !seen[run] {
|
||||||
|
seen[run] = true
|
||||||
|
det = append(det, "CJK-утечка в ru-выходе: "+preview(run))
|
||||||
|
}
|
||||||
|
i = j
|
||||||
|
}
|
||||||
|
return n, det
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- deterministic strip of the COSMETIC classes (D35.4a export fix) -------------
|
||||||
|
|
||||||
|
// leadingMarkdownStripRE matches the leading markdown ATX prefix on a line (optional indent, 1–6
|
||||||
|
// hashes, blank(s)) directly before a header WORD/number — the same shape markdownHeaderRE fires
|
||||||
|
// on. It captures the first header rune so ReplaceAll keeps the heading text, dropping only the
|
||||||
|
// «### » artifact («### Глава 5» → «Глава 5»). A decorative «### ---» scene break (punctuation
|
||||||
|
// after the hashes) does NOT match, so it is left intact (and it never fired the class either).
|
||||||
|
var leadingMarkdownStripRE = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+([\p{L}\p{Nd}])`)
|
||||||
|
|
||||||
|
// stripCosmetic deterministically removes the two STRIPPABLE cosmetic leak classes — leading
|
||||||
|
// markdown headers and CJK-leak runs — and tidies the whitespace the removal leaves, so the
|
||||||
|
// remainder is readable. It is a PURE function (resume re-derives the identical export), invoked
|
||||||
|
// ONLY on output the sanitizer classified as cosmeticOnly (classifyOutput guarantees the result
|
||||||
|
// is non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
|
||||||
|
// sanitizer_stripped). It never crosses newlines when tidying, so paragraph structure is kept.
|
||||||
|
func stripCosmetic(text string) string {
|
||||||
|
// 1) markdown header prefixes → keep the heading text, drop the «#### » artifact.
|
||||||
|
out := leadingMarkdownStripRE.ReplaceAllString(text, "$1")
|
||||||
|
// 2) neutralise CJK-leak runes. CONTENT-BEARING glyphs are FOLDED, not dropped (adversarial
|
||||||
|
// review MAJOR): a fullwidth ASCII form is a wrong-glyph rendering of a plain char, so «123»
|
||||||
|
// folds to «123» (the number is kept, not deleted) and «,」→«,» — the same NFKC fold the
|
||||||
|
// memory normaliser uses. Only genuinely contentless runes (Han/kana/CJK brackets/symbols) are
|
||||||
|
// removed, and a removed run leaves a SINGLE SPACE so adjacent Russian words do not merge
|
||||||
|
// («нашёл特产древний» → «нашёл древний», not «нашёлдревний»). U+3000 → a plain space (indent).
|
||||||
|
var b strings.Builder
|
||||||
|
b.Grow(len(out))
|
||||||
|
for _, r := range out {
|
||||||
|
switch {
|
||||||
|
case r == ' ': // U+3000 ideographic space → plain space
|
||||||
|
b.WriteByte(' ')
|
||||||
|
case r >= 0xFF01 && r <= 0xFF5E: // fullwidth ASCII form → fold to plain ASCII (1→1, A→A, ,→,)
|
||||||
|
b.WriteRune(r - 0xFEE0)
|
||||||
|
case r == '、': // ideographic comma → plain comma (keep the clause boundary)
|
||||||
|
b.WriteByte(',')
|
||||||
|
case r == '。': // ideographic full stop → plain period
|
||||||
|
b.WriteByte('.')
|
||||||
|
case isCJKLeakRune(r): // Han / kana / CJK brackets / other symbol → drop, leaving a space
|
||||||
|
b.WriteByte(' ')
|
||||||
|
default:
|
||||||
|
b.WriteRune(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = b.String()
|
||||||
|
// 3) tidy the seams the removal created, per line (horizontal whitespace only — never a
|
||||||
|
// newline): collapse doubled spaces, drop a bracket/quote pair the strip emptied («(羅漢拳)» →
|
||||||
|
// «()» → removed), drop a space now sitting before closing punctuation or after an opening
|
||||||
|
// bracket/quote (never-correct Russian spacing), and trim trailing blanks. ':' is deliberately
|
||||||
|
// NOT in the closing-punct set: a space-before-colon is legitimate in a Russian ratio/score/time
|
||||||
|
// («счёт 2 : 1») and must not be reformatted (adversarial review — false positive).
|
||||||
|
out = multiSpaceRE.ReplaceAllString(out, " ")
|
||||||
|
out = emptyBracketRE.ReplaceAllString(out, "")
|
||||||
|
out = spaceBeforePunctRE.ReplaceAllString(out, "$1")
|
||||||
|
out = spaceAfterOpenRE.ReplaceAllString(out, "$1")
|
||||||
|
out = multiSpaceRE.ReplaceAllString(out, " ") // an emptied bracket may leave a doubled space
|
||||||
|
out = trailingHSpaceRE.ReplaceAllString(out, "")
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
multiSpaceRE = regexp.MustCompile(`[ \t]{2,}`)
|
||||||
|
emptyBracketRE = regexp.MustCompile(`\([ \t]*\)|«[ \t]*»|\[[ \t]*\]`)
|
||||||
|
spaceBeforePunctRE = regexp.MustCompile(`[ \t]+([,.!?;…»)\]])`)
|
||||||
|
spaceAfterOpenRE = regexp.MustCompile(`([«(\[])[ \t]+`)
|
||||||
|
trailingHSpaceRE = regexp.MustCompile(`(?m)[ \t]+$`)
|
||||||
|
)
|
||||||
|
|
||||||
type scriptToken struct {
|
type scriptToken struct {
|
||||||
text string
|
text string
|
||||||
cyr bool // Cyrillic-only
|
cyr bool // Cyrillic-only
|
||||||
|
|
|
||||||
|
|
@ -211,6 +211,100 @@ func TestSanitizerCleanNarrative(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSanitizerCJKLeak — the D38.1 class: a stray Han/kana/fullwidth glyph in the ru final.
|
||||||
|
func TestSanitizerCJKLeak(t *testing.T) {
|
||||||
|
fire := []string{
|
||||||
|
// The span the orchestrator verified (arms/C/17.0): a raw Han run mid-Russian-sentence.
|
||||||
|
"«…охотники,却有 на это лишние деньги!»",
|
||||||
|
"Судзуки нашёл 特产 древнего клана.", // single Han run
|
||||||
|
"Разряд资质乙等 остался неясен.", // Han glued to Cyrillic
|
||||||
|
"Он сказал: すごい, и замолчал.", // kana leak (ja book)
|
||||||
|
"Цена была 123 золотых.", // fullwidth digits
|
||||||
|
"Он вошёл в дом、и дверь закрылась.", // ideographic comma (U+3001)
|
||||||
|
}
|
||||||
|
for _, s := range fire {
|
||||||
|
if got := sanitizeOutput(s); got.CJKLeak == 0 {
|
||||||
|
t.Errorf("CJK-leak not detected in %q: %+v", preview(s), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clean := []string{
|
||||||
|
"Судзуки шёл по коридорам Академии магии, вспоминая вчерашнюю долгую лекцию.",
|
||||||
|
"— Сегодня я открою дверь книгохранилища, — прошептал он, сжимая ключ.",
|
||||||
|
"На гербе была латинская надпись, но он её не разобрал в темноте.",
|
||||||
|
"Цена была 123 золотых, и он расплатился не торгуясь.", // ASCII digits — not fullwidth
|
||||||
|
// U+3000 ideographic space alone is an INDENT artifact, NOT a content leak → must NOT fire
|
||||||
|
// (research/18 §C#4); stripCosmetic still normalises it, but it does not flag the chunk.
|
||||||
|
" Судзуки медленно шёл по длинному коридору к башне.",
|
||||||
|
}
|
||||||
|
for _, s := range clean {
|
||||||
|
if got := sanitizeOutput(s); got.CJKLeak != 0 {
|
||||||
|
t.Errorf("CJK-leak FALSE POSITIVE in %q: %v", preview(s), got.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSanitizerCosmeticOnly pins the strip-vs-skip disposition matrix (D35.4a): only the markdown
|
||||||
|
// header and CJK-leak classes are strippable; any substantive class pulls the chunk into skip.
|
||||||
|
func TestSanitizerCosmeticOnly(t *testing.T) {
|
||||||
|
cosmetic := []string{
|
||||||
|
"### Глава 7\n\nСудзуки открыл седьмую дверь и замер на пороге.", // markdown only
|
||||||
|
"Судзуки нашёл 特产 древнего клана в тёмном углу книгохранилища.", // cjk only
|
||||||
|
"### Пролог\n\nСудзуки увидел 却有 на алтаре забытого клана.", // BOTH cosmetic classes
|
||||||
|
}
|
||||||
|
for _, s := range cosmetic {
|
||||||
|
got := sanitizeOutput(s)
|
||||||
|
if got.total() == 0 || !got.cosmeticOnly() {
|
||||||
|
t.Errorf("expected cosmeticOnly for %q: %+v", preview(s), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
substantive := []string{
|
||||||
|
"Вот перевод фрагмента: Судзуки открыл дверь.", // preamble (substantive)
|
||||||
|
// a LEADING preamble co-occurring with a CJK leak → the substantive class wins (skip, not strip).
|
||||||
|
"Вот перевод фрагмента: Судзуки увидел 特产 на алтаре забытого клана.",
|
||||||
|
"Судзуки открыл дверь.\n\nПримечание: имя героя оставлено как есть.", // trailing note
|
||||||
|
"Он прочитал: the quick brown fox jumps over lazy dog today.", // latin phrase
|
||||||
|
}
|
||||||
|
for _, s := range substantive {
|
||||||
|
if got := sanitizeOutput(s); got.total() == 0 || got.cosmeticOnly() {
|
||||||
|
t.Errorf("expected NOT cosmeticOnly (substantive) for %q: %+v", preview(s), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStripCosmetic pins the exact deterministic strip output — the "экспортит текст БЕЗ ведущего
|
||||||
|
// ###" contract — and that the result is itself clean (the classifyOutput guarantee).
|
||||||
|
func TestStripCosmetic(t *testing.T) {
|
||||||
|
cases := []struct{ in, want string }{
|
||||||
|
// markdown header prefix removed, heading text kept.
|
||||||
|
{"### Глава 7\n\nСудзуки открыл дверь.", "Глава 7\n\nСудзуки открыл дверь."},
|
||||||
|
{"# Пролог\nТекст главы.", "Пролог\nТекст главы."},
|
||||||
|
// Han run removed; the seam is tidied (no double space, no space-before-comma), no word-merge.
|
||||||
|
{"«…охотники,却有 на это лишние деньги!»", "«…охотники, на это лишние деньги!»"},
|
||||||
|
{"Судзуки нашёл 特产 древнего клана.", "Судзуки нашёл древнего клана."},
|
||||||
|
{"нашёл特产древний", "нашёл древний"}, // no spaces in source → strip leaves a space, not a merge
|
||||||
|
// CONTENT-BEARING fullwidth ASCII is FOLDED, not deleted (adversarial review MAJOR): the number survives.
|
||||||
|
{"Цена была 123 золотых.", "Цена была 123 золотых."},
|
||||||
|
{"Индекс ABC найден.", "Индекс ABC найден."},
|
||||||
|
// ideographic comma/period fold to plain — the clause boundary is kept.
|
||||||
|
{"Он вошёл в дом、и дверь закрылась。", "Он вошёл в дом,и дверь закрылась."},
|
||||||
|
// a parenthetical Han gloss is stripped and the emptied «()» removed (adversarial review).
|
||||||
|
{"Кулак Ло Хань (羅漢拳) был силён.", "Кулак Ло Хань был силён."},
|
||||||
|
// a legit ratio «2 : 1» is NOT reformatted (colon dropped from the tidy set).
|
||||||
|
{"### Итог\n\nСчёт был 2 : 1 却有 в пользу гостей.", "Итог\n\nСчёт был 2 : 1 в пользу гостей."},
|
||||||
|
// U+3000 ideographic space normalised to a plain space (the indent artifact).
|
||||||
|
{" Судзуки шёл.", " Судзуки шёл."},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
got := stripCosmetic(c.in)
|
||||||
|
if got != c.want {
|
||||||
|
t.Errorf("stripCosmetic(%q) = %q, want %q", c.in, got, c.want)
|
||||||
|
}
|
||||||
|
if san := sanitizeOutput(got); san.total() != 0 {
|
||||||
|
t.Errorf("stripCosmetic(%q) is not clean: %+v", c.in, san.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestSanitizerDeterministic asserts the verdict is a pure function of the text (resume-safe).
|
// TestSanitizerDeterministic asserts the verdict is a pure function of the text (resume-safe).
|
||||||
func TestSanitizerDeterministic(t *testing.T) {
|
func TestSanitizerDeterministic(t *testing.T) {
|
||||||
s := "### Заголовок\nВот отредактированный перевод: текст. Стольь странно.\nПримечание: правка."
|
s := "### Заголовок\nВот отредактированный перевод: текст. Стольь странно.\nПримечание: правка."
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,8 @@ package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -149,8 +151,12 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
||||||
runCost += esc.fb.runCost
|
runCost += esc.fb.runCost
|
||||||
anyFresh = anyFresh || esc.fb.freshCall
|
anyFresh = anyFresh || esc.fb.freshCall
|
||||||
escalated, escModel = true, st.EscalateTo
|
escalated, escModel = true, st.EscalateTo
|
||||||
if esc.fb.cls.ok() {
|
if esc.fb.cls.ok() || esc.fb.cls.Reason == FlagSanitizerStripped {
|
||||||
last = esc.fb // the fallback draft passed the re-gate → it is authoritative
|
// The fallback is authoritative when it passed the re-gate OR when it is a cosmetic
|
||||||
|
// sanitizer strip on a FINAL escalatable stage (adversarial review): a stripped fallback
|
||||||
|
// is a usable-but-flagged export, so it must be recovered (final_hash → its cleaned text)
|
||||||
|
// rather than discarded to an empty placeholder — mirroring the non-escalated path.
|
||||||
|
last = esc.fb
|
||||||
}
|
}
|
||||||
// else: the fallback also failed → keep the primary flag (last unchanged);
|
// else: the fallback also failed → keep the primary flag (last unchanged);
|
||||||
// the fallback call is billed and counted, the chunk stays flagged (1 hop).
|
// the fallback call is billed and counted, the chunk stays flagged (1 hop).
|
||||||
|
|
@ -158,12 +164,27 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
||||||
|
|
||||||
disposition := last.cls.Reason.disposition()
|
disposition := last.cls.Reason.disposition()
|
||||||
finalHash := ""
|
finalHash := ""
|
||||||
|
recovered := ""
|
||||||
if disposition == DispOK {
|
if disposition == DispOK {
|
||||||
finalHash = last.reqHash
|
finalHash = last.reqHash
|
||||||
|
} else if last.cls.Reason == FlagSanitizerStripped {
|
||||||
|
// Cosmetic leak (D35.4a): strip it and COMMIT the cleaned remainder as a $0 derived export
|
||||||
|
// checkpoint, then point final_hash at it — so the standard final_hash→checkpoint.response_text
|
||||||
|
// export (records.json / exp12_extract) yields the cleaned text instead of empty, while the
|
||||||
|
// chunk stays flagged for a human. classifyOutput guaranteed the strip is non-empty and clean.
|
||||||
|
// The derived checkpoint is durably written BEFORE chunk_status references it (the ok path's
|
||||||
|
// checkpoint-before-pointer discipline), so a resume never dangles final_hash at a missing row.
|
||||||
|
recovered = stripCosmetic(last.text)
|
||||||
|
dh, err := r.commitSanitizedExport(st, ch, job, last, attemptsMade-1, recovered)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
finalHash = dh
|
||||||
}
|
}
|
||||||
// chunk_status is a RESOLVE over the checkpoints: cost_usd sums every attempt
|
// chunk_status is a RESOLVE over the checkpoints: cost_usd sums every attempt
|
||||||
// (F3-honest — retries are counted), final_hash points at the authoritative
|
// (F3-honest — retries are counted), final_hash points at the authoritative
|
||||||
// checkpoint the ok path serves on resume.
|
// checkpoint the ok path serves on resume (or the derived sanitized-export checkpoint for
|
||||||
|
// a FlagSanitizerStripped chunk, so its cleaned text still exports).
|
||||||
if err := r.Store.UpsertChunkStatus(store.ChunkStatus{
|
if err := r.Store.UpsertChunkStatus(store.ChunkStatus{
|
||||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: st.Name,
|
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: st.Name,
|
||||||
SnapshotID: snapID, ContentHash: contentHash, Disposition: string(disposition), FlagReason: string(last.cls.Reason),
|
SnapshotID: snapID, ContentHash: contentHash, Disposition: string(disposition), FlagReason: string(last.cls.Reason),
|
||||||
|
|
@ -181,6 +202,7 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
||||||
CumCostUSD: cumCost,
|
CumCostUSD: cumCost,
|
||||||
LatencyMS: last.latency,
|
LatencyMS: last.latency,
|
||||||
FinishReason: last.finish,
|
FinishReason: last.finish,
|
||||||
|
RecoveredText: recovered,
|
||||||
Disposition: disposition,
|
Disposition: disposition,
|
||||||
FlagReason: last.cls.Reason,
|
FlagReason: last.cls.Reason,
|
||||||
Detail: last.cls.Detail,
|
Detail: last.cls.Detail,
|
||||||
|
|
@ -198,6 +220,24 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
||||||
return sr, nil
|
return sr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// commitSanitizedExport persists the cosmetic-stripped export text as a $0 derived checkpoint and
|
||||||
|
// returns its request_hash (→ chunk_status.final_hash). The hash is CONTENT-ADDRESSED and
|
||||||
|
// namespaced ("tm-sanitized-v1:") so it can never collide with a real attempt's hex request_hash
|
||||||
|
// and a resume/re-run re-derives the identical id for free. Called ONLY for a FlagSanitizerStripped
|
||||||
|
// final stage (D35.4a); `att` is the flagged completion the strip ran over.
|
||||||
|
func (r *Runner) commitSanitizedExport(st config.Stage, ch Chunk, job *store.Job, att stageAttempt, attempt int, stripped string) (string, error) {
|
||||||
|
sum := sha256.Sum256([]byte("tm-sanitized-v1\x00" + att.reqHash + "\x00" + stripped))
|
||||||
|
derivedHash := "tm-sanitized-v1:" + hex.EncodeToString(sum[:])
|
||||||
|
if err := r.Store.PutDerivedCheckpoint(store.Checkpoint{
|
||||||
|
RequestHash: derivedHash, JobID: job.ID, ChunkIdx: ch.ChunkIdx, Attempt: attempt,
|
||||||
|
Stage: st.Name, Role: st.Role, ModelRequested: att.modelActual, ModelActual: att.modelActual,
|
||||||
|
ResponseText: stripped, UsageJSON: "{}", FinishReason: "sanitized_export",
|
||||||
|
}); err != nil {
|
||||||
|
return "", fmt.Errorf("pipeline: commit sanitized export ch%d/chunk%d/%s: %w", ch.Chapter, ch.ChunkIdx, st.Name, err)
|
||||||
|
}
|
||||||
|
return derivedHash, nil
|
||||||
|
}
|
||||||
|
|
||||||
// stageAttempt is the result of one attempt (a checkpoint hit or a fresh call).
|
// stageAttempt is the result of one attempt (a checkpoint hit or a fresh call).
|
||||||
type stageAttempt struct {
|
type stageAttempt struct {
|
||||||
reqHash string
|
reqHash string
|
||||||
|
|
|
||||||
|
|
@ -104,7 +104,7 @@ func flagReasonSeverity(reason string) int {
|
||||||
case FlagCJKArtifact, FlagExcisionSuspect, FlagCoverageFail:
|
case FlagCJKArtifact, FlagExcisionSuspect, FlagCoverageFail:
|
||||||
return 1
|
return 1
|
||||||
case FlagSanitizerDefect:
|
case FlagSanitizerDefect:
|
||||||
// A contaminated output (leaked preamble / notes / ###) is unreadable-as-shipped —
|
// A contaminated output (leaked preamble / notes) that was DROPPED is unreadable-as-shipped —
|
||||||
// ranked with the deterministic content failures, above a mere budget symptom.
|
// ranked with the deterministic content failures, above a mere budget symptom.
|
||||||
return 2
|
return 2
|
||||||
case FlagLoopDegenerate:
|
case FlagLoopDegenerate:
|
||||||
|
|
@ -113,10 +113,15 @@ func flagReasonSeverity(reason string) int {
|
||||||
return 4
|
return 4
|
||||||
case FlagGlossaryMiss:
|
case FlagGlossaryMiss:
|
||||||
return 5
|
return 5
|
||||||
|
case FlagSanitizerStripped:
|
||||||
|
// A cosmetic leak the sanitizer STRIPPED and exported (D35.4a): the chunk shipped cleaned,
|
||||||
|
// so it is the least alarming flag — an "auto-cleaned, glance to verify" signal, ranked below
|
||||||
|
// a budget symptom (the chunk is not lost; a human need only spot-check the auto-clean).
|
||||||
|
return 7
|
||||||
case FlagLength, FlagEmpty:
|
case FlagLength, FlagEmpty:
|
||||||
return 6
|
return 6
|
||||||
}
|
}
|
||||||
return 7
|
return 8
|
||||||
}
|
}
|
||||||
|
|
||||||
// bookChunks re-derives the book's chunk manifest — Ingest + SplitChunks — for the honest N/M
|
// bookChunks re-derives the book's chunk manifest — Ingest + SplitChunks — for the honest N/M
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -41,3 +41,11 @@
|
||||||
第六章 序文漏洩
|
第六章 序文漏洩
|
||||||
|
|
||||||
鈴木は最後の扉の前に立ち、深く息を吸い込んだ。その先に何があるのか、誰も知らなかった。
|
鈴木は最後の扉の前に立ち、深く息を吸い込んだ。その先に何があるのか、誰も知らなかった。
|
||||||
|
|
||||||
|
第七章 見出漏洩
|
||||||
|
|
||||||
|
七層目の朝、鈴木は最後の書庫の扉を開けた。扉の奥には見出しだけが刻まれた石板が静かに待っていた。誰もその文字を読めなかった。
|
||||||
|
|
||||||
|
第八章 漢字漏洩
|
||||||
|
|
||||||
|
鈴木は石板の文字をなぞり、古い漢字の意味を思い出そうとした。だが記憶は霧のように薄れ、指先だけが淡く光っていた。
|
||||||
|
|
|
||||||
|
|
@ -170,6 +170,33 @@ func (s *Store) SettleWithCheckpoint(res Reservation, cost float64, cp Checkpoin
|
||||||
return tx.Commit()
|
return tx.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PutDerivedCheckpoint persists a $0 DERIVED checkpoint — a deterministic post-processing
|
||||||
|
// artifact, NOT a billed provider response (D38 infra-pack): the output-sanitizer's cosmetic
|
||||||
|
// strip commits the cleaned final text here so the standard final_hash→checkpoint.response_text
|
||||||
|
// export contract (records.json / exp12_extract) yields the cleaned text for a flagged chunk that
|
||||||
|
// would otherwise export empty (D35.4a). It touches NO spend/reservation (cost must be 0), keyed
|
||||||
|
// by its own derived request_hash (namespaced, collision-free with real attempt hashes), and is
|
||||||
|
// idempotent (ON CONFLICT DO NOTHING) so a re-run re-derives the identical row for free. Escalation
|
||||||
|
// is deliberately false so it never counts toward escalation.budget_usd. It is deleted with its
|
||||||
|
// stage's real checkpoints on `redrive` (ResetChunkStages joins by job/chunk/stage), never orphaned.
|
||||||
|
func (s *Store) PutDerivedCheckpoint(cp Checkpoint) error {
|
||||||
|
ctx, cancel := opContext()
|
||||||
|
defer cancel()
|
||||||
|
_, err := s.w.ExecContext(ctx, `
|
||||||
|
INSERT INTO checkpoints (
|
||||||
|
request_hash, job_id, chunk_idx, attempt, stage, role,
|
||||||
|
model_requested, model_actual, response_text, usage_json,
|
||||||
|
cost_usd, finish_reason, provider_request_id, escalation
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, '', 0)
|
||||||
|
ON CONFLICT (request_hash) DO NOTHING`,
|
||||||
|
cp.RequestHash, cp.JobID, cp.ChunkIdx, cp.Attempt, cp.Stage, cp.Role,
|
||||||
|
cp.ModelRequested, cp.ModelActual, cp.ResponseText, cp.UsageJSON, cp.FinishReason)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("store: derived checkpoint insert: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetCheckpoint returns the persisted response for a request hash, if any —
|
// GetCheckpoint returns the persisted response for a request hash, if any —
|
||||||
// the resume path: a hit means the call is NOT repeated or re-billed.
|
// the resume path: a hit means the call is NOT repeated or re-billed.
|
||||||
func (s *Store) GetCheckpoint(requestHash string) (*Checkpoint, error) {
|
func (s *Store) GetCheckpoint(requestHash string) (*Checkpoint, error) {
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,41 @@ func TestReserveSettleLifecycle(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestPutDerivedCheckpoint pins the $0 sanitized-export checkpoint (D38 infra-pack): it persists a
|
||||||
|
// retrievable response, books NO spend, and is idempotent (re-derive on resume is free).
|
||||||
|
func TestPutDerivedCheckpoint(t *testing.T) {
|
||||||
|
s, _ := openTemp(t)
|
||||||
|
job := mustSnapshotAndJob(t, s)
|
||||||
|
|
||||||
|
cp := Checkpoint{RequestHash: "tm-sanitized-v1:abc", JobID: job.ID, ChunkIdx: 0, Stage: "edit",
|
||||||
|
Role: "editor", ModelRequested: "m", ModelActual: "m", ResponseText: "очищенный текст",
|
||||||
|
UsageJSON: "{}", FinishReason: "sanitized_export"}
|
||||||
|
if err := s.PutDerivedCheckpoint(cp); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Retrievable via the standard checkpoint read (the final_hash→checkpoint export contract).
|
||||||
|
got, err := s.GetCheckpoint("tm-sanitized-v1:abc")
|
||||||
|
if err != nil || got == nil || got.ResponseText != "очищенный текст" {
|
||||||
|
t.Fatalf("derived checkpoint: %+v err=%v", got, err)
|
||||||
|
}
|
||||||
|
if got.CostUSD != 0 {
|
||||||
|
t.Fatalf("derived checkpoint must be $0, got %v", got.CostUSD)
|
||||||
|
}
|
||||||
|
// No spend booked (it is a post-processing artifact, not a provider call).
|
||||||
|
committed, reserved, _ := s.SpentUSD("book")
|
||||||
|
if committed != 0 || reserved != 0 {
|
||||||
|
t.Fatalf("derived checkpoint booked spend: committed=%v reserved=%v", committed, reserved)
|
||||||
|
}
|
||||||
|
// Idempotent: a resume re-derives the identical row for free (ON CONFLICT DO NOTHING).
|
||||||
|
if err := s.PutDerivedCheckpoint(cp); err != nil {
|
||||||
|
t.Fatalf("re-put must be idempotent: %v", err)
|
||||||
|
}
|
||||||
|
committed, _, _ = s.SpentUSD("book")
|
||||||
|
if committed != 0 {
|
||||||
|
t.Fatalf("idempotent re-put booked spend: committed=%v", committed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCeilingsDeny(t *testing.T) {
|
func TestCeilingsDeny(t *testing.T) {
|
||||||
s, _ := openTemp(t)
|
s, _ := openTemp(t)
|
||||||
mustSnapshotAndJob(t, s)
|
mustSnapshotAndJob(t, s)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue