From 8957046424493d22ca611174b845035689d42cb6 Mon Sep 17 00:00:00 2001 From: "Claude (backend session)" Date: Sat, 4 Jul 2026 23:33:42 +0300 Subject: [PATCH] Add the configurable excision coverage-gate porting refusal_bench sentence-coverage bit-for-bit, emitting excision_suspect and folded into the snapshot --- backend/internal/config/config_test.go | 10 +- backend/internal/config/pipeline.go | 25 ++- backend/internal/pipeline/coverage.go | 169 +++++++++++++++++ backend/internal/pipeline/coverage_test.go | 201 +++++++++++++++++++++ backend/internal/pipeline/runner.go | 62 ++++++- backend/internal/pipeline/runner_test.go | 152 +++++++++++++++- docs/PROGRESS.md | 10 +- 7 files changed, 613 insertions(+), 16 deletions(-) create mode 100644 backend/internal/pipeline/coverage.go create mode 100644 backend/internal/pipeline/coverage_test.go diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index 87a965c..bf699ad 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -16,7 +16,9 @@ func TestCheckRunnableRejectsPhase2Mechanics(t *testing.T) { {"c2 rejected", Pipeline{Core: "C2", Stages: []Stage{{Name: "draft", Role: "translator"}}}, "C2"}, {"fanout rejected", Pipeline{Core: "C1", Fanout: Fanout{Candidates: 3}, Stages: []Stage{{Name: "draft", Role: "translator"}}}, "fanout"}, {"judge rejected", Pipeline{Core: "C1", Stages: []Stage{{Name: "select", Role: "judge"}}}, "judge"}, - {"enabled gate rejected", Pipeline{Core: "C1", Gates: Gates{Coverage: CoverageGate{Enabled: true}}, Stages: []Stage{{Name: "draft", Role: "translator"}}}, "gates"}, + // The coverage gate is EXECUTABLE from Веха 2.5 — CheckRunnable no longer + // rejects it (threshold sanity is LoadPipeline's job, not a mechanics gate). + {"enabled gate runnable", Pipeline{Core: "C1", Gates: Gates{Coverage: CoverageGate{Enabled: true}}, Stages: []Stage{{Name: "draft", Role: "translator"}}}, ""}, } for _, c := range cases { err := c.p.CheckRunnable() @@ -39,9 +41,9 @@ func TestCheckKeysOnlyUsedNonLocalModels(t *testing.T) { "local": {Kind: "local", BaseURL: "http://127.0.0.1", Model: "q"}, }, Models: map[string]Model{ - "cloud-model": {Provider: "cloud"}, - "local-model": {Provider: "local"}, - "unused-cloud": {Provider: "cloud"}, + "cloud-model": {Provider: "cloud"}, + "local-model": {Provider: "local"}, + "unused-cloud": {Provider: "cloud"}, }, } // local-only pipeline не должен падать на облачном ключе. diff --git a/backend/internal/config/pipeline.go b/backend/internal/config/pipeline.go index 2d13e63..21a6457 100644 --- a/backend/internal/config/pipeline.go +++ b/backend/internal/config/pipeline.go @@ -114,12 +114,10 @@ func (p *Pipeline) CheckRunnable() error { if p.Fanout.Candidates > 1 { return fmt.Errorf("pipeline fanout.candidates=%d не исполним в Фазе 0 (fan-out N кандидатов — механика раннера Фазы 2; C1 = один черновик)", p.Fanout.Candidates) } - // QA-гейты — механика Фазы 1; раннер Фазы 0 их НЕ исполняет. Включённый - // гейт молча пропустил бы негейченный вывод как done (против Р7) — - // поэтому fail-loud, а не fail-open (находка внешнего ревью F5). - if p.gatesEnabled() { - return fmt.Errorf("pipeline gates включены, но не исполнимы в Фазе 0 (QA-гейты — механика Фазы 1; иначе вывод пройдёт негейченным и будет ложно помечен done)") - } + // Coverage QA-гейт исполним с Вехи 2.5 (excision-детект, coverage.go): раннер + // считает его по результату стадии и эмитит excision_suspect. Бывший fail-loud + // на gates.enabled снят; корректность порогов включённого гейта проверяет + // LoadPipeline (иначе тихий no-op — против Р7). for _, st := range p.Stages { if st.Role == "judge" { return fmt.Errorf("pipeline stage %q role=judge не исполним в Фазе 0 (селектор получает N кандидатов — механика Фазы 2, а раннер гонит стадии линейно)", st.Name) @@ -207,6 +205,21 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) { } } } + // Coverage QA-gate thresholds (шаг 6): an ENABLED gate must be able to flag + // something, otherwise it is a silent no-op that passes every chunk (against Р7). + if cov := p.Gates.Coverage; cov.Enabled { + if cov.SentCovMin < 0 || cov.SentCovMin > 1 { + bad("gates.coverage.sent_cov_min must be within [0,1], got %v", cov.SentCovMin) + } + for pair, b := range cov.LenRatio { + if len(b) != 2 || b[0] <= 0 || b[1] < b[0] { + bad("gates.coverage.len_ratio_bounds[%q] must be [low, high] with 0 < low <= high, got %v", pair, b) + } + } + if cov.SentCovMin <= 0 && len(cov.LenRatio) == 0 { + bad("gates.coverage.enabled is true but neither sent_cov_min nor len_ratio_bounds is set — the gate would silently pass every chunk") + } + } if len(problems) > 0 { return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - ")) } diff --git a/backend/internal/pipeline/coverage.go b/backend/internal/pipeline/coverage.go new file mode 100644 index 0000000..020b68b --- /dev/null +++ b/backend/internal/pipeline/coverage.go @@ -0,0 +1,169 @@ +package pipeline + +import ( + "fmt" + "strings" + "unicode" + + "textmachine/backend/internal/config" +) + +// coverage.go: the configurable EXCISION coverage-gate (шаг 6 / D12 Q3). Unlike the +// intrinsic classify() robustness (echo/empty/refusal — always on), this gate is +// opt-in (gates.coverage.enabled) and threshold-driven: it flags a stage OUTPUT +// that covers too LITTLE of the ORIGINAL source (silent excision — куски пропали, +// HTTP 200). It is a pure, deterministic RUNNER-side check (not an adapter concern — +// the adapter is translation-neutral, Q1/D2.4): given the checkpointed output it +// reproduces the same verdict on resume without re-billing. +// +// Ported 1:1 from eval/refusal_bench.py::classify_output's excision branch, which is +// BOTH our reference AND the Polygon acceptance oracle (D12 Q1). Metric = characters +// WITHOUT spaces (not tokens, §3.7). Only the LOWER bound is checked: sent_cov < +// sent_cov_min OR len_ratio < corridor-low → excision_suspect. The UPPER bound +// (anomaly/дописывание) is deliberately NOT flagged here (D12 Q3: ru output is +// 1.4–1.9× the source, so an upper cut false-flags; catching additions needs an +// entity in/out count + a bilingual judge — Phase 2, §04-unhappy mode 2). + +// coverageGateVersion versions the segmentation + metric of this gate. It is folded +// into the job snapshot (like chunkerVersion/estimatorVersion), so a change to the +// algorithm — or turning the gate ON — is a loud --resnapshot that re-evaluates every +// chunk from its checkpoint for FREE (no re-bill), never a silent divergence between +// an old checkpoint's stored verdict and a new gate. The segmentation is pinned 1:1 +// to eval/refusal_bench.py (D12 Q1); any change is Python-first, in lock-step, and +// bumps this constant. +const coverageGateVersion = "coverage-v1-naive-split-lower-bound" + +// isSentenceTerminator is the branch-1 lookbehind class of refusal_bench's +// SENT_SPLIT_RE `[.!?…。！？]` (Latin + ellipsis + fullwidth CJK): a whitespace run +// after any of these splits a sentence. +func isSentenceTerminator(r rune) bool { + switch r { + case '.', '!', '?', '…', '。', '！', '？': + return true + } + return false +} + +// isCJKSentenceTerminator is the branch-2 lookbehind class `[。！？]` (fullwidth CJK +// only): these split ZERO-WIDTH, with or without a following space, because CJK prose +// has no inter-sentence space. +func isCJKSentenceTerminator(r rune) bool { + switch r { + case '。', '！', '？': + return true + } + return false +} + +// splitSentences ports refusal_bench.py::split_sentences 1:1. Go's RE2 has no +// lookbehind, so the regex `(?<=[.!?…。！？])\s+|(?<=[。！？])` is reproduced by a manual +// left-to-right scan (branch-1 before branch-2, matching Python's alternation order): +// +// (1) a whitespace RUN preceded by ANY terminator — the run is the separator and +// is consumed; then +// (2) a ZERO-WIDTH point preceded by a CJK terminator — nothing consumed. +// +// Each raw piece is TrimSpace'd and empty pieces dropped (the Python list-comp). +// Pinned bit-for-bit to the live oracle by TestSplitSentencesOracleParity. +func splitSentences(text string) []string { + runes := []rune(text) + var segs []string + emit := func(a, b int) { + if s := strings.TrimSpace(string(runes[a:b])); s != "" { + segs = append(segs, s) + } + } + segStart := 0 + for i := 1; i < len(runes); i++ { + prev := runes[i-1] + // Branch 1: any terminator, then a whitespace run (the run is consumed). + if isSentenceTerminator(prev) && unicode.IsSpace(runes[i]) { + emit(segStart, i) + j := i + for j < len(runes) && unicode.IsSpace(runes[j]) { + j++ + } + segStart = j + i = j - 1 // the loop's i++ lands on j + continue + } + // Branch 2: CJK terminator, zero-width split (the terminator stays with the + // left piece; scanning continues from the same char, now the segment start). + if isCJKSentenceTerminator(prev) { + emit(segStart, i) + segStart = i + } + } + emit(segStart, len(runes)) + return segs +} + +// nonSpaceCount is the coverage metric's unit — runes that are NOT whitespace +// (refusal_bench: `sum(1 for c in text if not c.isspace())`). Characters, not tokens. +func nonSpaceCount(s string) int { + n := 0 + for _, r := range s { + if !unicode.IsSpace(r) { + n++ + } + } + return n +} + +// langPairKey builds the `len_ratio_bounds` config key from the book's languages, +// e.g. ("zh","ru") → "zh-ru". Lower-cased and trimmed so casing in book.yaml does +// not miss the corridor. +func langPairKey(srcLang, dstLang string) string { + return strings.ToLower(strings.TrimSpace(srcLang)) + "-" + strings.ToLower(strings.TrimSpace(dstLang)) +} + +// coverageResult carries the metric values alongside the verdict for telemetry/tests. +type coverageResult struct { + cls classification + applied bool // false when skipped (too short / disabled) + sentCov float64 // sentences(out)/sentences(src) + lenRatio float64 // nonspace(out)/nonspace(src) +} + +// coverageCheck runs the excision gate on ONE stage output against the ORIGINAL +// source. It is pair-aware: len_ratio is judged against the corridor for +// srcLang-dstLang; sentence coverage is pair-independent. A source shorter than +// MinChunkChars (non-space) is not gated — the corridors are calibrated on +// 500–2500-char fragments and short chunks' ratios are noise (§3.7). Pure & +// deterministic. Returns excision_suspect (lower-bound only) or ok. +func coverageCheck(cfg config.CoverageGate, src, out, srcLang, dstLang string) coverageResult { + srcChars := nonSpaceCount(src) + if srcChars < cfg.MinChunkChars { + return coverageResult{cls: classification{reasonOK, ""}, applied: false} + } + + nSrc := len(splitSentences(src)) + sentCov := 1.0 + if nSrc > 0 { + sentCov = float64(len(splitSentences(out))) / float64(nSrc) + } + denom := srcChars + if denom < 1 { + denom = 1 + } + lenRatio := float64(nonSpaceCount(out)) / float64(denom) + + res := coverageResult{applied: true, sentCov: sentCov, lenRatio: lenRatio} + var flags []string + if sentCov < cfg.SentCovMin { + flags = append(flags, fmt.Sprintf("sent_cov=%.2f<%.2f", sentCov, cfg.SentCovMin)) + } + // len_ratio needs a corridor for this pair; without one we still apply sentence + // coverage (pair-independent) rather than silently pass everything. + if bounds, ok := cfg.LenRatio[langPairKey(srcLang, dstLang)]; ok && len(bounds) >= 1 { + if lo := bounds[0]; lenRatio < lo { + flags = append(flags, fmt.Sprintf("len_ratio=%.2f<%.2f", lenRatio, lo)) + } + } + if len(flags) > 0 { + res.cls = classification{FlagExcisionSuspect, strings.Join(flags, "; ")} + return res + } + res.cls = classification{reasonOK, ""} + return res +} diff --git a/backend/internal/pipeline/coverage_test.go b/backend/internal/pipeline/coverage_test.go new file mode 100644 index 0000000..35b7218 --- /dev/null +++ b/backend/internal/pipeline/coverage_test.go @@ -0,0 +1,201 @@ +package pipeline + +import ( + "strings" + "testing" + + "textmachine/backend/internal/config" +) + +// coverage_test.go pins the excision coverage-gate. The segmentation/metric is +// PINNED BIT-FOR-BIT to the Polygon acceptance oracle (eval/refusal_bench.py, +// D12 Q1): the expected values below were captured from the LIVE oracle +// (scripts/oracle_groundtruth.py over refusal_bench.split_sentences/cjk_share). If +// the oracle changes, this table must change Python-first, in lock-step — that is +// the contract, not an accident. + +// oracleSplit is the ground-truth from refusal_bench.py::split_sentences. +var oracleSplit = []struct { + in string + want []string +}{ + {"", nil}, + {" ", nil}, + {"Hello world", []string{"Hello world"}}, + {"Hello. World.", []string{"Hello.", "World."}}, + {"Hello.World.", []string{"Hello.World."}}, // no space after Latin '.' → no split + {"Hello!! World?? Yes.", []string{"Hello!!", "World??", "Yes."}}, + {"One. Two! Three? Four…", []string{"One.", "Two!", "Three?", "Four…"}}, + {"Wait… really?", []string{"Wait…", "really?"}}, + {"猫。", []string{"猫。"}}, + {"猫。狗。鸟。", []string{"猫。", "狗。", "鸟。"}}, + {"猫。 狗。 鸟。", []string{"猫。", "狗。", "鸟。"}}, + {"猫。。狗", []string{"猫。", "。", "狗"}}, // consecutive CJK terminators → lone 。 fragment + {"「こんにちは！」彼は笑った。", []string{"「こんにちは！", "」彼は笑った。"}}, // ！ splits zero-width, 」 joins next + {"『引用』。次の文。", []string{"『引用』。", "次の文。"}}, + {"«Привет!» — он засмеялся. «Пока».", []string{"«Привет!» — он засмеялся.", "«Пока»."}}, // !» does NOT split + {"“Hi!” he said. “Bye.”", []string{"“Hi!” he said.", "“Bye.”"}}, + {"阿Q正传。第一章。", []string{"阿Q正传。", "第一章。"}}, + {"彼は言った：「行こう！」", []string{"彼は言った：「行こう！", "」"}}, + {"はい。　いいえ。", []string{"はい。", "いいえ。"}}, // U+3000 ideographic space after 。 is consumed + {"Yes.\tNo.\nMaybe.", []string{"Yes.", "No.", "Maybe."}}, + {"こんにちは、世界。さようなら！", []string{"こんにちは、世界。", "さようなら！"}}, // 、 is NOT a terminator + {"……", []string{"……"}}, // Latin ellipsis, no whitespace → no split + {"。。。", []string{"。", "。", "。"}}, // each CJK period splits zero-width + {"A。B. C。D", []string{"A。", "B.", "C。", "D"}}, + {"Full？width。terminators！here", []string{"Full？", "width。", "terminators！", "here"}}, + {" leading and trailing. ", []string{"leading and trailing."}}, + {"第一。\n\n第二。", []string{"第一。", "第二。"}}, + {"Mr. Smith went. He left.", []string{"Mr.", "Smith went.", "He left."}}, // naive over-split (known) +} + +func TestSplitSentencesOracleParity(t *testing.T) { + for _, c := range oracleSplit { + got := splitSentences(c.in) + if len(got) != len(c.want) { + t.Errorf("splitSentences(%q) = %#v (n=%d), want %#v (n=%d)", c.in, got, len(got), c.want, len(c.want)) + continue + } + for i := range got { + if got[i] != c.want[i] { + t.Errorf("splitSentences(%q)[%d] = %q, want %q", c.in, i, got[i], c.want[i]) + } + } + } +} + +func TestNonSpaceCountOracleParity(t *testing.T) { + cases := []struct { + in string + want int + }{ + {"Hello. World.", 12}, + {"猫。 狗。 鸟。", 6}, + {"Yes.\tNo.\nMaybe.", 13}, + {"はい。　いいえ。", 7}, // U+3000 counts as space → excluded + {" leading and trailing. ", 19}, + {"", 0}, + } + for _, c := range cases { + if got := nonSpaceCount(c.in); got != c.want { + t.Errorf("nonSpaceCount(%q) = %d, want %d", c.in, got, c.want) + } + } +} + +// boevoyBounds mirrors the boevoy pipeline-c1.yaml corridors. +func boevoyGate(minChars int) config.CoverageGate { + return config.CoverageGate{ + Enabled: true, + LenRatio: map[string][]float64{"zh-ru": {2.2, 4.2}, "ja-ru": {1.4, 2.6}, "en-ru": {0.7, 1.4}}, + SentCovMin: 0.75, + MinChunkChars: minChars, + } +} + +func TestCoverageCheckExcision(t *testing.T) { + // Mirrors the three classify_output cases captured from the live oracle. + zhExcised := coverageCheck(boevoyGate(0), strings.Repeat("这是一个测试。", 20), "Короткий перевод.", "zh", "ru") + if zhExcised.cls.Reason != FlagExcisionSuspect { + t.Fatalf("zh short output must flag excision, got %+v", zhExcised) + } + if zhExcised.sentCov > 0.1 || zhExcised.lenRatio > 0.2 { + t.Fatalf("zh metrics off (oracle: sent_cov=0.05, len_ratio=0.114): %+v", zhExcised) + } + + zhHealthy := coverageCheck(boevoyGate(0), + strings.Repeat("这是第一句。这是第二句。这是第三句。", 4), + strings.Repeat("Это первое предложение. Это второе предложение. Это третье предложение. ", 4), "zh", "ru") + if !zhHealthy.cls.ok() { + t.Fatalf("zh healthy ~3.5× translation must pass, got %+v", zhHealthy) + } + + // ja: sentence coverage 0.25 flags even though len_ratio (1.52) is inside [1.4,2.6]. + jaExcised := coverageCheck(boevoyGate(0), + "最初の文。二番目の文。三番目の文。四番目の文。", "Только одно предложение вместо четырёх.", "ja", "ru") + if jaExcised.cls.Reason != FlagExcisionSuspect || !strings.Contains(jaExcised.cls.Detail, "sent_cov") { + t.Fatalf("ja must flag on sentence coverage alone, got %+v", jaExcised) + } + if strings.Contains(jaExcised.cls.Detail, "len_ratio") { + t.Fatalf("ja len_ratio 1.52 is inside the corridor — must NOT be in the detail: %q", jaExcised.cls.Detail) + } +} + +func TestCoverageCheckApplicabilityAndCorridor(t *testing.T) { + // Below min_chunk_chars → gate is NOT applied (short-chunk ratios are noise). + short := coverageCheck(boevoyGate(500), "这是一个测试。", "x", "zh", "ru") + if short.applied || !short.cls.ok() { + t.Fatalf("a chunk shorter than min_chunk_chars must be skipped, got %+v", short) + } + + // No corridor for the pair → len_ratio is skipped but sentence coverage still + // applies (never silently pass everything). + noCorridor := coverageCheck(boevoyGate(0), + "가나。다라。마바。사아。", "Одно предложение.", "ko", "ru") + if noCorridor.cls.Reason != FlagExcisionSuspect || strings.Contains(noCorridor.cls.Detail, "len_ratio") { + t.Fatalf("no-corridor pair must still flag on sent_cov (no len_ratio), got %+v", noCorridor) + } + + // Upper-bound anomaly (len_ratio > hi) is DELIBERATELY NOT flagged (D12 Q3): + // matching sentence count (sent_cov=1.0) but a very long per-sentence output so + // len_ratio ≫ 4.2. The gate must still pass — additions are Phase-2 territory. + over := coverageCheck(boevoyGate(0), + strings.Repeat("这是句子。", 10), + strings.Repeat("Это очень длинное предложение с большим количеством слов. ", 10), "zh", "ru") + if !over.cls.ok() { + t.Fatalf("an over-length output must NOT flag (upper bound is out of scope, D12 Q3), got %+v", over) + } + if over.lenRatio <= 4.2 { + t.Fatalf("test premise broken: len_ratio must exceed the upper bound to prove it is ignored, got %.2f", over.lenRatio) + } +} + +// TestCoverageGateCatchesArtificialExcision is the Phase-1 acceptance shape (§A): +// a mini-set of fragments with sentences excised from the translation — the gate +// must catch ≥90%. Built synthetically by dropping a fraction of the output +// sentences from an otherwise-faithful translation; the control set (no drop) must +// produce ZERO false flags (precision). Honest scope: this proves the MECHANISM at +// the calibrated thresholds; real-corpus ≥90% is the Polygon oracle + owner precision +// validation (D12 Q4). +func TestCoverageGateCatchesArtificialExcision(t *testing.T) { + gate := boevoyGate(0) + // (srcLang, source sentences, faithful ru sentences) triples, ≥ ~40 sentences + // each so a fractional drop is meaningful. + build := func(unit string, n int) string { return strings.Repeat(unit, n) } + fixtures := []struct { + lang string + src string + full string + }{ + {"zh", build("这是一个相当长的句子用于测试覆盖率。", 40), build("Это довольно длинное предложение для проверки покрытия. ", 40)}, + {"ja", build("これは覆盖率を検証するための長い文である。", 40), build("Это длинное предложение для проверки покрытия текста. ", 40)}, + {"en", build("This is a reasonably long sentence used to test coverage. ", 40), build("Это разумно длинное предложение для проверки покрытия. ", 40)}, + } + + dropFractions := []float64{0.30, 0.40, 0.50, 0.60} // >25% → sent_cov < 0.75 + + caught, total := 0, 0 + for _, f := range fixtures { + outSents := splitSentences(f.full) + for _, frac := range dropFractions { + keep := int(float64(len(outSents)) * (1 - frac)) + excised := strings.Join(outSents[:keep], " ") + res := coverageCheck(gate, f.src, excised, f.lang, "ru") + total++ + if res.cls.Reason == FlagExcisionSuspect { + caught++ + } else { + t.Logf("MISS: %s drop=%.0f%% sent_cov=%.2f len_ratio=%.2f", f.lang, frac*100, res.sentCov, res.lenRatio) + } + } + // Control: the faithful full translation must NOT flag (precision). + if res := coverageCheck(gate, f.src, f.full, f.lang, "ru"); res.cls.Reason == FlagExcisionSuspect { + t.Errorf("FALSE POSITIVE on faithful %s translation: %s (sent_cov=%.2f len_ratio=%.2f)", f.lang, res.cls.Detail, res.sentCov, res.lenRatio) + } + } + rate := float64(caught) / float64(total) + if rate < 0.90 { + t.Fatalf("coverage gate caught %d/%d (%.0f%%) artificial excisions — below the ≥90%% acceptance bar", caught, total, rate*100) + } + t.Logf("mini-set: caught %d/%d (%.0f%%) artificial excisions", caught, total, rate*100) +} diff --git a/backend/internal/pipeline/runner.go b/backend/internal/pipeline/runner.go index ef403f9..6ec436e 100644 --- a/backend/internal/pipeline/runner.go +++ b/backend/internal/pipeline/runner.go @@ -130,6 +130,55 @@ type contextSnap struct { CacheTTL string `json:"cache_ttl"` } +// coverageSnap freezes the excision coverage-gate config inside the snapshot, so +// enabling the gate OR tuning its thresholds is a loud --resnapshot that re-derives +// every chunk's verdict from its checkpoint for free (checkpoint-hit → re-classify, +// no re-bill), never a silent mismatch between an old checkpoint's stored disposition +// and a changed gate. Only folded when ENABLED: tweaking a disabled gate's bounds must +// not force a re-pin (the gate produces no verdicts while off). json.Marshal sorts the +// LenRatio map keys → deterministic render. Mirrors the estimator/max_tokens-policy +// discipline (render.go) for a NON-wire input that still determines a checkpoint's +// resolved verdict. +type coverageSnap struct { + Enabled bool `json:"enabled"` + Version string `json:"version,omitempty"` + LenRatio map[string][]float64 `json:"len_ratio,omitempty"` + SentCovMin float64 `json:"sent_cov_min,omitempty"` + MinChunkChars int `json:"min_chunk_chars,omitempty"` +} + +// coverageSnapshot renders the gate's snapshot component: {enabled:false} when off +// (so toggling on is a visible change), the full config + algorithm version when on. +func (r *Runner) coverageSnapshot() coverageSnap { + cov := coverageSnap{Enabled: r.Pipeline.Gates.Coverage.Enabled} + if cov.Enabled { + cov.Version = coverageGateVersion + cov.LenRatio = r.Pipeline.Gates.Coverage.LenRatio + cov.SentCovMin = r.Pipeline.Gates.Coverage.SentCovMin + cov.MinChunkChars = r.Pipeline.Gates.Coverage.MinChunkChars + } + return cov +} + +// classifyOutput resolves a completion's disposition: FIRST the always-on intrinsic +// classifier (echo/empty/refusal/length — Веха 2), then, only if that is ok AND the +// configurable coverage gate is enabled, the excision gate (шаг 6 / D12 Q3). Both are +// pure and deterministic over (source, output, finish), so a resumed checkpoint +// reproduces the identical verdict for free — the intrinsic part unconditionally, the +// gate part under the same coverage config (folded into the snapshot, so a gate change +// re-pins loudly). The gate compares the output against the ORIGINAL source (ch.Text) +// at EVERY stage, so excision introduced by the draft OR the editor is caught. +func (r *Runner) classifyOutput(source, output, finish string) classification { + cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang}) + if !cls.ok() || !r.Pipeline.Gates.Coverage.Enabled { + return cls + } + if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() { + return cov.cls + } + return cls +} + // memoryVersion is the content-hash of the deterministically materialized // injected memory (approved glossary + summaries + series-bible), the memory // component of the snapshot (D5.2/D8). Until the memory bank v2 migration lands @@ -204,8 +253,12 @@ func (r *Runner) snapshotID() (id, payload string, err error) { // переоплаты D5.2. До миграции банка памяти (v2) материализация пуста — // хэш стабилен; когда память войдёт в msgs, поле сменится и resnapshot- // гейт сработает громко. STM сюда НЕ входит (пересобирается из чекпоинтов). - MemoryVersion string `json:"memory_version"` - Stages []stageSnap `json:"stages"` + MemoryVersion string `json:"memory_version"` + // Coverage — excision QA-gate config (D12 Q3). It does not touch the wire, + // but it determines a checkpoint's RESOLVED verdict, so a gate change must be + // a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot). + Coverage coverageSnap `json:"coverage"` + Stages []stageSnap `json:"stages"` }{ BriefHash: r.Book.BriefHash(), ChunkerVersion: chunkerVersion, @@ -222,6 +275,7 @@ func (r *Runner) snapshotID() (id, payload string, err error) { CacheTTL: r.Pipeline.Context.CacheTTL, }, MemoryVersion: r.memoryVersion(), + Coverage: r.coverageSnapshot(), } for _, st := range r.Pipeline.Stages { ss := stageSnap{ @@ -599,7 +653,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, snapID string, att.finish = cp.FinishReason att.modelActual = cp.ModelActual att.cumCost = cp.CostUSD - att.cls = classify(classifyInput{Source: ch.Text, Output: cp.ResponseText, Finish: cp.FinishReason, TargetLang: r.Book.TargetLang}) + att.cls = r.classifyOutput(ch.Text, cp.ResponseText, cp.FinishReason) r.Store.LogRequest(ctx, r.Log, store.RequestLog{ BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: cp.ModelActual, @@ -744,7 +798,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, snapID string, // non-empty truncated length draft is now classified (flagged/retried), never // silently passed downstream as OK; an empty completion is flagged too, not a // run-crash. - att.cls = classify(classifyInput{Source: ch.Text, Output: resp.Text, Finish: resp.FinishReason, TargetLang: r.Book.TargetLang}) + att.cls = r.classifyOutput(ch.Text, resp.Text, resp.FinishReason) r.Store.LogRequest(ctx, r.Log, store.RequestLog{ BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, diff --git a/backend/internal/pipeline/runner_test.go b/backend/internal/pipeline/runner_test.go index 163d12e..710caf8 100644 --- a/backend/internal/pipeline/runner_test.go +++ b/backend/internal/pipeline/runner_test.go @@ -104,6 +104,7 @@ type projectOpts struct { regenerate int minMaxTokens int bookUSD float64 + gatesYAML string // optional gates:/... block appended to pipeline.yaml } func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string { @@ -146,7 +147,7 @@ retries: { regenerate_before_escalate: %d } stages: - { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" } - { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" } -`, o.minMaxTokens, o.regenerate)) +%s`, o.minMaxTokens, o.regenerate, o.gatesYAML)) writeFile(t, filepath.Join(dir, "source.txt"), o.source) writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(` @@ -811,3 +812,152 @@ func TestRunnerLegacyEmptyCheckpointSelfHeal(t *testing.T) { t.Fatalf("chunk_status must be rebuilt as flagged, got %+v", cs) } } + +// --- Веха 2.5: excision coverage-gate (A, D12 Q3) --------------------------- + +// coverageGateBlock enables the gate with the boevoy ja-ru corridor; min_chunk_chars +// is low so the test fixtures qualify. +const coverageGateBlock = ` +gates: + coverage: + enabled: true + len_ratio_bounds: { ja-ru: [1.4, 2.6] } + sent_cov_min: 0.75 + min_chunk_chars: 100 +` + +// A draft that excises a long source into one short sentence must be flagged +// excision_suspect (non-retryable → edit skipped, exit 2); resume reproduces the +// verdict from chunk_status without re-attacking. +func TestRunnerCoverageGateFlagsExcision(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + if isEditBody(body) { + return "ред", "stop" + } + return "Короткий огрызок перевода.", "stop" // 30 source sentences → 1 + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{ + source: strings.Repeat("これは長い文である。", 30), regenerate: 1, gatesYAML: coverageGateBlock, + }) + ctx := context.Background() + + r1 := newRunner(t, bookPath) + res1, err := r1.TranslateBook(ctx) + if err != nil { + t.Fatalf("an excised chunk must flag, not crash: %v", err) + } + if rec.count() != 1 { + t.Fatalf("edit must be skipped after an excision flag, calls=%d", rec.count()) + } + ch := res1.Chunks[0] + if ch.Disposition != DispFlagged || ch.FlagReason != FlagExcisionSuspect { + t.Fatalf("draft must be flagged excision_suspect, got %+v", ch) + } + if ch.Stages[0].Attempts != 1 { + t.Fatalf("excision_suspect is non-retryable — exactly one attempt, got %d", ch.Stages[0].Attempts) + } + if res1.ExitCode() != 2 { + t.Fatalf("exit code must be 2, got %d", res1.ExitCode()) + } + r1.Close() + + r2 := newRunner(t, bookPath) + defer r2.Close() + res2, err := r2.TranslateBook(ctx) + if err != nil { + t.Fatal(err) + } + if rec.count() != 1 { + t.Fatalf("resume must not re-attack the excised chunk, calls=%d", rec.count()) + } + if res2.Chunks[0].FlagReason != FlagExcisionSuspect { + t.Fatalf("resume verdict differs: %+v", res2.Chunks[0]) + } +} + +// A faithful, full-length translation must pass the gate — both stages run, no flag +// (precision: the gate must not false-flag a healthy chunk). +func TestRunnerCoverageGatePassesHealthy(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, func(body string) (string, string) { + // The editor is ALSO gated against the original source (excision can happen + // at edit too), so it must return a full-length polish, not a short stub. + if isEditBody(body) { + return strings.Repeat("Правленое предложение перевода. ", 30), "stop" + } + return strings.Repeat("Это предложение перевода. ", 30), "stop" // 30 sentences ≈ source + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{ + source: strings.Repeat("これは長い文である。", 30), regenerate: 1, gatesYAML: coverageGateBlock, + }) + + r := newRunner(t, bookPath) + defer r.Close() + res, err := r.TranslateBook(context.Background()) + if err != nil { + t.Fatal(err) + } + if rec.count() != 2 { + t.Fatalf("a healthy chunk must run both stages, calls=%d", rec.count()) + } + if res.Flagged != 0 || res.ExitCode() != 0 || res.Chunks[0].Disposition != DispOK { + t.Fatalf("healthy chunk must pass the gate, got %+v", res.Chunks[0]) + } +} + +// The gate config is folded into the snapshot: enabling it on an in-progress book +// changes the snapshot, so resume fails loud until --resnapshot (else already-done +// chunks would silently never be re-checked by the now-on gate). +func TestRunnerCoverageGateEntersSnapshot(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, draftEdit) + defer srv.Close() + bookPath := setupProject(t, srv.URL) // gate OFF + ctx := context.Background() + + r1 := newRunner(t, bookPath) + if _, err := r1.TranslateBook(ctx); err != nil { + t.Fatal(err) + } + r1.Close() + + pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml") + raw, err := os.ReadFile(pipePath) + if err != nil { + t.Fatal(err) + } + writeFile(t, pipePath, string(raw)+coverageGateBlock) + + r2 := newRunner(t, bookPath) + _, err = r2.TranslateBook(ctx) + r2.Close() + if err == nil || !strings.Contains(err.Error(), "resnapshot") { + t.Fatalf("enabling the coverage gate must fail loud mentioning --resnapshot, got: %v", err) + } +} + +// An enabled gate with NO usable thresholds is a silent no-op (passes everything) — +// LoadPipeline must fail-fast rather than fail-open (against Р7). +func TestRunnerRejectsCoverageGateWithoutThresholds(t *testing.T) { + rec := &reqRec{} + srv := newJSONProvider(rec, draftEdit) + defer srv.Close() + bookPath := setupProject(t, srv.URL) + + pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml") + raw, err := os.ReadFile(pipePath) + if err != nil { + t.Fatal(err) + } + writeFile(t, pipePath, string(raw)+"\ngates:\n coverage:\n enabled: true\n") + + if _, err := NewRunner(bookPath, obs.NewLogger()); err == nil || !strings.Contains(err.Error(), "silently pass") { + t.Fatalf("an enabled gate with no thresholds must fail-fast, got: %v", err) + } + if rec.count() != 0 { + t.Fatalf("a rejected config must not reach the provider, calls=%d", rec.count()) + } +} diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 47b6c41..04bcc62 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -306,7 +306,15 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор **Тесты (мутационно-проверенные):** `TestDeepSeekEchoMineRejected` — 5 кейсов (2 поверхности + `effort` + GLM-безопасность + дефолт); `TestBoevoyConfigDeepSeekThinkingStaysOn` — гард на реальном `configs/models.yaml` (флаг на месте + `deepseek-v4-flash`→`ReasoningNone`; unmarshal напрямую, чтобы обойти date-бомбу `prices_checked`). Убрать `echoMineViolation` — armed-кейсы перестают падать (мутация ловится). -Дальше по §7 — контрактные вопросы владельцу перед coverage-гейтом/эскалацией (сегментация 1:1 vs §3.7, скоуп эскалации, скоуп coverage_fail/D10, порядок). +Контрактные вопросы (§7) заданы → оркестратор ответил **D12 (все A: 1A наивный split, 2A single-hop, 3A только excision_suspect, 4A opt-in)** + модель устойчивости (три класса отказа, F3 at-most-once, `supports_idempotency_key`, editor-pinned, global call-budget). Строю по D12. + +### 2026-07-04 — Веха 2.5 (A): coverage-гейт вырезания (excision_suspect) + +Порт excision-классификации `refusal_bench.py::classify_output` в `pipeline/coverage.go`. **Сегментация (D12 Q1) — бит-в-бит с оракулом:** RE2 без lookbehind, поэтому `split_sentences` реплицирован ручным сканом (branch-1 «терминатор+пробел-ран поглощается» → branch-2 «CJK-терминатор zero-width»), **пинится к живому оракулу** `TestSplitSentencesOracleParity` (28 крайних кейсов сняты из `refusal_bench`: диалоги `「…！」`, подряд `。。。`→3, U+3000, `……`→1, fullwidth `？。！`, `Mr.`-oversplit). Метрика — символы без пробелов. **Только нижняя граница (D12 Q3):** `sent_cov