Add the configurable excision coverage-gate porting refusal_bench sentence-coverage bit-for-bit, emitting excision_suspect and folded into the snapshot
This commit is contained in:
Claude (backend session)
parent
d15cdb3b71
commit
8957046424
7 changed files with 613 additions and 16 deletions
|
|
@ -16,7 +16,9 @@ func TestCheckRunnableRejectsPhase2Mechanics(t *testing.T) {
|
|||
{"c2 rejected", Pipeline{Core: "C2", Stages: []Stage{{Name: "draft", Role: "translator"}}}, "C2"},
|
||||
{"fanout rejected", Pipeline{Core: "C1", Fanout: Fanout{Candidates: 3}, Stages: []Stage{{Name: "draft", Role: "translator"}}}, "fanout"},
|
||||
{"judge rejected", Pipeline{Core: "C1", Stages: []Stage{{Name: "select", Role: "judge"}}}, "judge"},
|
||||
{"enabled gate rejected", Pipeline{Core: "C1", Gates: Gates{Coverage: CoverageGate{Enabled: true}}, Stages: []Stage{{Name: "draft", Role: "translator"}}}, "gates"},
|
||||
// The coverage gate is EXECUTABLE from Веха 2.5 — CheckRunnable no longer
|
||||
// rejects it (threshold sanity is LoadPipeline's job, not a mechanics gate).
|
||||
{"enabled gate runnable", Pipeline{Core: "C1", Gates: Gates{Coverage: CoverageGate{Enabled: true}}, Stages: []Stage{{Name: "draft", Role: "translator"}}}, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
err := c.p.CheckRunnable()
|
||||
|
|
@ -39,9 +41,9 @@ func TestCheckKeysOnlyUsedNonLocalModels(t *testing.T) {
|
|||
"local": {Kind: "local", BaseURL: "http://127.0.0.1", Model: "q"},
|
||||
},
|
||||
Models: map[string]Model{
|
||||
"cloud-model": {Provider: "cloud"},
|
||||
"local-model": {Provider: "local"},
|
||||
"unused-cloud": {Provider: "cloud"},
|
||||
"cloud-model": {Provider: "cloud"},
|
||||
"local-model": {Provider: "local"},
|
||||
"unused-cloud": {Provider: "cloud"},
|
||||
},
|
||||
}
|
||||
// local-only pipeline не должен падать на облачном ключе.
|
||||
|
|
|
|||
|
|
@ -114,12 +114,10 @@ func (p *Pipeline) CheckRunnable() error {
|
|||
if p.Fanout.Candidates > 1 {
|
||||
return fmt.Errorf("pipeline fanout.candidates=%d не исполним в Фазе 0 (fan-out N кандидатов — механика раннера Фазы 2; C1 = один черновик)", p.Fanout.Candidates)
|
||||
}
|
||||
// QA-гейты — механика Фазы 1; раннер Фазы 0 их НЕ исполняет. Включённый
|
||||
// гейт молча пропустил бы негейченный вывод как done (против Р7) —
|
||||
// поэтому fail-loud, а не fail-open (находка внешнего ревью F5).
|
||||
if p.gatesEnabled() {
|
||||
return fmt.Errorf("pipeline gates включены, но не исполнимы в Фазе 0 (QA-гейты — механика Фазы 1; иначе вывод пройдёт негейченным и будет ложно помечен done)")
|
||||
}
|
||||
// Coverage QA-гейт исполним с Вехи 2.5 (excision-детект, coverage.go): раннер
|
||||
// считает его по результату стадии и эмитит excision_suspect. Бывший fail-loud
|
||||
// на gates.enabled снят; корректность порогов включённого гейта проверяет
|
||||
// LoadPipeline (иначе тихий no-op — против Р7).
|
||||
for _, st := range p.Stages {
|
||||
if st.Role == "judge" {
|
||||
return fmt.Errorf("pipeline stage %q role=judge не исполним в Фазе 0 (селектор получает N кандидатов — механика Фазы 2, а раннер гонит стадии линейно)", st.Name)
|
||||
|
|
@ -207,6 +205,21 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) {
|
|||
}
|
||||
}
|
||||
}
|
||||
// Coverage QA-gate thresholds (шаг 6): an ENABLED gate must be able to flag
|
||||
// something, otherwise it is a silent no-op that passes every chunk (against Р7).
|
||||
if cov := p.Gates.Coverage; cov.Enabled {
|
||||
if cov.SentCovMin < 0 || cov.SentCovMin > 1 {
|
||||
bad("gates.coverage.sent_cov_min must be within [0,1], got %v", cov.SentCovMin)
|
||||
}
|
||||
for pair, b := range cov.LenRatio {
|
||||
if len(b) != 2 || b[0] <= 0 || b[1] < b[0] {
|
||||
bad("gates.coverage.len_ratio_bounds[%q] must be [low, high] with 0 < low <= high, got %v", pair, b)
|
||||
}
|
||||
}
|
||||
if cov.SentCovMin <= 0 && len(cov.LenRatio) == 0 {
|
||||
bad("gates.coverage.enabled is true but neither sent_cov_min nor len_ratio_bounds is set — the gate would silently pass every chunk")
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||||
}
|
||||
|
|
|
|||
169
backend/internal/pipeline/coverage.go
Normal file
169
backend/internal/pipeline/coverage.go
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"textmachine/backend/internal/config"
|
||||
)
|
||||
|
||||
// coverage.go: the configurable EXCISION coverage-gate (шаг 6 / D12 Q3). Unlike the
|
||||
// intrinsic classify() robustness (echo/empty/refusal — always on), this gate is
|
||||
// opt-in (gates.coverage.enabled) and threshold-driven: it flags a stage OUTPUT
|
||||
// that covers too LITTLE of the ORIGINAL source (silent excision — куски пропали,
|
||||
// HTTP 200). It is a pure, deterministic RUNNER-side check (not an adapter concern —
|
||||
// the adapter is translation-neutral, Q1/D2.4): given the checkpointed output it
|
||||
// reproduces the same verdict on resume without re-billing.
|
||||
//
|
||||
// Ported 1:1 from eval/refusal_bench.py::classify_output's excision branch, which is
|
||||
// BOTH our reference AND the Polygon acceptance oracle (D12 Q1). Metric = characters
|
||||
// WITHOUT spaces (not tokens, §3.7). Only the LOWER bound is checked: sent_cov <
|
||||
// sent_cov_min OR len_ratio < corridor-low → excision_suspect. The UPPER bound
|
||||
// (anomaly/дописывание) is deliberately NOT flagged here (D12 Q3: ru output is
|
||||
// 1.4–1.9× the source, so an upper cut false-flags; catching additions needs an
|
||||
// entity in/out count + a bilingual judge — Phase 2, §04-unhappy mode 2).
|
||||
|
||||
// coverageGateVersion versions the segmentation + metric of this gate. It is folded
|
||||
// into the job snapshot (like chunkerVersion/estimatorVersion), so a change to the
|
||||
// algorithm — or turning the gate ON — is a loud --resnapshot that re-evaluates every
|
||||
// chunk from its checkpoint for FREE (no re-bill), never a silent divergence between
|
||||
// an old checkpoint's stored verdict and a new gate. The segmentation is pinned 1:1
|
||||
// to eval/refusal_bench.py (D12 Q1); any change is Python-first, in lock-step, and
|
||||
// bumps this constant.
|
||||
const coverageGateVersion = "coverage-v1-naive-split-lower-bound"
|
||||
|
||||
// isSentenceTerminator is the branch-1 lookbehind class of refusal_bench's
|
||||
// SENT_SPLIT_RE `[.!?…。！？]` (Latin + ellipsis + fullwidth CJK): a whitespace run
|
||||
// after any of these splits a sentence.
|
||||
func isSentenceTerminator(r rune) bool {
|
||||
switch r {
|
||||
case '.', '!', '?', '…', '。', '！', '？':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isCJKSentenceTerminator is the branch-2 lookbehind class `[。！？]` (fullwidth CJK
|
||||
// only): these split ZERO-WIDTH, with or without a following space, because CJK prose
|
||||
// has no inter-sentence space.
|
||||
func isCJKSentenceTerminator(r rune) bool {
|
||||
switch r {
|
||||
case '。', '！', '？':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// splitSentences ports refusal_bench.py::split_sentences 1:1. Go's RE2 has no
|
||||
// lookbehind, so the regex `(?<=[.!?…。！？])\s+|(?<=[。！？])` is reproduced by a manual
|
||||
// left-to-right scan (branch-1 before branch-2, matching Python's alternation order):
|
||||
//
|
||||
// (1) a whitespace RUN preceded by ANY terminator — the run is the separator and
|
||||
// is consumed; then
|
||||
// (2) a ZERO-WIDTH point preceded by a CJK terminator — nothing consumed.
|
||||
//
|
||||
// Each raw piece is TrimSpace'd and empty pieces dropped (the Python list-comp).
|
||||
// Pinned bit-for-bit to the live oracle by TestSplitSentencesOracleParity.
|
||||
func splitSentences(text string) []string {
|
||||
runes := []rune(text)
|
||||
var segs []string
|
||||
emit := func(a, b int) {
|
||||
if s := strings.TrimSpace(string(runes[a:b])); s != "" {
|
||||
segs = append(segs, s)
|
||||
}
|
||||
}
|
||||
segStart := 0
|
||||
for i := 1; i < len(runes); i++ {
|
||||
prev := runes[i-1]
|
||||
// Branch 1: any terminator, then a whitespace run (the run is consumed).
|
||||
if isSentenceTerminator(prev) && unicode.IsSpace(runes[i]) {
|
||||
emit(segStart, i)
|
||||
j := i
|
||||
for j < len(runes) && unicode.IsSpace(runes[j]) {
|
||||
j++
|
||||
}
|
||||
segStart = j
|
||||
i = j - 1 // the loop's i++ lands on j
|
||||
continue
|
||||
}
|
||||
// Branch 2: CJK terminator, zero-width split (the terminator stays with the
|
||||
// left piece; scanning continues from the same char, now the segment start).
|
||||
if isCJKSentenceTerminator(prev) {
|
||||
emit(segStart, i)
|
||||
segStart = i
|
||||
}
|
||||
}
|
||||
emit(segStart, len(runes))
|
||||
return segs
|
||||
}
|
||||
|
||||
// nonSpaceCount is the coverage metric's unit — runes that are NOT whitespace
|
||||
// (refusal_bench: `sum(1 for c in text if not c.isspace())`). Characters, not tokens.
|
||||
func nonSpaceCount(s string) int {
|
||||
n := 0
|
||||
for _, r := range s {
|
||||
if !unicode.IsSpace(r) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// langPairKey builds the `len_ratio_bounds` config key from the book's languages,
|
||||
// e.g. ("zh","ru") → "zh-ru". Lower-cased and trimmed so casing in book.yaml does
|
||||
// not miss the corridor.
|
||||
func langPairKey(srcLang, dstLang string) string {
|
||||
return strings.ToLower(strings.TrimSpace(srcLang)) + "-" + strings.ToLower(strings.TrimSpace(dstLang))
|
||||
}
|
||||
|
||||
// coverageResult carries the metric values alongside the verdict for telemetry/tests.
|
||||
type coverageResult struct {
|
||||
cls classification
|
||||
applied bool // false when skipped (too short / disabled)
|
||||
sentCov float64 // sentences(out)/sentences(src)
|
||||
lenRatio float64 // nonspace(out)/nonspace(src)
|
||||
}
|
||||
|
||||
// coverageCheck runs the excision gate on ONE stage output against the ORIGINAL
|
||||
// source. It is pair-aware: len_ratio is judged against the corridor for
|
||||
// srcLang-dstLang; sentence coverage is pair-independent. A source shorter than
|
||||
// MinChunkChars (non-space) is not gated — the corridors are calibrated on
|
||||
// 500–2500-char fragments and short chunks' ratios are noise (§3.7). Pure &
|
||||
// deterministic. Returns excision_suspect (lower-bound only) or ok.
|
||||
func coverageCheck(cfg config.CoverageGate, src, out, srcLang, dstLang string) coverageResult {
|
||||
srcChars := nonSpaceCount(src)
|
||||
if srcChars < cfg.MinChunkChars {
|
||||
return coverageResult{cls: classification{reasonOK, ""}, applied: false}
|
||||
}
|
||||
|
||||
nSrc := len(splitSentences(src))
|
||||
sentCov := 1.0
|
||||
if nSrc > 0 {
|
||||
sentCov = float64(len(splitSentences(out))) / float64(nSrc)
|
||||
}
|
||||
denom := srcChars
|
||||
if denom < 1 {
|
||||
denom = 1
|
||||
}
|
||||
lenRatio := float64(nonSpaceCount(out)) / float64(denom)
|
||||
|
||||
res := coverageResult{applied: true, sentCov: sentCov, lenRatio: lenRatio}
|
||||
var flags []string
|
||||
if sentCov < cfg.SentCovMin {
|
||||
flags = append(flags, fmt.Sprintf("sent_cov=%.2f<%.2f", sentCov, cfg.SentCovMin))
|
||||
}
|
||||
// len_ratio needs a corridor for this pair; without one we still apply sentence
|
||||
// coverage (pair-independent) rather than silently pass everything.
|
||||
if bounds, ok := cfg.LenRatio[langPairKey(srcLang, dstLang)]; ok && len(bounds) >= 1 {
|
||||
if lo := bounds[0]; lenRatio < lo {
|
||||
flags = append(flags, fmt.Sprintf("len_ratio=%.2f<%.2f", lenRatio, lo))
|
||||
}
|
||||
}
|
||||
if len(flags) > 0 {
|
||||
res.cls = classification{FlagExcisionSuspect, strings.Join(flags, "; ")}
|
||||
return res
|
||||
}
|
||||
res.cls = classification{reasonOK, ""}
|
||||
return res
|
||||
}
|
||||
201
backend/internal/pipeline/coverage_test.go
Normal file
201
backend/internal/pipeline/coverage_test.go
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/config"
|
||||
)
|
||||
|
||||
// coverage_test.go pins the excision coverage-gate. The segmentation/metric is
|
||||
// PINNED BIT-FOR-BIT to the Polygon acceptance oracle (eval/refusal_bench.py,
|
||||
// D12 Q1): the expected values below were captured from the LIVE oracle
|
||||
// (scripts/oracle_groundtruth.py over refusal_bench.split_sentences/cjk_share). If
|
||||
// the oracle changes, this table must change Python-first, in lock-step — that is
|
||||
// the contract, not an accident.
|
||||
|
||||
// oracleSplit is the ground-truth from refusal_bench.py::split_sentences.
|
||||
var oracleSplit = []struct {
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{"", nil},
|
||||
{" ", nil},
|
||||
{"Hello world", []string{"Hello world"}},
|
||||
{"Hello. World.", []string{"Hello.", "World."}},
|
||||
{"Hello.World.", []string{"Hello.World."}}, // no space after Latin '.' → no split
|
||||
{"Hello!! World?? Yes.", []string{"Hello!!", "World??", "Yes."}},
|
||||
{"One. Two! Three? Four…", []string{"One.", "Two!", "Three?", "Four…"}},
|
||||
{"Wait… really?", []string{"Wait…", "really?"}},
|
||||
{"猫。", []string{"猫。"}},
|
||||
{"猫。狗。鸟。", []string{"猫。", "狗。", "鸟。"}},
|
||||
{"猫。 狗。 鸟。", []string{"猫。", "狗。", "鸟。"}},
|
||||
{"猫。。狗", []string{"猫。", "。", "狗"}}, // consecutive CJK terminators → lone 。 fragment
|
||||
{"「こんにちは！」彼は笑った。", []string{"「こんにちは！", "」彼は笑った。"}}, // ！ splits zero-width, 」 joins next
|
||||
{"『引用』。次の文。", []string{"『引用』。", "次の文。"}},
|
||||
{"«Привет!» — он засмеялся. «Пока».", []string{"«Привет!» — он засмеялся.", "«Пока»."}}, // !» does NOT split
|
||||
{"“Hi!” he said. “Bye.”", []string{"“Hi!” he said.", "“Bye.”"}},
|
||||
{"阿Q正传。第一章。", []string{"阿Q正传。", "第一章。"}},
|
||||
{"彼は言った：「行こう！」", []string{"彼は言った：「行こう！", "」"}},
|
||||
{"はい。 いいえ。", []string{"はい。", "いいえ。"}}, // U+3000 ideographic space after 。 is consumed
|
||||
{"Yes.\tNo.\nMaybe.", []string{"Yes.", "No.", "Maybe."}},
|
||||
{"こんにちは、世界。さようなら！", []string{"こんにちは、世界。", "さようなら！"}}, // 、 is NOT a terminator
|
||||
{"……", []string{"……"}}, // Latin ellipsis, no whitespace → no split
|
||||
{"。。。", []string{"。", "。", "。"}}, // each CJK period splits zero-width
|
||||
{"A。B. C。D", []string{"A。", "B.", "C。", "D"}},
|
||||
{"Full？width。terminators！here", []string{"Full？", "width。", "terminators！", "here"}},
|
||||
{" leading and trailing. ", []string{"leading and trailing."}},
|
||||
{"第一。\n\n第二。", []string{"第一。", "第二。"}},
|
||||
{"Mr. Smith went. He left.", []string{"Mr.", "Smith went.", "He left."}}, // naive over-split (known)
|
||||
}
|
||||
|
||||
func TestSplitSentencesOracleParity(t *testing.T) {
|
||||
for _, c := range oracleSplit {
|
||||
got := splitSentences(c.in)
|
||||
if len(got) != len(c.want) {
|
||||
t.Errorf("splitSentences(%q) = %#v (n=%d), want %#v (n=%d)", c.in, got, len(got), c.want, len(c.want))
|
||||
continue
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != c.want[i] {
|
||||
t.Errorf("splitSentences(%q)[%d] = %q, want %q", c.in, i, got[i], c.want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonSpaceCountOracleParity(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want int
|
||||
}{
|
||||
{"Hello. World.", 12},
|
||||
{"猫。 狗。 鸟。", 6},
|
||||
{"Yes.\tNo.\nMaybe.", 13},
|
||||
{"はい。 いいえ。", 7}, // U+3000 counts as space → excluded
|
||||
{" leading and trailing. ", 19},
|
||||
{"", 0},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := nonSpaceCount(c.in); got != c.want {
|
||||
t.Errorf("nonSpaceCount(%q) = %d, want %d", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// boevoyBounds mirrors the boevoy pipeline-c1.yaml corridors.
|
||||
func boevoyGate(minChars int) config.CoverageGate {
|
||||
return config.CoverageGate{
|
||||
Enabled: true,
|
||||
LenRatio: map[string][]float64{"zh-ru": {2.2, 4.2}, "ja-ru": {1.4, 2.6}, "en-ru": {0.7, 1.4}},
|
||||
SentCovMin: 0.75,
|
||||
MinChunkChars: minChars,
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageCheckExcision(t *testing.T) {
|
||||
// Mirrors the three classify_output cases captured from the live oracle.
|
||||
zhExcised := coverageCheck(boevoyGate(0), strings.Repeat("这是一个测试。", 20), "Короткий перевод.", "zh", "ru")
|
||||
if zhExcised.cls.Reason != FlagExcisionSuspect {
|
||||
t.Fatalf("zh short output must flag excision, got %+v", zhExcised)
|
||||
}
|
||||
if zhExcised.sentCov > 0.1 || zhExcised.lenRatio > 0.2 {
|
||||
t.Fatalf("zh metrics off (oracle: sent_cov=0.05, len_ratio=0.114): %+v", zhExcised)
|
||||
}
|
||||
|
||||
zhHealthy := coverageCheck(boevoyGate(0),
|
||||
strings.Repeat("这是第一句。这是第二句。这是第三句。", 4),
|
||||
strings.Repeat("Это первое предложение. Это второе предложение. Это третье предложение. ", 4), "zh", "ru")
|
||||
if !zhHealthy.cls.ok() {
|
||||
t.Fatalf("zh healthy ~3.5× translation must pass, got %+v", zhHealthy)
|
||||
}
|
||||
|
||||
// ja: sentence coverage 0.25 flags even though len_ratio (1.52) is inside [1.4,2.6].
|
||||
jaExcised := coverageCheck(boevoyGate(0),
|
||||
"最初の文。二番目の文。三番目の文。四番目の文。", "Только одно предложение вместо четырёх.", "ja", "ru")
|
||||
if jaExcised.cls.Reason != FlagExcisionSuspect || !strings.Contains(jaExcised.cls.Detail, "sent_cov") {
|
||||
t.Fatalf("ja must flag on sentence coverage alone, got %+v", jaExcised)
|
||||
}
|
||||
if strings.Contains(jaExcised.cls.Detail, "len_ratio") {
|
||||
t.Fatalf("ja len_ratio 1.52 is inside the corridor — must NOT be in the detail: %q", jaExcised.cls.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageCheckApplicabilityAndCorridor(t *testing.T) {
|
||||
// Below min_chunk_chars → gate is NOT applied (short-chunk ratios are noise).
|
||||
short := coverageCheck(boevoyGate(500), "这是一个测试。", "x", "zh", "ru")
|
||||
if short.applied || !short.cls.ok() {
|
||||
t.Fatalf("a chunk shorter than min_chunk_chars must be skipped, got %+v", short)
|
||||
}
|
||||
|
||||
// No corridor for the pair → len_ratio is skipped but sentence coverage still
|
||||
// applies (never silently pass everything).
|
||||
noCorridor := coverageCheck(boevoyGate(0),
|
||||
"가나。다라。마바。사아。", "Одно предложение.", "ko", "ru")
|
||||
if noCorridor.cls.Reason != FlagExcisionSuspect || strings.Contains(noCorridor.cls.Detail, "len_ratio") {
|
||||
t.Fatalf("no-corridor pair must still flag on sent_cov (no len_ratio), got %+v", noCorridor)
|
||||
}
|
||||
|
||||
// Upper-bound anomaly (len_ratio > hi) is DELIBERATELY NOT flagged (D12 Q3):
|
||||
// matching sentence count (sent_cov=1.0) but a very long per-sentence output so
|
||||
// len_ratio ≫ 4.2. The gate must still pass — additions are Phase-2 territory.
|
||||
over := coverageCheck(boevoyGate(0),
|
||||
strings.Repeat("这是句子。", 10),
|
||||
strings.Repeat("Это очень длинное предложение с большим количеством слов. ", 10), "zh", "ru")
|
||||
if !over.cls.ok() {
|
||||
t.Fatalf("an over-length output must NOT flag (upper bound is out of scope, D12 Q3), got %+v", over)
|
||||
}
|
||||
if over.lenRatio <= 4.2 {
|
||||
t.Fatalf("test premise broken: len_ratio must exceed the upper bound to prove it is ignored, got %.2f", over.lenRatio)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoverageGateCatchesArtificialExcision is the Phase-1 acceptance shape (§A):
|
||||
// a mini-set of fragments with sentences excised from the translation — the gate
|
||||
// must catch ≥90%. Built synthetically by dropping a fraction of the output
|
||||
// sentences from an otherwise-faithful translation; the control set (no drop) must
|
||||
// produce ZERO false flags (precision). Honest scope: this proves the MECHANISM at
|
||||
// the calibrated thresholds; real-corpus ≥90% is the Polygon oracle + owner precision
|
||||
// validation (D12 Q4).
|
||||
func TestCoverageGateCatchesArtificialExcision(t *testing.T) {
|
||||
gate := boevoyGate(0)
|
||||
// (srcLang, source sentences, faithful ru sentences) triples, ≥ ~40 sentences
|
||||
// each so a fractional drop is meaningful.
|
||||
build := func(unit string, n int) string { return strings.Repeat(unit, n) }
|
||||
fixtures := []struct {
|
||||
lang string
|
||||
src string
|
||||
full string
|
||||
}{
|
||||
{"zh", build("这是一个相当长的句子用于测试覆盖率。", 40), build("Это довольно длинное предложение для проверки покрытия. ", 40)},
|
||||
{"ja", build("これは覆盖率を検証するための長い文である。", 40), build("Это длинное предложение для проверки покрытия текста. ", 40)},
|
||||
{"en", build("This is a reasonably long sentence used to test coverage. ", 40), build("Это разумно длинное предложение для проверки покрытия. ", 40)},
|
||||
}
|
||||
|
||||
dropFractions := []float64{0.30, 0.40, 0.50, 0.60} // >25% → sent_cov < 0.75
|
||||
|
||||
caught, total := 0, 0
|
||||
for _, f := range fixtures {
|
||||
outSents := splitSentences(f.full)
|
||||
for _, frac := range dropFractions {
|
||||
keep := int(float64(len(outSents)) * (1 - frac))
|
||||
excised := strings.Join(outSents[:keep], " ")
|
||||
res := coverageCheck(gate, f.src, excised, f.lang, "ru")
|
||||
total++
|
||||
if res.cls.Reason == FlagExcisionSuspect {
|
||||
caught++
|
||||
} else {
|
||||
t.Logf("MISS: %s drop=%.0f%% sent_cov=%.2f len_ratio=%.2f", f.lang, frac*100, res.sentCov, res.lenRatio)
|
||||
}
|
||||
}
|
||||
// Control: the faithful full translation must NOT flag (precision).
|
||||
if res := coverageCheck(gate, f.src, f.full, f.lang, "ru"); res.cls.Reason == FlagExcisionSuspect {
|
||||
t.Errorf("FALSE POSITIVE on faithful %s translation: %s (sent_cov=%.2f len_ratio=%.2f)", f.lang, res.cls.Detail, res.sentCov, res.lenRatio)
|
||||
}
|
||||
}
|
||||
rate := float64(caught) / float64(total)
|
||||
if rate < 0.90 {
|
||||
t.Fatalf("coverage gate caught %d/%d (%.0f%%) artificial excisions — below the ≥90%% acceptance bar", caught, total, rate*100)
|
||||
}
|
||||
t.Logf("mini-set: caught %d/%d (%.0f%%) artificial excisions", caught, total, rate*100)
|
||||
}
|
||||
|
|
@ -130,6 +130,55 @@ type contextSnap struct {
|
|||
CacheTTL string `json:"cache_ttl"`
|
||||
}
|
||||
|
||||
// coverageSnap freezes the excision coverage-gate config inside the snapshot, so
|
||||
// enabling the gate OR tuning its thresholds is a loud --resnapshot that re-derives
|
||||
// every chunk's verdict from its checkpoint for free (checkpoint-hit → re-classify,
|
||||
// no re-bill), never a silent mismatch between an old checkpoint's stored disposition
|
||||
// and a changed gate. Only folded when ENABLED: tweaking a disabled gate's bounds must
|
||||
// not force a re-pin (the gate produces no verdicts while off). json.Marshal sorts the
|
||||
// LenRatio map keys → deterministic render. Mirrors the estimator/max_tokens-policy
|
||||
// discipline (render.go) for a NON-wire input that still determines a checkpoint's
|
||||
// resolved verdict.
|
||||
type coverageSnap struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Version string `json:"version,omitempty"`
|
||||
LenRatio map[string][]float64 `json:"len_ratio,omitempty"`
|
||||
SentCovMin float64 `json:"sent_cov_min,omitempty"`
|
||||
MinChunkChars int `json:"min_chunk_chars,omitempty"`
|
||||
}
|
||||
|
||||
// coverageSnapshot renders the gate's snapshot component: {enabled:false} when off
|
||||
// (so toggling on is a visible change), the full config + algorithm version when on.
|
||||
func (r *Runner) coverageSnapshot() coverageSnap {
|
||||
cov := coverageSnap{Enabled: r.Pipeline.Gates.Coverage.Enabled}
|
||||
if cov.Enabled {
|
||||
cov.Version = coverageGateVersion
|
||||
cov.LenRatio = r.Pipeline.Gates.Coverage.LenRatio
|
||||
cov.SentCovMin = r.Pipeline.Gates.Coverage.SentCovMin
|
||||
cov.MinChunkChars = r.Pipeline.Gates.Coverage.MinChunkChars
|
||||
}
|
||||
return cov
|
||||
}
|
||||
|
||||
// classifyOutput resolves a completion's disposition: FIRST the always-on intrinsic
|
||||
// classifier (echo/empty/refusal/length — Веха 2), then, only if that is ok AND the
|
||||
// configurable coverage gate is enabled, the excision gate (шаг 6 / D12 Q3). Both are
|
||||
// pure and deterministic over (source, output, finish), so a resumed checkpoint
|
||||
// reproduces the identical verdict for free — the intrinsic part unconditionally, the
|
||||
// gate part under the same coverage config (folded into the snapshot, so a gate change
|
||||
// re-pins loudly). The gate compares the output against the ORIGINAL source (ch.Text)
|
||||
// at EVERY stage, so excision introduced by the draft OR the editor is caught.
|
||||
func (r *Runner) classifyOutput(source, output, finish string) classification {
|
||||
cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang})
|
||||
if !cls.ok() || !r.Pipeline.Gates.Coverage.Enabled {
|
||||
return cls
|
||||
}
|
||||
if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() {
|
||||
return cov.cls
|
||||
}
|
||||
return cls
|
||||
}
|
||||
|
||||
// memoryVersion is the content-hash of the deterministically materialized
|
||||
// injected memory (approved glossary + summaries + series-bible), the memory
|
||||
// component of the snapshot (D5.2/D8). Until the memory bank v2 migration lands
|
||||
|
|
@ -204,8 +253,12 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
|||
// переоплаты D5.2. До миграции банка памяти (v2) материализация пуста —
|
||||
// хэш стабилен; когда память войдёт в msgs, поле сменится и resnapshot-
|
||||
// гейт сработает громко. STM сюда НЕ входит (пересобирается из чекпоинтов).
|
||||
MemoryVersion string `json:"memory_version"`
|
||||
Stages []stageSnap `json:"stages"`
|
||||
MemoryVersion string `json:"memory_version"`
|
||||
// Coverage — excision QA-gate config (D12 Q3). It does not touch the wire,
|
||||
// but it determines a checkpoint's RESOLVED verdict, so a gate change must be
|
||||
// a loud --resnapshot, not a silent re-verdict on resume (coverageSnapshot).
|
||||
Coverage coverageSnap `json:"coverage"`
|
||||
Stages []stageSnap `json:"stages"`
|
||||
}{
|
||||
BriefHash: r.Book.BriefHash(),
|
||||
ChunkerVersion: chunkerVersion,
|
||||
|
|
@ -222,6 +275,7 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
|||
CacheTTL: r.Pipeline.Context.CacheTTL,
|
||||
},
|
||||
MemoryVersion: r.memoryVersion(),
|
||||
Coverage: r.coverageSnapshot(),
|
||||
}
|
||||
for _, st := range r.Pipeline.Stages {
|
||||
ss := stageSnap{
|
||||
|
|
@ -599,7 +653,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, snapID string,
|
|||
att.finish = cp.FinishReason
|
||||
att.modelActual = cp.ModelActual
|
||||
att.cumCost = cp.CostUSD
|
||||
att.cls = classify(classifyInput{Source: ch.Text, Output: cp.ResponseText, Finish: cp.FinishReason, TargetLang: r.Book.TargetLang})
|
||||
att.cls = r.classifyOutput(ch.Text, cp.ResponseText, cp.FinishReason)
|
||||
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
||||
Stage: st.Name, Role: st.Role, ModelRequested: st.Model, ModelActual: cp.ModelActual,
|
||||
|
|
@ -744,7 +798,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, snapID string,
|
|||
// non-empty truncated length draft is now classified (flagged/retried), never
|
||||
// silently passed downstream as OK; an empty completion is flagged too, not a
|
||||
// run-crash.
|
||||
att.cls = classify(classifyInput{Source: ch.Text, Output: resp.Text, Finish: resp.FinishReason, TargetLang: r.Book.TargetLang})
|
||||
att.cls = r.classifyOutput(ch.Text, resp.Text, resp.FinishReason)
|
||||
|
||||
r.Store.LogRequest(ctx, r.Log, store.RequestLog{
|
||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx,
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ type projectOpts struct {
|
|||
regenerate int
|
||||
minMaxTokens int
|
||||
bookUSD float64
|
||||
gatesYAML string // optional gates:/... block appended to pipeline.yaml
|
||||
}
|
||||
|
||||
func setupProjectOpts(t *testing.T, providerURL string, o projectOpts) string {
|
||||
|
|
@ -146,7 +147,7 @@ retries: { regenerate_before_escalate: %d }
|
|||
stages:
|
||||
- { name: draft, role: translator, model: fake-model, prompt: prompts/translator.md, prompt_version: v-test, temperature: 0.3, reasoning: "off" }
|
||||
- { name: edit, role: editor, model: fake-model, prompt: prompts/editor.md, prompt_version: v-test, temperature: 0.4, reasoning: "off" }
|
||||
`, o.minMaxTokens, o.regenerate))
|
||||
%s`, o.minMaxTokens, o.regenerate, o.gatesYAML))
|
||||
|
||||
writeFile(t, filepath.Join(dir, "source.txt"), o.source)
|
||||
writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(`
|
||||
|
|
@ -811,3 +812,152 @@ func TestRunnerLegacyEmptyCheckpointSelfHeal(t *testing.T) {
|
|||
t.Fatalf("chunk_status must be rebuilt as flagged, got %+v", cs)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Веха 2.5: excision coverage-gate (A, D12 Q3) ---------------------------
|
||||
|
||||
// coverageGateBlock enables the gate with the boevoy ja-ru corridor; min_chunk_chars
|
||||
// is low so the test fixtures qualify.
|
||||
const coverageGateBlock = `
|
||||
gates:
|
||||
coverage:
|
||||
enabled: true
|
||||
len_ratio_bounds: { ja-ru: [1.4, 2.6] }
|
||||
sent_cov_min: 0.75
|
||||
min_chunk_chars: 100
|
||||
`
|
||||
|
||||
// A draft that excises a long source into one short sentence must be flagged
|
||||
// excision_suspect (non-retryable → edit skipped, exit 2); resume reproduces the
|
||||
// verdict from chunk_status without re-attacking.
|
||||
func TestRunnerCoverageGateFlagsExcision(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
return "ред", "stop"
|
||||
}
|
||||
return "Короткий огрызок перевода.", "stop" // 30 source sentences → 1
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||||
source: strings.Repeat("これは長い文である。", 30), regenerate: 1, gatesYAML: coverageGateBlock,
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
res1, err := r1.TranslateBook(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("an excised chunk must flag, not crash: %v", err)
|
||||
}
|
||||
if rec.count() != 1 {
|
||||
t.Fatalf("edit must be skipped after an excision flag, calls=%d", rec.count())
|
||||
}
|
||||
ch := res1.Chunks[0]
|
||||
if ch.Disposition != DispFlagged || ch.FlagReason != FlagExcisionSuspect {
|
||||
t.Fatalf("draft must be flagged excision_suspect, got %+v", ch)
|
||||
}
|
||||
if ch.Stages[0].Attempts != 1 {
|
||||
t.Fatalf("excision_suspect is non-retryable — exactly one attempt, got %d", ch.Stages[0].Attempts)
|
||||
}
|
||||
if res1.ExitCode() != 2 {
|
||||
t.Fatalf("exit code must be 2, got %d", res1.ExitCode())
|
||||
}
|
||||
r1.Close()
|
||||
|
||||
r2 := newRunner(t, bookPath)
|
||||
defer r2.Close()
|
||||
res2, err := r2.TranslateBook(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.count() != 1 {
|
||||
t.Fatalf("resume must not re-attack the excised chunk, calls=%d", rec.count())
|
||||
}
|
||||
if res2.Chunks[0].FlagReason != FlagExcisionSuspect {
|
||||
t.Fatalf("resume verdict differs: %+v", res2.Chunks[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A faithful, full-length translation must pass the gate — both stages run, no flag
|
||||
// (precision: the gate must not false-flag a healthy chunk).
|
||||
func TestRunnerCoverageGatePassesHealthy(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
// The editor is ALSO gated against the original source (excision can happen
|
||||
// at edit too), so it must return a full-length polish, not a short stub.
|
||||
if isEditBody(body) {
|
||||
return strings.Repeat("Правленое предложение перевода. ", 30), "stop"
|
||||
}
|
||||
return strings.Repeat("Это предложение перевода. ", 30), "stop" // 30 sentences ≈ source
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||||
source: strings.Repeat("これは長い文である。", 30), regenerate: 1, gatesYAML: coverageGateBlock,
|
||||
})
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
res, err := r.TranslateBook(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec.count() != 2 {
|
||||
t.Fatalf("a healthy chunk must run both stages, calls=%d", rec.count())
|
||||
}
|
||||
if res.Flagged != 0 || res.ExitCode() != 0 || res.Chunks[0].Disposition != DispOK {
|
||||
t.Fatalf("healthy chunk must pass the gate, got %+v", res.Chunks[0])
|
||||
}
|
||||
}
|
||||
|
||||
// The gate config is folded into the snapshot: enabling it on an in-progress book
|
||||
// changes the snapshot, so resume fails loud until --resnapshot (else already-done
|
||||
// chunks would silently never be re-checked by the now-on gate).
|
||||
func TestRunnerCoverageGateEntersSnapshot(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, draftEdit)
|
||||
defer srv.Close()
|
||||
bookPath := setupProject(t, srv.URL) // gate OFF
|
||||
ctx := context.Background()
|
||||
|
||||
r1 := newRunner(t, bookPath)
|
||||
if _, err := r1.TranslateBook(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r1.Close()
|
||||
|
||||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||||
raw, err := os.ReadFile(pipePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFile(t, pipePath, string(raw)+coverageGateBlock)
|
||||
|
||||
r2 := newRunner(t, bookPath)
|
||||
_, err = r2.TranslateBook(ctx)
|
||||
r2.Close()
|
||||
if err == nil || !strings.Contains(err.Error(), "resnapshot") {
|
||||
t.Fatalf("enabling the coverage gate must fail loud mentioning --resnapshot, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An enabled gate with NO usable thresholds is a silent no-op (passes everything) —
|
||||
// LoadPipeline must fail-fast rather than fail-open (against Р7).
|
||||
func TestRunnerRejectsCoverageGateWithoutThresholds(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
srv := newJSONProvider(rec, draftEdit)
|
||||
defer srv.Close()
|
||||
bookPath := setupProject(t, srv.URL)
|
||||
|
||||
pipePath := filepath.Join(filepath.Dir(bookPath), "pipeline.yaml")
|
||||
raw, err := os.ReadFile(pipePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFile(t, pipePath, string(raw)+"\ngates:\n coverage:\n enabled: true\n")
|
||||
|
||||
if _, err := NewRunner(bookPath, obs.NewLogger()); err == nil || !strings.Contains(err.Error(), "silently pass") {
|
||||
t.Fatalf("an enabled gate with no thresholds must fail-fast, got: %v", err)
|
||||
}
|
||||
if rec.count() != 0 {
|
||||
t.Fatalf("a rejected config must not reach the provider, calls=%d", rec.count())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -306,7 +306,15 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор
|
|||
|
||||
**Тесты (мутационно-проверенные):** `TestDeepSeekEchoMineRejected` — 5 кейсов (2 поверхности + `effort` + GLM-безопасность + дефолт); `TestBoevoyConfigDeepSeekThinkingStaysOn` — гард на реальном `configs/models.yaml` (флаг на месте + `deepseek-v4-flash`→`ReasoningNone`; unmarshal напрямую, чтобы обойти date-бомбу `prices_checked`). Убрать `echoMineViolation` — armed-кейсы перестают падать (мутация ловится).
|
||||
|
||||
Дальше по §7 — контрактные вопросы владельцу перед coverage-гейтом/эскалацией (сегментация 1:1 vs §3.7, скоуп эскалации, скоуп coverage_fail/D10, порядок).
|
||||
Контрактные вопросы (§7) заданы → оркестратор ответил **D12 (все A: 1A наивный split, 2A single-hop, 3A только excision_suspect, 4A opt-in)** + модель устойчивости (три класса отказа, F3 at-most-once, `supports_idempotency_key`, editor-pinned, global call-budget). Строю по D12.
|
||||
|
||||
### 2026-07-04 — Веха 2.5 (A): coverage-гейт вырезания (excision_suspect)
|
||||
|
||||
Порт excision-классификации `refusal_bench.py::classify_output` в `pipeline/coverage.go`. **Сегментация (D12 Q1) — бит-в-бит с оракулом:** RE2 без lookbehind, поэтому `split_sentences` реплицирован ручным сканом (branch-1 «терминатор+пробел-ран поглощается» → branch-2 «CJK-терминатор zero-width»), **пинится к живому оракулу** `TestSplitSentencesOracleParity` (28 крайних кейсов сняты из `refusal_bench`: диалоги `「…！」`, подряд `。。。`→3, U+3000, `……`→1, fullwidth `？。！`, `Mr.`-oversplit). Метрика — символы без пробелов. **Только нижняя граница (D12 Q3):** `sent_cov<sent_cov_min` ИЛИ `len_ratio<коридор-low` → `excision_suspect`; верхняя (аномалия-дописывание) НЕ флагуется (ru 1.4–1.9×, ложно; нужен entity+судья Ф2). Мини-набор искусственных пропусков — **ловит 12/12 (100%)**, ложных на верных — 0.
|
||||
|
||||
**Вайринг раннера:** `classifyOutput` = intrinsic `classify` (эхо/пусто/refusal, всегда) → потом coverage-гейт **если** `gates.coverage.enabled`, по РЕЗУЛЬТАТУ каждой стадии против ИСХОДНИКА (ловит вырезание и на draft, и на edit). `excision_suspect` non-retryable (детерминирован per-модель → эскалация, не same-model retry — интент под B). Гейт-конфиг **свёрнут в snapshot** (`coverageSnapshot`, только когда enabled) → флип/твик = громкий `--resnapshot` + бесплатная ре-оценка чанков из чекпоинтов. `CheckRunnable` больше не валит `gates.enabled`; `LoadPipeline` fail-fast на включённом гейте без порогов (тихий no-op против Р7). **Боевой `pipeline-c1.yaml` — opt-in `enabled:false`** (D12 Q4: флип после валидации precision владельцем). Тесты: excision→флаг+skip+exit2+resume-детерминизм, healthy→pass, snapshot-флип→`--resnapshot`, no-thresholds→fail-fast. Всё зелёное `-race`.
|
||||
|
||||
Дальше: D (провайдерская домашка — фактчек слагов `/models`) → B (single-hop эскалация) → C (live-conformance).
|
||||
|
||||
## Полигон
|
||||
(секция параллельной сессии — записи добавлять сюда)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue