package pipeline import ( "fmt" "strings" "unicode" "textmachine/backend/internal/config" ) // coverage.go: the configurable EXCISION coverage-gate (шаг 6 / D12 Q3). Unlike the // intrinsic classify() robustness (echo/empty/refusal — always on), this gate is // opt-in (gates.coverage.enabled) and threshold-driven: it flags a stage OUTPUT // that covers too LITTLE of the ORIGINAL source (silent excision — куски пропали, // HTTP 200). It is a pure, deterministic RUNNER-side check (not an adapter concern — // the adapter is translation-neutral, Q1/D2.4): given the checkpointed output it // reproduces the same verdict on resume without re-billing. // // Ported 1:1 from eval/refusal_bench.py::classify_output's excision branch, which is // BOTH our reference AND the Polygon acceptance oracle (D12 Q1). Metric = characters // WITHOUT spaces (not tokens, §3.7). Only the LOWER bound is checked: sent_cov < // sent_cov_min OR len_ratio < corridor-low → excision_suspect. The UPPER bound // (anomaly/дописывание) is deliberately NOT flagged here (D12 Q3: ru output is // 1.4–1.9× the source, so an upper cut false-flags; catching additions needs an // entity in/out count + a bilingual judge — Phase 2, §04-unhappy mode 2). // coverageGateVersion versions the segmentation + metric of this gate. It is folded // into the job snapshot (like chunkerVersion/estimatorVersion), so a change to the // algorithm — or turning the gate ON — is a loud --resnapshot that re-evaluates every // chunk from its checkpoint for FREE (no re-bill), never a silent divergence between // an old checkpoint's stored verdict and a new gate. The segmentation is pinned 1:1 // to eval/refusal_bench.py (D12 Q1); any change is Python-first, in lock-step, and // bumps this constant. const coverageGateVersion = "coverage-v1-naive-split-lower-bound" // oracleDefaultCorridorLow is refusal_bench.py's fallback len_ratio LOWER bound for a // source language without an explicit corridor (`EXPECT_LEN_RATIO.get(lang, (0.5, // 3.0))`). The Go gate applies it too, so a book pair absent from len_ratio_bounds is // checked exactly as the oracle would — not silently skipped (external-review parity). const oracleDefaultCorridorLow = 0.5 // isSentenceTerminator is the branch-1 lookbehind class of refusal_bench's // SENT_SPLIT_RE `[.!?…。!?]` (Latin + ellipsis + fullwidth CJK): a whitespace run // after any of these splits a sentence. func isSentenceTerminator(r rune) bool { switch r { case '.', '!', '?', '…', '。', '!', '?': return true } return false } // isCJKSentenceTerminator is the branch-2 lookbehind class `[。!?]` (fullwidth CJK // only): these split ZERO-WIDTH, with or without a following space, because CJK prose // has no inter-sentence space. func isCJKSentenceTerminator(r rune) bool { switch r { case '。', '!', '?': return true } return false } // isOracleSpace matches Python's whitespace notion used by the oracle — `re.\s` // (splitting) and `str.isspace()`/`.strip()` (trimming & the non-space count). It is // unicode.IsSpace PLUS the C0 information separators U+001C–U+001F, which Python // classifies as whitespace but Go's unicode.IsSpace (Unicode White_Space) does not. // These survive EPUB/PDF extraction, so matching Python here is required for the // bit-parity contract (D12 Q1), not cosmetic (self-review finding). func isOracleSpace(r rune) bool { return unicode.IsSpace(r) || (r >= 0x1C && r <= 0x1F) } // trimOracleSpace is strings.TrimSpace with the oracle's whitespace set (mirrors the // Python `.strip()` in split_sentences). func trimOracleSpace(s string) string { return strings.TrimFunc(s, isOracleSpace) } // splitSentences ports refusal_bench.py::split_sentences 1:1. Go's RE2 has no // lookbehind, so the regex `(?<=[.!?…。!?])\s+|(?<=[。!?])` is reproduced by a manual // left-to-right scan (branch-1 before branch-2, matching Python's alternation order): // // (1) a whitespace RUN preceded by ANY terminator — the run is the separator and // is consumed; then // (2) a ZERO-WIDTH point preceded by a CJK terminator — nothing consumed. // // Each raw piece is TrimSpace'd and empty pieces dropped (the Python list-comp). // Pinned bit-for-bit to the live oracle by TestSplitSentencesOracleParity. func splitSentences(text string) []string { runes := []rune(text) var segs []string emit := func(a, b int) { if s := trimOracleSpace(string(runes[a:b])); s != "" { segs = append(segs, s) } } segStart := 0 for i := 1; i < len(runes); i++ { prev := runes[i-1] // Branch 1: any terminator, then a whitespace run (the run is consumed). if isSentenceTerminator(prev) && isOracleSpace(runes[i]) { emit(segStart, i) j := i for j < len(runes) && isOracleSpace(runes[j]) { j++ } segStart = j i = j - 1 // the loop's i++ lands on j continue } // Branch 2: CJK terminator, zero-width split (the terminator stays with the // left piece; scanning continues from the same char, now the segment start). if isCJKSentenceTerminator(prev) { emit(segStart, i) segStart = i } } emit(segStart, len(runes)) return segs } // nonSpaceCount is the coverage metric's unit — runes that are NOT whitespace // (refusal_bench: `sum(1 for c in text if not c.isspace())`). Characters, not tokens. func nonSpaceCount(s string) int { n := 0 for _, r := range s { if !isOracleSpace(r) { n++ } } return n } // langPairKey builds the `len_ratio_bounds` config key from the book's languages, // e.g. ("zh","ru") → "zh-ru". Lower-cased and trimmed so casing in book.yaml does // not miss the corridor. func langPairKey(srcLang, dstLang string) string { return strings.ToLower(strings.TrimSpace(srcLang)) + "-" + strings.ToLower(strings.TrimSpace(dstLang)) } // coverageResult carries the metric values alongside the verdict for telemetry/tests. type coverageResult struct { cls classification applied bool // false when skipped (too short / disabled) sentCov float64 // sentences(out)/sentences(src) lenRatio float64 // nonspace(out)/nonspace(src) } // coverageCheck runs the excision gate on ONE stage output against the ORIGINAL // source. It is pair-aware: len_ratio is judged against the corridor for // srcLang-dstLang; sentence coverage is pair-independent. A source shorter than // MinChunkChars (non-space) is not gated — the corridors are calibrated on // 500–2500-char fragments and short chunks' ratios are noise (§3.7). Pure & // deterministic. Returns excision_suspect (lower-bound only) or ok. func coverageCheck(cfg config.CoverageGate, src, out, srcLang, dstLang string) coverageResult { // Strip a leaked block BEFORE measuring, exactly as classify() // and the Python oracle do (refusal_bench: re.sub then .strip()). Without this a // reasoning block leaked into content inflates the sentence/char counts and MASKS // a real excision — a false negative, and a break of oracle parity (self-review). out = trimOracleSpace(stripThink(out)) srcChars := nonSpaceCount(src) if srcChars < cfg.MinChunkChars { return coverageResult{cls: classification{reasonOK, ""}, applied: false} } nSrc := len(splitSentences(src)) sentCov := 1.0 if nSrc > 0 { sentCov = float64(len(splitSentences(out))) / float64(nSrc) } denom := srcChars if denom < 1 { denom = 1 } lenRatio := float64(nonSpaceCount(out)) / float64(denom) res := coverageResult{applied: true, sentCov: sentCov, lenRatio: lenRatio} var flags []string if sentCov < cfg.SentCovMin { flags = append(flags, fmt.Sprintf("sent_cov=%.2f<%.2f", sentCov, cfg.SentCovMin)) } // len_ratio corridor: the book's pair from config, else the ORACLE's fallback // (0.5) so a pair without a configured corridor still gets the exact check the // Python oracle applies (`EXPECT_LEN_RATIO.get(lang, (0.5, 3.0))`) — Go must not // silently skip it (external-review parity finding). Only the lower bound is used // (upper is out of scope, D12 Q3). lo := oracleDefaultCorridorLow if bounds, ok := cfg.LenRatio[langPairKey(srcLang, dstLang)]; ok && len(bounds) >= 1 { lo = bounds[0] } if lenRatio < lo { flags = append(flags, fmt.Sprintf("len_ratio=%.2f<%.2f", lenRatio, lo)) } if len(flags) > 0 { res.cls = classification{FlagExcisionSuspect, strings.Join(flags, "; ")} return res } res.cls = classification{reasonOK, ""} return res }