textmachine/backend/internal/pipeline/coverage.go

package pipeline

import (
	"fmt"
	"strings"
	"unicode"

	"textmachine/backend/internal/config"
)

// coverage.go: the configurable EXCISION coverage-gate (шаг 6 / D12 Q3). Unlike the
// intrinsic classify() robustness (echo/empty/refusal — always on), this gate is
// opt-in (gates.coverage.enabled) and threshold-driven: it flags a stage OUTPUT
// that covers too LITTLE of the ORIGINAL source (silent excision — куски пропали,
// HTTP 200). It is a pure, deterministic RUNNER-side check (not an adapter concern —
// the adapter is translation-neutral, Q1/D2.4): given the checkpointed output it
// reproduces the same verdict on resume without re-billing.
//
// Ported 1:1 from eval/refusal_bench.py::classify_output's excision branch, which is
// BOTH our reference AND the Polygon acceptance oracle (D12 Q1). Metric = characters
// WITHOUT spaces (not tokens, §3.7). Only the LOWER bound is checked: sent_cov <
// sent_cov_min OR len_ratio < corridor-low → excision_suspect. The UPPER bound
// (anomaly/дописывание) is deliberately NOT flagged here (D12 Q3: ru output is
// 1.4–1.9× the source, so an upper cut false-flags; catching additions needs an
// entity in/out count + a bilingual judge — Phase 2, §04-unhappy mode 2).

// coverageGateVersion versions the segmentation + metric of this gate. It is folded
// into the job snapshot (like chunkerVersion/estimatorVersion), so a change to the
// algorithm — or turning the gate ON — is a loud --resnapshot, never a silent divergence
// between an old checkpoint's stored verdict and a new gate. HONEST COST (D15): because
// the snapshotID is baked into every call's RequestHash (render.go), that --resnapshot
// re-bills the WHOLE book — every checkpoint misses on the new snapshot, even wire-identical
// requests whose verdict merely needs re-classifying (the content-addressed reuse that
// could make them free is gated on an unchanged snapshot). Acceptable for the static
// acceptance book (loud, no divergence); the reason content-addressed checkpoint reuse is
// required before ongoing (D15.2). The segmentation is pinned 1:1 to eval/refusal_bench.py
// (D12 Q1); any change is Python-first, in lock-step, and bumps this constant.
const coverageGateVersion = "coverage-v1-naive-split-lower-bound"

// oracleDefaultCorridorLow is refusal_bench.py's fallback len_ratio LOWER bound for a
// source language without an explicit corridor (`EXPECT_LEN_RATIO.get(lang, (0.5,
// 3.0))`). The Go gate applies it too, so a book pair absent from len_ratio_bounds is
// checked exactly as the oracle would — not silently skipped (external-review parity).
const oracleDefaultCorridorLow = 0.5

// isSentenceTerminator is the branch-1 lookbehind class of refusal_bench's
// SENT_SPLIT_RE `[.!?…。！？]` (Latin + ellipsis + fullwidth CJK): a whitespace run
// after any of these splits a sentence.
func isSentenceTerminator(r rune) bool {
	switch r {
	case '.', '!', '?', '…', '。', '！', '？':
		return true
	}
	return false
}

// isCJKSentenceTerminator is the branch-2 lookbehind class `[。！？]` (fullwidth CJK
// only): these split ZERO-WIDTH, with or without a following space, because CJK prose
// has no inter-sentence space.
func isCJKSentenceTerminator(r rune) bool {
	switch r {
	case '。', '！', '？':
		return true
	}
	return false
}

// isOracleSpace matches Python's whitespace notion used by the oracle — `re.\s`
// (splitting) and `str.isspace()`/`.strip()` (trimming & the non-space count). It is
// unicode.IsSpace PLUS the C0 information separators U+001C–U+001F, which Python
// classifies as whitespace but Go's unicode.IsSpace (Unicode White_Space) does not.
// These survive EPUB/PDF extraction, so matching Python here is required for the
// bit-parity contract (D12 Q1), not cosmetic (self-review finding).
func isOracleSpace(r rune) bool {
	return unicode.IsSpace(r) || (r >= 0x1C && r <= 0x1F)
}

// trimOracleSpace is strings.TrimSpace with the oracle's whitespace set (mirrors the
// Python `.strip()` in split_sentences).
func trimOracleSpace(s string) string { return strings.TrimFunc(s, isOracleSpace) }

// splitSentences ports refusal_bench.py::split_sentences 1:1. Go's RE2 has no
// lookbehind, so the regex `(?<=[.!?…。！？])\s+|(?<=[。！？])` is reproduced by a manual
// left-to-right scan (branch-1 before branch-2, matching Python's alternation order):
//
//	(1) a whitespace RUN preceded by ANY terminator — the run is the separator and
//	    is consumed; then
//	(2) a ZERO-WIDTH point preceded by a CJK terminator — nothing consumed.
//
// Each raw piece is TrimSpace'd and empty pieces dropped (the Python list-comp).
// Pinned bit-for-bit to the live oracle by TestSplitSentencesOracleParity.
func splitSentences(text string) []string {
	runes := []rune(text)
	var segs []string
	emit := func(a, b int) {
		if s := trimOracleSpace(string(runes[a:b])); s != "" {
			segs = append(segs, s)
		}
	}
	segStart := 0
	for i := 1; i < len(runes); i++ {
		prev := runes[i-1]
		// Branch 1: any terminator, then a whitespace run (the run is consumed).
		if isSentenceTerminator(prev) && isOracleSpace(runes[i]) {
			emit(segStart, i)
			j := i
			for j < len(runes) && isOracleSpace(runes[j]) {
				j++
			}
			segStart = j
			i = j - 1 // the loop's i++ lands on j
			continue
		}
		// Branch 2: CJK terminator, zero-width split (the terminator stays with the
		// left piece; scanning continues from the same char, now the segment start).
		if isCJKSentenceTerminator(prev) {
			emit(segStart, i)
			segStart = i
		}
	}
	emit(segStart, len(runes))
	return segs
}

// nonSpaceCount is the coverage metric's unit — runes that are NOT whitespace
// (refusal_bench: `sum(1 for c in text if not c.isspace())`). Characters, not tokens.
func nonSpaceCount(s string) int {
	n := 0
	for _, r := range s {
		if !isOracleSpace(r) {
			n++
		}
	}
	return n
}

// langPairKey builds the `len_ratio_bounds` config key from the book's languages,
// e.g. ("zh","ru") → "zh-ru". Lower-cased and trimmed so casing in book.yaml does
// not miss the corridor.
func langPairKey(srcLang, dstLang string) string {
	return strings.ToLower(strings.TrimSpace(srcLang)) + "-" + strings.ToLower(strings.TrimSpace(dstLang))
}

// coverageResult carries the metric values alongside the verdict for telemetry/tests.
type coverageResult struct {
	cls      classification
	applied  bool    // false when skipped (too short / disabled)
	sentCov  float64 // sentences(out)/sentences(src)
	lenRatio float64 // nonspace(out)/nonspace(src)
}

// coverageCheck runs the excision gate on ONE stage output against the ORIGINAL
// source. It is pair-aware: len_ratio is judged against the corridor for
// srcLang-dstLang; sentence coverage is pair-independent. A source shorter than
// MinChunkChars (non-space) is not gated — the corridors are calibrated on
// 500–2500-char fragments and short chunks' ratios are noise (§3.7). Pure &
// deterministic. Returns excision_suspect (lower-bound only) or ok.
func coverageCheck(cfg config.CoverageGate, src, out, srcLang, dstLang string) coverageResult {
	// Strip a leaked <think>…</think> block BEFORE measuring, exactly as classify()
	// and the Python oracle do (refusal_bench: re.sub then .strip()). Without this a
	// reasoning block leaked into content inflates the sentence/char counts and MASKS
	// a real excision — a false negative, and a break of oracle parity (self-review).
	out = trimOracleSpace(stripThink(out))
	srcChars := nonSpaceCount(src)
	if srcChars < cfg.MinChunkChars {
		return coverageResult{cls: classification{reasonOK, ""}, applied: false}
	}

	nSrc := len(splitSentences(src))
	sentCov := 1.0
	if nSrc > 0 {
		sentCov = float64(len(splitSentences(out))) / float64(nSrc)
	}
	denom := srcChars
	if denom < 1 {
		denom = 1
	}
	lenRatio := float64(nonSpaceCount(out)) / float64(denom)

	res := coverageResult{applied: true, sentCov: sentCov, lenRatio: lenRatio}
	var flags []string
	if sentCov < cfg.SentCovMin {
		flags = append(flags, fmt.Sprintf("sent_cov=%.2f<%.2f", sentCov, cfg.SentCovMin))
	}
	// len_ratio corridor: the book's pair from config, else the ORACLE's fallback
	// (0.5) so a pair without a configured corridor still gets the exact check the
	// Python oracle applies (`EXPECT_LEN_RATIO.get(lang, (0.5, 3.0))`) — Go must not
	// silently skip it (external-review parity finding). Only the lower bound is used
	// (upper is out of scope, D12 Q3).
	lo := oracleDefaultCorridorLow
	if bounds, ok := cfg.LenRatio[langPairKey(srcLang, dstLang)]; ok && len(bounds) >= 1 {
		lo = bounds[0]
	}
	if lenRatio < lo {
		flags = append(flags, fmt.Sprintf("len_ratio=%.2f<%.2f", lenRatio, lo))
	}
	if len(flags) > 0 {
		res.cls = classification{FlagExcisionSuspect, strings.Join(flags, "; ")}
		return res
	}
	res.cls = classification{reasonOK, ""}
	return res
}