textmachine/backend/internal/pipeline/coverage.go

package pipeline

import (
	"fmt"
	"strings"
	"unicode"

	"textmachine/backend/internal/config"
)

// coverage.go: the configurable EXCISION coverage-gate (шаг 6 / D12 Q3). Unlike the
// intrinsic classify() robustness (echo/empty/refusal — always on), this gate is
// opt-in (gates.coverage.enabled) and threshold-driven: it flags a stage OUTPUT
// that covers too LITTLE of the ORIGINAL source (silent excision — куски пропали,
// HTTP 200). It is a pure, deterministic RUNNER-side check (not an adapter concern —
// the adapter is translation-neutral, Q1/D2.4): given the checkpointed output it
// reproduces the same verdict on resume without re-billing.
//
// Ported 1:1 from eval/refusal_bench.py::classify_output's excision branch, which is
// BOTH our reference AND the Polygon acceptance oracle (D12 Q1). Metric = characters
// WITHOUT spaces (not tokens, §3.7). Only the LOWER bound is checked: sent_cov <
// sent_cov_min OR len_ratio < corridor-low → excision_suspect. The UPPER bound
// (anomaly/дописывание) is deliberately NOT flagged here (D12 Q3: ru output is
// 1.4–1.9× the source, so an upper cut false-flags; catching additions needs an
// entity in/out count + a bilingual judge — Phase 2, §04-unhappy mode 2).

// coverageGateVersion versions the segmentation + metric of this gate. It is folded
// into the job snapshot (like chunkerVersion/estimatorVersion), so a change to the
// algorithm — or turning the gate ON — is a loud --resnapshot that re-evaluates every
// chunk from its checkpoint for FREE (no re-bill), never a silent divergence between
// an old checkpoint's stored verdict and a new gate. The segmentation is pinned 1:1
// to eval/refusal_bench.py (D12 Q1); any change is Python-first, in lock-step, and
// bumps this constant.
const coverageGateVersion = "coverage-v1-naive-split-lower-bound"

// isSentenceTerminator is the branch-1 lookbehind class of refusal_bench's
// SENT_SPLIT_RE `[.!?…。！？]` (Latin + ellipsis + fullwidth CJK): a whitespace run
// after any of these splits a sentence.
func isSentenceTerminator(r rune) bool {
	switch r {
	case '.', '!', '?', '…', '。', '！', '？':
		return true
	}
	return false
}

// isCJKSentenceTerminator is the branch-2 lookbehind class `[。！？]` (fullwidth CJK
// only): these split ZERO-WIDTH, with or without a following space, because CJK prose
// has no inter-sentence space.
func isCJKSentenceTerminator(r rune) bool {
	switch r {
	case '。', '！', '？':
		return true
	}
	return false
}

// splitSentences ports refusal_bench.py::split_sentences 1:1. Go's RE2 has no
// lookbehind, so the regex `(?<=[.!?…。！？])\s+|(?<=[。！？])` is reproduced by a manual
// left-to-right scan (branch-1 before branch-2, matching Python's alternation order):
//
//	(1) a whitespace RUN preceded by ANY terminator — the run is the separator and
//	    is consumed; then
//	(2) a ZERO-WIDTH point preceded by a CJK terminator — nothing consumed.
//
// Each raw piece is TrimSpace'd and empty pieces dropped (the Python list-comp).
// Pinned bit-for-bit to the live oracle by TestSplitSentencesOracleParity.
func splitSentences(text string) []string {
	runes := []rune(text)
	var segs []string
	emit := func(a, b int) {
		if s := strings.TrimSpace(string(runes[a:b])); s != "" {
			segs = append(segs, s)
		}
	}
	segStart := 0
	for i := 1; i < len(runes); i++ {
		prev := runes[i-1]
		// Branch 1: any terminator, then a whitespace run (the run is consumed).
		if isSentenceTerminator(prev) && unicode.IsSpace(runes[i]) {
			emit(segStart, i)
			j := i
			for j < len(runes) && unicode.IsSpace(runes[j]) {
				j++
			}
			segStart = j
			i = j - 1 // the loop's i++ lands on j
			continue
		}
		// Branch 2: CJK terminator, zero-width split (the terminator stays with the
		// left piece; scanning continues from the same char, now the segment start).
		if isCJKSentenceTerminator(prev) {
			emit(segStart, i)
			segStart = i
		}
	}
	emit(segStart, len(runes))
	return segs
}

// nonSpaceCount is the coverage metric's unit — runes that are NOT whitespace
// (refusal_bench: `sum(1 for c in text if not c.isspace())`). Characters, not tokens.
func nonSpaceCount(s string) int {
	n := 0
	for _, r := range s {
		if !unicode.IsSpace(r) {
			n++
		}
	}
	return n
}

// langPairKey builds the `len_ratio_bounds` config key from the book's languages,
// e.g. ("zh","ru") → "zh-ru". Lower-cased and trimmed so casing in book.yaml does
// not miss the corridor.
func langPairKey(srcLang, dstLang string) string {
	return strings.ToLower(strings.TrimSpace(srcLang)) + "-" + strings.ToLower(strings.TrimSpace(dstLang))
}

// coverageResult carries the metric values alongside the verdict for telemetry/tests.
type coverageResult struct {
	cls      classification
	applied  bool    // false when skipped (too short / disabled)
	sentCov  float64 // sentences(out)/sentences(src)
	lenRatio float64 // nonspace(out)/nonspace(src)
}

// coverageCheck runs the excision gate on ONE stage output against the ORIGINAL
// source. It is pair-aware: len_ratio is judged against the corridor for
// srcLang-dstLang; sentence coverage is pair-independent. A source shorter than
// MinChunkChars (non-space) is not gated — the corridors are calibrated on
// 500–2500-char fragments and short chunks' ratios are noise (§3.7). Pure &
// deterministic. Returns excision_suspect (lower-bound only) or ok.
func coverageCheck(cfg config.CoverageGate, src, out, srcLang, dstLang string) coverageResult {
	srcChars := nonSpaceCount(src)
	if srcChars < cfg.MinChunkChars {
		return coverageResult{cls: classification{reasonOK, ""}, applied: false}
	}

	nSrc := len(splitSentences(src))
	sentCov := 1.0
	if nSrc > 0 {
		sentCov = float64(len(splitSentences(out))) / float64(nSrc)
	}
	denom := srcChars
	if denom < 1 {
		denom = 1
	}
	lenRatio := float64(nonSpaceCount(out)) / float64(denom)

	res := coverageResult{applied: true, sentCov: sentCov, lenRatio: lenRatio}
	var flags []string
	if sentCov < cfg.SentCovMin {
		flags = append(flags, fmt.Sprintf("sent_cov=%.2f<%.2f", sentCov, cfg.SentCovMin))
	}
	// len_ratio needs a corridor for this pair; without one we still apply sentence
	// coverage (pair-independent) rather than silently pass everything.
	if bounds, ok := cfg.LenRatio[langPairKey(srcLang, dstLang)]; ok && len(bounds) >= 1 {
		if lo := bounds[0]; lenRatio < lo {
			flags = append(flags, fmt.Sprintf("len_ratio=%.2f<%.2f", lenRatio, lo))
		}
	}
	if len(flags) > 0 {
		res.cls = classification{FlagExcisionSuspect, strings.Join(flags, "; ")}
		return res
	}
	res.cls = classification{reasonOK, ""}
	return res
}