textmachine/backend/internal/pipeline/disposition.go

package pipeline

import (
	"fmt"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"

	"textmachine/backend/internal/checks"
	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/langscreen"
	"textmachine/backend/internal/llm"
)

// disposition.go: the per-chunk×stage verdict machinery of the Milestone-2 runner
// (D2). It answers three questions about a completion — is it usable, and if not
// WHY, and may we re-attack it — WITHOUT ever touching the wire (classify is a
// pure function of the completion text + finish reason + a little context, so a
// resume reproduces the same verdict from the same checkpoint, no re-billing).
//
// This is deliberately SEPARATE from the configurable coverage QA-gate
// (Gates.Coverage, step 6): that gate is opt-in and threshold-driven; the
// classifier here is intrinsic runner robustness that is always on, so a bad
// chunk is flagged-and-skipped instead of poisoning the pipeline (empty draft →
// empty edit → empty export) or wedging the whole book on a single call.

// Disposition is the resolved state of a chunk×stage over its checkpoints.
type Disposition string

const (
	// DispOK — a usable completion; its text feeds the next stage and is the
	// authoritative checkpoint.
	DispOK Disposition = "ok"
	// DispFlagged — the completion is unusable (refusal / echo / truncation loop
	// / empty / decode error) and retries (if any) are exhausted; the chunk is
	// flagged for a human and later stages are skipped. Money is still accounted.
	DispFlagged Disposition = "flagged"
	// DispSkipped — this stage was NOT attempted because an earlier stage of the
	// same chunk was flagged (no garbage draft → no paid edit over garbage).
	DispSkipped Disposition = "skipped"
)

// FlagReason is the tagged cause of a flag — a contract enum like the Finish*
// constants (D2.2: tagging drives retry policy). The runner re-attacks ONLY the
// retryable subset; the rest are deterministic per model/channel, so a same-model
// retry would just re-refuse and re-bill (routing to channel B / escalation is
// Phase 2). Some constants are DEFINED here for contract completeness but are not
// yet emitted by Milestone 2 — see the notes — they belong to later steps.
type FlagReason string

const (
	reasonOK FlagReason = "" // sentinel: not a flag

	// Emitted by classify() — the retryable pair (D2: bigger max_tokens on the
	// attempt axis, up to the regenerate cap, then flag).
	FlagLength FlagReason = "length" // truncated at max_tokens (a genuine cut, not a loop)
	FlagEmpty  FlagReason = "empty"  // 2xx with no usable text (thinking likely ate the whole budget)

	// Emitted by classify() — deterministic, NOT retryable on the same model.
	FlagLoopDegenerate FlagReason = "loop_degenerate" // repetition loop at a length cut (bigger budget just buys more loop, D2.3)
	FlagHardRefusal    FlagReason = "hard_refusal"    // provider finish_reason=refusal
	FlagSoftRefusal    FlagReason = "soft_refusal"    // refusal-blacklist match on a short output (the real insurance, D2.4)
	FlagContentFilter  FlagReason = "content_filter"  // provider finish_reason=content_filter (best-effort — emission unverified across DS/GLM/Kimi/grok, D2.4)
	FlagCJKArtifact    FlagReason = "cjk_artifact"    // output echoed the CJK source instead of translating (D3.4; fallback draft — step 7, just a flag for now)
	// FlagOffTargetLang: the completion came back in some language OTHER than the one asked for, and not
	// as an echo of the source — the residue the echo rule cannot see BY CONSTRUCTION, whose live
	// specimens are two fluent ENGLISH drafts that scored 0.000 on a source-script detector and shipped as
	// `ok` (D39.86, backlog row 46).
	//
	// ⚠ ITS OWN NAME, not FlagCJKArtifact. The design that shipped with the plan reused the echo flag to
	// save an edit to escalatable(), and the owner struck that in D39.93 п.2: the saving is real and the
	// cost is a LIE about the cause — an English draft would enter the telemetry as «an artifact of the
	// source script», which is the one thing it demonstrably is not. Reusing a flag to skip a whitelist is
	// how a new failure class becomes invisible in the numbers that are supposed to find it.
	FlagOffTargetLang FlagReason = "off_target_lang"

	// Emitted by the runner's billed-decode path.
	FlagDecodeError FlagReason = "decode_error" // 2xx with an unreadable body — billed, conservatively settled, flagged

	// Reserved for later steps — DEFINED for contract stability, NOT emitted by
	// Milestone 2. coverage_fail / excision_suspect are verdicts of the configurable
	// coverage gate (step 6); hard_block / upstream_not_ok are for HTTP-level
	// content blocks handled when escalation lands (step 7).
	FlagCoverageFail    FlagReason = "coverage_fail"
	FlagExcisionSuspect FlagReason = "excision_suspect"
	FlagHardBlock       FlagReason = "hard_block"
	FlagUpstreamNotOK   FlagReason = "upstream_not_ok"

	// FlagGlossaryMiss is the memory-bank post-check verdict (E1, step 4): an approved
	// term's src fired in the chunk but no accepted dst form appears in the output — the
	// model ignored the glossary, or we injected the wrong dst and it obeyed. Emitted
	// ONLY when the opt-in glossary post-check gate is enabled (config); in the default
	// flagger mode a miss is recorded in the retrieval-state, not a disposition. NOT
	// retryable (a same-model retry re-produces the same rendering) and NOT auto-
	// escalatable in v1 (the L3 targeted re-ask is a Phase-2 remedy, research/14 §9).
	FlagGlossaryMiss FlagReason = "glossary_miss"

	// FlagSanitizerDefect is the output-sanitizer verdict for a SUBSTANTIVE defect (D30.3,
	// sanitizer.go): the stage output carries an "instant unreadability" defect with no reliable
	// removable boundary — a leaked service preamble, a trailing note/edit block, a Latin-script
	// insertion, or a broken word form. Emitted ONLY when the opt-in Gates.Sanitizer is enabled;
	// the contaminated output is flagged AND skipped (D2 flag+skip) so it never commits to
	// TM/export (FinalText stays ""). NOT retryable (a same-model retry re-produces the artifact)
	// and NOT escalatable in v1 (the editor is pinned; a redrive surfaces the defect to a human).
	FlagSanitizerDefect FlagReason = "sanitizer_defect"

	// FlagSanitizerStripped is the output-sanitizer verdict for a COSMETIC-ONLY defect that was
	// STRIPPED and the remainder EXPORTED (D38 infra-pack, D35.4a): a leading markdown «### Глава»
	// header and/or a stray CJK-leak run, removed deterministically by checks.StripCosmetic. Unlike
	// FlagSanitizerDefect the chunk is NOT lost — the cleaned text is committed as the export
	// (final_hash points at a derived sanitized checkpoint, so the standard
	// final_hash→checkpoint.response_text export contract yields the cleaned text) — but it STAYS
	// flagged so a human verifies the auto-clean ("don't lose the chunk ... a flag for a human"). NOT
	// retryable / NOT escalatable (deterministic; a redrive would re-produce the same leak).
	FlagSanitizerStripped FlagReason = "sanitizer_stripped"
)

// classifierVersion versions the INTRINSIC classify() verdict logic — the refusal
// blacklist, the CJK-echo threshold (cjkEchoThreshold), the degeneration detector and
// the order in which they run. classify() resolves the ok↔flagged disposition and is
// re-run on resume over legacy / in-flight-crash checkpoints, so a change to any
// threshold (e.g. cjkEchoThreshold 0.15→0.20, a new refusal pattern) would otherwise
// keep the SAME snapshot id and silently re-verdict a resumed chunk — flagged→ok
// re-runs a skipped stage (fresh spend), ok→flagged burns a fresh escalation hop.
// Folded into the snapshot exactly like checks.CoverageGateVersion, so such a change is a
// loud --resnapshot, not a silent divergence (external-review finding).
// Ш-2 (see text/norm.go): unicode.Version is woven in so a toolchain Unicode bump that shifts the
// echo/refusal normalization or character classification re-verdicts a resumed chunk LOUDLY (--resnapshot).
// v3 (wire batch, backlog row 46): the TARGET-language screen joined the verdict set with its own named
// flag. Its rule version rides in the literal rather than being pasted, so a change inside langscreen —
// its threshold, its evidence floor, what it counts — moves this string by itself and takes the loud
// --resnapshot with it. That is the mechanism this constant exists to be, and it was the one axis of the
// screen's design that could not be left to discipline.
// ⚠ A var, not a const: langscreen.Version is DERIVED from the screen's own calibrated numbers (see there),
// so that editing a threshold moves this string by construction instead of by somebody remembering to.
var classifierVersion = "classify-v3-refusal+srcscript-echo015+" + langscreen.Version + "+loop+u" + unicode.Version

// decodeErrorFinish is the finish_reason the runner stores on a billed-but-
// unreadable 2xx (BilledDecodeError). classify() recognises it so a RESUMED
// decode checkpoint re-resolves to the same FlagDecodeError verdict the live
// path assigned — live and resume must agree (determinism of the resolve).
const decodeErrorFinish = "decode_error"

// retryable reports whether a flag may be re-attacked on the SAME model along the
// attempt axis. Only length/empty: both are budget symptoms a bigger max_tokens
// can cure. Everything else is deterministic (refusal/filter/echo/loop/decode) —
// re-attacking burns money on a guaranteed repeat (D2.2/D2.3).
func (r FlagReason) retryable() bool {
	return r == FlagLength || r == FlagEmpty
}

// escalatable reports whether a flag is a DETERMINISTIC content-failure that a
// DIFFERENT model might fix — the single-hop escalation trigger (D12, the
// deterministic-content-failure class). These arrive as HTTP 200 (the provider
// billed a "translation" that is echo / excision / a refusal), so a same-model
// retry just re-produces them (which is why they are non-retryable) — escalation
// routes them to another model ONCE. length/empty are excluded (retryable on the
// same model with a bigger budget); decode_error is transport-shaped (a billed
// unreadable body), not a content verdict another model would reliably avoid.
func (r FlagReason) escalatable() bool {
	switch r {
	case FlagCJKArtifact, FlagOffTargetLang, FlagExcisionSuspect, FlagLoopDegenerate,
		FlagHardRefusal, FlagSoftRefusal, FlagContentFilter:
		return true
	}
	return false
}

// disposition maps a reason to the resolved chunk×stage state.
func (r FlagReason) disposition() Disposition {
	if r == reasonOK {
		return DispOK
	}
	return DispFlagged
}

// classifyInput is everything classify needs. It is more than the "(text,
// finish)" shorthand because a faithful 1:1 port of eval/refusal_bench.py
// (03-implementation-notes §3.7) needs the source length (soft-refusal is a
// refusal pattern on an ANOMALOUSLY SHORT output — a long translation that
// merely quotes "I'm sorry, but…" as dialogue must NOT flag) and the target
// language (CJK-echo detection is meaningless when translating INTO a CJK
// language).
type classifyInput struct {
	Source     string
	Output     string
	Finish     string
	TargetLang string
	// NonProseReply says this completion is a TABLE the engine specified, not prose in the target language,
	// and it turns off the two rules that only make sense over prose: the source-echo share and the
	// target-language screen.
	//
	// Both would be not merely wrong here but INVERTED. A bank role answers `<source term><TAB><answer>`
	// per line, so its source-script share is the specified format (the echo rule would flag every healthy
	// call — and did: `verdict=cjk_artifact` with `bad_lines=0` on a 6/6 clean probe), and its letters are
	// engine identifiers plus source terms, so its target-script share is near zero (the screen would flag
	// every healthy call a SECOND way). Left on, both poison the one signal that says a provider is
	// misbehaving. Every OTHER check — refusal, content-filter, empty, truncation — still runs.
	//
	// ⚠ It names the reply's FORM rather than a role, and it used to be true for the terminologist only
	// (`role == roleTerminologist`), leaving the classifier — whose reply has the same shape — flagged on
	// every batch. That was latent while no shipping config ran the classifier; backlog row 140 turns it
	// on, so the omission would have gone live with it (13-tech-debt-anchors §Б-105).
	NonProseReply bool
	// TargetScripts is the TARGET language's declared writing script(s) (lang.LangScripts) — the data the
	// off-target screen judges against. Empty (a target nobody has authored a row for) ⇒ the screen
	// abstains and the whole rule is inert BY ABSENCE, the layer-7 discipline. Adding a pair is a row in
	// `internal/lang/data/lang-script.txt` and no Go edit, which is the review question this file answers.
	TargetScripts []*unicode.RangeTable
	// SourceScripts is the SOURCE language's declared writing script(s) (lang.LangScripts), the alphabet the
	// echo detector measures the share of in the output. Data-driven (D39.60 §5 G3): zh→Han, ja→Han+kana,
	// ko→Hangul, en→Latin — so a ko or en echo is no longer a blind spot (the old hard-coded Han+kana was).
	// nil (an unknown source) → the echo share is 0, i.e. the detector measures nothing rather than guessing.
	SourceScripts []*unicode.RangeTable
}

// classification is classify's verdict.
type classification struct {
	Reason FlagReason
	Detail string
}

func (c classification) ok() bool { return c.Reason == reasonOK }

// classify is the single verdict function. ORDER MATTERS (D2.1/D2.4): the
// deterministic hard signals (decode / content_filter / refusal finish) and the
// text refusal-blacklist / CJK-echo are checked BEFORE length/empty, so a
// truncated refusal is flagged as a refusal (deterministic, no paid re-attack)
// rather than as a length cut (retryable). Pure: no time, no randomness, no map
// iteration — resume reproduces the identical verdict from the identical
// checkpoint.
func classify(in classifyInput) classification {
	// Strip a <think>…</think> block the model may have leaked into content
	// (mirrors refusal_bench) before judging emptiness/length.
	out := strings.TrimSpace(checks.StripThink(in.Output))
	finish := in.Finish

	// 1) Deterministic finish-reason signals — never retried.
	switch finish {
	case decodeErrorFinish:
		return classification{FlagDecodeError, "billed 2xx with an unreadable body"}
	case llm.FinishContentFilter:
		return classification{FlagContentFilter, "provider finish_reason=content_filter"}
	case llm.FinishRefusal:
		return classification{FlagHardRefusal, "provider finish_reason=refusal"}
	}

	// 2) Text refusal-blacklist / CJK-echo — BEFORE length/empty (only meaningful on
	//    non-empty text; a truncated refusal must not become a paid length retry).
	if out != "" {
		if isRefusal(out, in.Source) {
			return classification{FlagSoftRefusal, "refusal-blacklist match on a short output"}
		}
		if !in.NonProseReply {
			// ⚠ THE CJK GATE BELONGS TO THE ECHO RULE ALONE, and it used to hold both. «Is the output in the
			// SOURCE's script» is a meaningless question for a CJK TARGET — a correct →ja rendering of a zh
			// book is full of han — so the echo rule must not run there. The TARGET-side screen asks the
			// opposite question and is perfectly answerable: an English completion for a →ja book is
			// off-target and nothing about the target's script prevents saying so. Holding both under one
			// gate left →ja/→zh/→ko with NO language screen at all, while langscreen's own doc comment
			// promised the opposite — and it made the answer to the standing review question («does a pair
			// the repo has never seen work without editing Go») a straight NO for every CJK target.
			if !isCJKTarget(in.TargetLang) {
				if share := sourceScriptShare(out, in.SourceScripts); share > cjkEchoThreshold {
					return classification{FlagCJKArtifact, fmt.Sprintf("CJK share %.0f%% in output (untranslated echo)", share*100)}
				}
			}
			// The TARGET-side screen, AFTER the echo rule and never instead of it (backlog row 46, design
			// frozen by D39.92 п.1 + D39.93). The two answer different questions and the order is the
			// diagnosis: an output full of source script is an ECHO and must be named one, and only what
			// survives that is asked «then what language IS this». The residue is small and real — two
			// fluent English drafts in 1162 stand completions — and it is the one failure here that can
			// ship SILENTLY, because a bilingual editor may render Russian from an English draft and
			// deliver a relay translation nobody ordered.
			if scr := langscreen.Screen(out, in.TargetScripts); scr.Verdict == langscreen.OffTarget {
				return classification{FlagOffTargetLang, fmt.Sprintf("only %.0f%% of %d letters are in the target's script — the answer is not in the language that was asked for", scr.Share*100, scr.Letters)}
			}
		}
	}

	// 3) Empty completion — retryable (thinking usually ate the entire budget).
	if out == "" {
		return classification{FlagEmpty, "empty completion (finish=" + finish + ")"}
	}

	// 4) Length cut — retryable, UNLESS the output is a degenerate repetition
	//    loop, in which case a bigger budget only buys more loop (D2.3): flag now.
	if finish == llm.FinishLength {
		if degenerateLoop(out) {
			return classification{FlagLoopDegenerate, "repetition loop at a length cut"}
		}
		return classification{FlagLength, "truncated at max_tokens"}
	}

	// 5) A usable completion.
	return classification{reasonOK, ""}
}

// --- refusal blacklist (en/ru/zh/ja) — ported 1:1 from eval/refusal_bench.py ---
//
// The patterns are UNIVERSAL engine safety data (a model can refuse in any language regardless of the
// book's pair), so they live in internal/lang as an embedded, per-language-sectioned file (pair-14 §3), NOT
// a book pack — a no-langpack book (the ja→ru golden) still flags a refusal. Joined here into one
// case-insensitive regex, byte-identically to the ported reference.
var refusalRE = regexp.MustCompile("(?im)" + strings.Join(lang.RefusalPatterns(), "|"))

// isRefusal ports refusal_bench: a refusal-blacklist match AND an anomalously
// short output (shorter than max(400 runes, half the source), so a long faithful
// translation that merely contains a refusal-like phrase does not flag).
func isRefusal(out, src string) bool {
	if !refusalRE.MatchString(out) {
		return false
	}
	threshold := utf8.RuneCountInString(src) / 2
	if threshold < 400 {
		threshold = 400
	}
	return utf8.RuneCountInString(out) < threshold
}

// --- CJK-echo (untranslated source returned instead of a translation, D3.4) ---

// cjkEchoThreshold: >15% of the output in the SOURCE's declared script, in a non-source-script target, means
// the model echoed the source instead of translating (refusal_bench cjk_share > 0.15). The name keeps the
// flag's «cjk» vocabulary (FlagCJKArtifact), but the measurement is now source-script-general.
const cjkEchoThreshold = 0.15

// sourceScriptShare is the fraction of runes in s that belong to the SOURCE language's declared script(s)
// (data-driven, D39.60 §5 G3): an untranslated echo leaves the source script in a →other-script output. With
// no declared scripts (an unknown source) it is 0 — the detector measures nothing rather than guessing. It
// replaces the old cjkShare, which hard-coded Han+kana and was therefore BLIND to a ko (Hangul) or en (Latin)
// echo (the live ko blind-spot, D39.60 §3.4 / bug row 75).
func sourceScriptShare(s string, scripts []*unicode.RangeTable) float64 {
	if s == "" || len(scripts) == 0 {
		return 0
	}
	inScript, total := 0, 0
	for _, r := range s {
		total++
		if unicode.In(r, scripts...) {
			inScript++
		}
	}
	return float64(inScript) / float64(total)
}

// isCJKTarget reports whether the translation TARGET is a CJK-SCRIPT language, in which case source-script
// runes in the output ARE the translation and the echo check is skipped. Data-driven (lang.IsCJKScriptLang);
// phase-1 scope is →ru, so this is normally false. Replaces the old isCJKLang Go switch (which also drove the
// echo-exposure warning): ONE data-derived predicate, so the source and target queries can never drift apart.
func isCJKTarget(targetLang string) bool { return lang.IsCJKScriptLang(targetLang) }

// The READABILITY gate set (output-sanitizer + cheap style flaggers + Latin-residue repair) used to be
// guarded by an isRuTarget Go predicate. It is now gated BY DATA on *checks.Checkers: TargetActive() (the
// target ships readability data) and TargetScriptNonLatin() (the Latin-residue class). A →en book runs the
// whole of layer 7 inert BY ABSENCE of target data, with no Go branch (D39.62/П1, layer-7 target seam).

// --- degeneration detector (n-gram loop) — runs BEFORE the length retry (D2.3) ---

// degenerateLoop reports whether the text is a repetition loop: the word-level
// trigram distinctness collapses when a phrase/sentence repeats to fill the
// budget. Pure and deterministic (only len(distinct) is used, never map order).
// Below a floor of words a length cut of genuinely short text is not judged a
// loop. CJK output (no spaces → few "words") is left to the echo check.
func degenerateLoop(text string) bool {
	const n = 3
	const minWords = 30
	const distinctRatio = 0.25 // <25% distinct trigrams ⇒ ≥75% are repeats ⇒ loop
	words := strings.Fields(text)
	if len(words) < minWords {
		return false
	}
	total := len(words) - n + 1
	seen := make(map[string]struct{}, total)
	for i := 0; i+n <= len(words); i++ {
		seen[strings.Join(words[i:i+n], "\x00")] = struct{}{}
	}
	return float64(len(seen))/float64(total) < distinctRatio
}

// --- max_tokens on the attempt axis (D2.3) ---

// maxTokensForAttempt is the PURE output-token budget for a retry attempt:
// attempt 0 = base, each regeneration DOUBLES it (D2.3 remedy for a length cut —
// the previous budget was too small). Purity is load-bearing: the value enters
// request_hash, so resume must reproduce the identical per-attempt budget. Its
// FORMULA is versioned into the snapshot (maxTokensPolicyVersion) so a change is
// a loud --resnapshot, not a silent checkpoint miss on retried chunks (the same
// discipline as estimatorVersion).
func maxTokensForAttempt(base, attempt int) int {
	if attempt <= 0 {
		return base
	}
	if attempt > 20 { // defensive: never shift by a runaway amount (overflow guard)
		attempt = 20
	}
	return base << uint(attempt)
}