textmachine/backend/internal/pipeline/disposition.go

package pipeline

import (
	"fmt"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"

	"textmachine/backend/internal/checks"
	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/langscreen"
	"textmachine/backend/internal/llm"
	"textmachine/backend/internal/store"
)

// disposition.go: the per-chunk×stage verdict machinery of the Milestone-2 runner
// (D2). It answers three questions about a completion — is it usable, and if not
// WHY, and may we re-attack it — WITHOUT ever touching the wire (classify is a
// pure function of the completion text + finish reason + a little context, so a
// resume reproduces the same verdict from the same checkpoint, no re-billing).
//
// This is deliberately SEPARATE from the configurable coverage QA-gate
// (Gates.Coverage, step 6): that gate is opt-in and threshold-driven; the
// classifier here is intrinsic runner robustness that is always on, so a bad
// chunk is flagged-and-skipped instead of poisoning the pipeline (empty draft →
// empty edit → empty export) or wedging the whole book on a single call.

// Disposition is the resolved state of a chunk×stage over its checkpoints.
type Disposition string

const (
	// DispOK — a usable completion; its text feeds the next stage and is the
	// authoritative checkpoint.
	DispOK Disposition = "ok"
	// DispFlagged — the completion is unusable (refusal / echo / truncation loop
	// / empty / decode error) and retries (if any) are exhausted; the chunk is
	// flagged for a human and later stages are skipped. Money is still accounted.
	DispFlagged Disposition = "flagged"
	// DispSkipped — this stage was NOT attempted because an earlier stage of the
	// same chunk was flagged (no garbage draft → no paid edit over garbage).
	DispSkipped Disposition = "skipped"
)

// FlagReason is the tagged cause of a flag — a contract enum like the Finish*
// constants (D2.2: tagging drives retry policy). The runner re-attacks ONLY the
// retryable subset; the rest are deterministic per model/channel, so a same-model
// retry would just re-refuse and re-bill (routing to channel B / escalation is
// Phase 2). Some constants are DEFINED here for contract completeness but are not
// yet emitted by Milestone 2 — see the notes — they belong to later steps.
type FlagReason string

const (
	reasonOK FlagReason = "" // sentinel: not a flag

	// Emitted by classify() — the retryable pair (D2: bigger max_tokens on the
	// attempt axis, up to the regenerate cap, then flag).
	FlagLength FlagReason = "length" // truncated at max_tokens (a genuine cut, not a loop)
	FlagEmpty  FlagReason = "empty"  // 2xx with no usable text (thinking likely ate the whole budget)

	// Emitted by classify() — deterministic, NOT retryable on the same model.
	FlagLoopDegenerate FlagReason = "loop_degenerate" // repetition loop at a length cut (bigger budget just buys more loop, D2.3)
	FlagHardRefusal    FlagReason = "hard_refusal"    // provider finish_reason=refusal
	FlagSoftRefusal    FlagReason = "soft_refusal"    // refusal-blacklist match on a short output (the real insurance, D2.4)
	FlagContentFilter  FlagReason = "content_filter"  // provider finish_reason=content_filter (best-effort — emission unverified across DS/GLM/Kimi/grok, D2.4)
	FlagCJKArtifact    FlagReason = "cjk_artifact"    // output echoed the CJK source instead of translating (D3.4; fallback draft — step 7, just a flag for now)
	// FlagOffTargetLang: the completion came back in some language OTHER than the one asked for, and not
	// as an echo of the source — the residue the echo rule cannot see BY CONSTRUCTION, whose live
	// specimens are two fluent ENGLISH drafts that scored 0.000 on a source-script detector and shipped as
	// `ok` (D39.86, backlog row 46).
	//
	// ⚠ ITS OWN NAME, not FlagCJKArtifact. The design that shipped with the plan reused the echo flag to
	// save an edit to escalatable(), and the owner struck that in D39.93 п.2: the saving is real and the
	// cost is a LIE about the cause — an English draft would enter the telemetry as «an artifact of the
	// source script», which is the one thing it demonstrably is not. Reusing a flag to skip a whitelist is
	// how a new failure class becomes invisible in the numbers that are supposed to find it.
	FlagOffTargetLang FlagReason = "off_target_lang"

	// Emitted by the runner's billed-decode path.
	FlagDecodeError FlagReason = "decode_error" // 2xx with an unreadable body — billed, conservatively settled, flagged

	// FlagAttemptTimeout and FlagCancelled are the two verdicts of a call THE ENGINE ITSELF cut short
	// (backlog row 360). Both name their cause exactly, and they are two constants rather than one
	// because the disposition differs on every axis that matters:
	//
	//   - attempt_timeout — OUR deadline fired while the provider was generating. Deterministic on the
	//     same budget, so it is NOT retryable and NOT escalatable: a same-model retry re-buys the same
	//     generation, which is the defect row 360 names. The remedy is a bigger deadline (now derived
	//     from the budget) or a redrive, not another call inside this run.
	//   - cancelled — a HUMAN stopped the run over a call that was already on the wire. Nothing was
	//     wrong with it, so the chunk is not «needs a person»; it is «stopped, resume will re-do it».
	//     It still marks the position: a hole with no mark at all would export as an unexplained gap.
	//
	// ⚠ The vocabulary is CONVERTED from the transport's causes, not re-typed beside them: one carrier
	// for the three names the wire, the checkpoint, the flag and the operator all read.
	FlagAttemptTimeout FlagReason = FlagReason(llm.CutBySelfDeadline)
	FlagCancelled      FlagReason = FlagReason(llm.CutByParent)
	// ⚠ A LOST CONNECTION HAS NO FLAG, and that is a statement about the mechanism rather than an
	// omission. It is an infra pause, not a chunk that needs a human, and the checkpoint it leaves
	// records money with no result — which burnedByCut catches BEFORE anything classifies it, so no
	// replay of that row ever reaches a disposition. The guard is that predicate, not a name.

	// FlagRetryUnaffordable is the verdict of a unit whose FIRST attempt was paid for and came back
	// retryable (length/empty, or an echo the re-roll was meant to answer) and whose RETRY a USD ceiling
	// refused to reserve. D2's own ending — «retry up to the cap, then flag» — with the retry never bought.
	//
	// ⛔ IT IS ITS OWN REASON BECAUSE `length` WOULD LIE ABOUT THE CAUSE (D39.204 п.4). A reader and an
	// operator have to tell «we tried everything the budget allowed» from «the money ran out before the
	// second attempt»: the first is answered by a redrive or a better model, the second by topping up, and
	// a flag that says the former sends a person to fix something that is not broken. Before it existed
	// the refusal left the attempt loop as an ERROR instead — earlier than chunk_status — so the unit had
	// no row at all: it read `pending` forever, every resume replayed attempt 0 for $0 and died on the
	// retry again, and the run departed `exit 4` with every BOUGHT unit delivered.
	//
	// ⚠ NEITHER `retryable` NOR `escalatable` IS ASKED OF IT, and that is a fact about where it comes
	// from rather than a policy: both predicates are asked of a LIVE classification (stagerun.go,
	// escalation.go), and classify() never produces this reason — it is written by the stop mark
	// (cutcall.go) on a run that is ending. Nothing in this run re-attacks or escalates the position
	// because the run stops; what finishes it is money, which is why the row is deliberately NOT resolved
	// for resume (resolvedForResume) — raise the ceiling and the resume re-attacks the unit and walks the
	// whole path, escalation hop included.
	//
	// ⚠ That last sentence holds only while resolvedForResume keeps answering false for it. A change that
	// made this row terminal would turn the cheapest flag in the book into a permanent one, and would do
	// it silently: the row, its reason and its money all look exactly the same either way.
	FlagRetryUnaffordable FlagReason = "retry_unaffordable"

	// Reserved for later steps — DEFINED for contract stability, NOT emitted by
	// Milestone 2. coverage_fail / excision_suspect are verdicts of the configurable
	// coverage gate (step 6); hard_block / upstream_not_ok are for HTTP-level
	// content blocks handled when escalation lands (step 7).
	FlagCoverageFail    FlagReason = "coverage_fail"
	FlagExcisionSuspect FlagReason = "excision_suspect"
	FlagHardBlock       FlagReason = "hard_block"
	FlagUpstreamNotOK   FlagReason = "upstream_not_ok"

	// FlagGlossaryMiss is the memory-bank post-check verdict (E1, step 4): an approved
	// term's src fired in the chunk but no accepted dst form appears in the output — the
	// model did not render the term the way the bank told it to. It says nothing about
	// whether the dst we injected was RIGHT: a wrong rendering the model OBEYED is present
	// in the output, so it raises no miss and no flag (mempostcheck.go). Emitted
	// ONLY when the opt-in glossary post-check gate is enabled (config); in the default
	// flagger mode a miss is recorded in the retrieval-state, not a disposition. NOT
	// retryable (a same-model retry re-produces the same rendering) and NOT auto-
	// escalatable in v1 (the L3 targeted re-ask is a Phase-2 remedy, research/14 §9).
	FlagGlossaryMiss FlagReason = "glossary_miss"

	// FlagSanitizerDefect is the output-sanitizer verdict for a SUBSTANTIVE defect (D30.3,
	// sanitizer.go): the stage output carries an "instant unreadability" defect with no reliable
	// removable boundary — a leaked service preamble, a trailing note/edit block, a Latin-script
	// insertion, or a broken word form. Emitted ONLY when the opt-in Gates.Sanitizer is enabled;
	// the contaminated output is flagged AND skipped (D2 flag+skip) so it never commits to
	// TM/export (FinalText stays ""). NOT retryable (a same-model retry re-produces the artifact)
	// and NOT escalatable in v1 (the editor is pinned; a redrive surfaces the defect to a human).
	FlagSanitizerDefect FlagReason = "sanitizer_defect"

	// FlagSanitizerStripped is the output-sanitizer verdict for a COSMETIC-ONLY defect that was
	// STRIPPED and the remainder EXPORTED (D38 infra-pack, D35.4a): a leading markdown «### Глава»
	// header and/or a stray CJK-leak run, removed deterministically by checks.StripCosmetic. Unlike
	// FlagSanitizerDefect the chunk is NOT lost — the cleaned text is committed as the export
	// (final_hash points at a derived sanitized checkpoint, so the standard
	// final_hash→checkpoint.response_text export contract yields the cleaned text) — but it STAYS
	// flagged so a human verifies the auto-clean ("don't lose the chunk ... a flag for a human"). NOT
	// retryable / NOT escalatable (deterministic; a redrive would re-produce the same leak).
	FlagSanitizerStripped FlagReason = "sanitizer_stripped"
)

// classifierVersion versions the INTRINSIC classify() verdict logic — the refusal
// blacklist, the CJK-echo threshold (cjkEchoThreshold), the degeneration detector and
// the order in which they run. classify() resolves the ok↔flagged disposition and is
// re-run on resume over legacy / in-flight-crash checkpoints, so a change to any
// threshold (e.g. cjkEchoThreshold 0.15→0.20, a new refusal pattern) would otherwise
// keep the SAME snapshot id and silently re-verdict a resumed chunk — flagged→ok
// re-runs a skipped stage (fresh spend), ok→flagged burns a fresh escalation hop.
// Folded into the snapshot exactly like checks.CoverageGateVersion, so such a change is a
// loud --resnapshot, not a silent divergence (external-review finding).
// Ш-2 (see text/norm.go): unicode.Version is woven in so a toolchain Unicode bump that shifts the
// echo/refusal normalization or character classification re-verdicts a resumed chunk LOUDLY (--resnapshot).
// v3 (wire batch, backlog row 46): the TARGET-language screen joined the verdict set with its own named
// flag. Its rule version rides in the literal rather than being pasted, so a change inside langscreen —
// its threshold, its evidence floor, what it counts — moves this string by itself and takes the loud
// --resnapshot with it. That is the mechanism this constant exists to be, and it was the one axis of the
// screen's design that could not be left to discipline.
// ⚠ A var, not a const: langscreen.Version is DERIVED from the screen's own calibrated numbers (see there),
// so that editing a threshold moves this string by construction instead of by somebody remembering to.
var classifierVersion = "classify-v3-refusal+srcscript-echo015+" + langscreen.Version + "+loop+u" + unicode.Version

// decodeErrorFinish is the finish_reason the runner stores on a billed-but-
// unreadable 2xx (BilledDecodeError). classify() recognises it so a RESUMED
// decode checkpoint re-resolves to the same FlagDecodeError verdict the live
// path assigned — live and resume must agree (determinism of the resolve).
const decodeErrorFinish = "decode_error"

// The finish_reason a cut call's checkpoint carries. They come from the transport's own cause
// vocabulary (llm.CutCause) so that the wire fact, the stored row and the flag cannot be named three
// different things by three files.
//
// ⛔ TWO OF THEM ARE NOT RESULTS, and that is the load-bearing distinction. attemptTimeoutFinish IS a
// verdict — classify resolves it below and a resume serves it without calling anybody, which is the
// point: we already paid for that generation and will not buy it again. The other two record MONEY for
// a call that has no outcome at all, so replaying them would hand the stage an empty answer it never
// received; runAttempt burns them instead (burnedByCut) and the loop re-asks at the SAME budget.
const (
	attemptTimeoutFinish = string(llm.CutBySelfDeadline)
	cancelledFinish      = string(llm.CutByParent)
	connectionLostFinish = string(llm.CutByConnection)
)

// burnedByCut says a checkpoint records what a call COST without recording what it produced. Such a
// row is money only: it must never be served as a result, and its attempt index is spent — the next
// index re-asks the same budget under a fresh request_hash.
//
// This is the whole reason the money can be booked at all. The store has exactly three money verbs
// and none of them writes spend without a checkpoint, while the checkpoint IS the resume key
// (`ON CONFLICT (request_hash) DO NOTHING`) — so «pay and re-do under the same key» is impossible by
// construction. Separating the ATTEMPT INDEX from the count of budget DOUBLINGS (runStage) is what
// dissolves that: the re-done call is a different key at the same budget, its reserve→settle is
// fresh, and `committed == sum(checkpoints)` never wobbles.
//
// ⚠ THE TEXT IS PART OF THE TEST, and the finish_reason alone was not enough. That string shares a
// namespace with whatever a vendor decides to print — the adapter already normalises invented values
// like «sensitive» — so a provider answering 200 with a real translation and finish_reason
// «cancelled» would have had its answer thrown away and re-bought. A genuine burn is written here and
// is always textless (cutcall.go), so asking for both costs nothing and closes the namespace.
func burnedByCut(cp *store.Checkpoint) bool {
	if cp.ResponseText != "" {
		return false // a reply with content is an answer, whatever it calls its finish reason
	}
	return cp.FinishReason == cancelledFinish || cp.FinishReason == connectionLostFinish
}

// retryable reports whether a flag may be re-attacked on the SAME model along the
// attempt axis. Only length/empty: both are budget symptoms a bigger max_tokens
// can cure. Everything else is deterministic (refusal/filter/echo/loop/decode) —
// re-attacking burns money on a guaranteed repeat (D2.2/D2.3).
func (r FlagReason) retryable() bool {
	return r == FlagLength || r == FlagEmpty
}

// escalatable reports whether a flag is a DETERMINISTIC content-failure that a
// DIFFERENT model might fix — the single-hop escalation trigger (D12, the
// deterministic-content-failure class). These arrive as HTTP 200 (the provider
// billed a "translation" that is echo / excision / a refusal), so a same-model
// retry just re-produces them (which is why they are non-retryable) — escalation
// routes them to another model ONCE. length/empty are excluded (retryable on the
// same model with a bigger budget); decode_error is transport-shaped (a billed
// unreadable body), not a content verdict another model would reliably avoid.
func (r FlagReason) escalatable() bool {
	switch r {
	case FlagCJKArtifact, FlagOffTargetLang, FlagExcisionSuspect, FlagLoopDegenerate,
		FlagHardRefusal, FlagSoftRefusal, FlagContentFilter:
		return true
	}
	return false
}

// AnswersForResume says a flag is a VERDICT about what came back, rather than a MARK that a run stopped
// over this position with the work unfinished. The two stop marks — a call a person cut (`cancelled`) and
// a re-attack a ceiling refused (`retry_unaffordable`) — are the whole of the second class: both record
// money spent and work NOT done, and the next run re-does them.
//
// ⛔ IT IS ASKED OF THE REASON SO EVERY SURFACE CAN ASK THE SAME QUESTION, and it is EXPORTED for the
// same reason: three of the surfaces that must ask it hold an export RECORD and not a stored row, so a
// row-shaped predicate leaves them nothing to ask and they each invent a rule.
//
// TWELVE sites ask it, counted rather than remembered: grep -i for resolvedForResume and AnswersForResume
// over the non-test sources gives 25 lines today, which are 9 prose mentions (this paragraph included) + 3
// declarations + the delegate's own body + the TWELVE askers. ⚠ Only the last number is worth anything: the
// other three move the moment any comment mentions the predicate, so a reader who finds 26 lines should
// re-derive the breakdown rather than conclude an asker appeared. NINE ask it of the stored ROW: the resume fast-path (stagerun.go), the wave counters (the
// `waveShape.resolved` pair, and through them the money ledger), the unit's state (`resolveChunkState`),
// the redrive's target AND its stage list, the volume classifier's two questions (`unitPositionsOnFile`,
// `rowsResumeFree`), this mark's own prev-row guard, and the stopped-run account in the CLI. THREE ask it
// of the REASON, because what they hold is an export record and not a row: the reader's phrase in the book
// writer, the operator's refusal text beside it, and the plaintext export's banner. Four of the twelve
// already asked (the prev-row guard, the resume fast-path, `resolveChunkState`, `rowsResumeFree` — that is
// what `git show HEAD` answers); the rest either asked by NAME (`== FlagCancelled`, a copy that went wrong
// the day a second non-resolved reason existed) or did not ask at all — and three of those left a person
// wrong about the same position, though not in the same way. TWO said something false, in these words: the reader's own file says «требует проверки человеком» (langpacks reader.txt,
// `hole.withheld`) and the plaintext export's banner says «flagged for a human», about a position that needs
// nothing but the next run. The THIRD, the operator's refusal text, printed the bare `withheld (cancelled)`
// — it never said a human was needed, and it never said the thing that would have let him act either.
func (r FlagReason) AnswersForResume() bool {
	switch r {
	case FlagCancelled, FlagRetryUnaffordable:
		return false
	}
	return true
}

// disposition maps a reason to the resolved chunk×stage state.
func (r FlagReason) disposition() Disposition {
	if r == reasonOK {
		return DispOK
	}
	return DispFlagged
}

// classifyInput is everything classify needs. It is more than the "(text,
// finish)" shorthand because a faithful 1:1 port of eval/refusal_bench.py
// (03-implementation-notes §3.7) needs the source length (soft-refusal is a
// refusal pattern on an ANOMALOUSLY SHORT output — a long translation that
// merely quotes "I'm sorry, but…" as dialogue must NOT flag) and the target
// language (CJK-echo detection is meaningless when translating INTO a CJK
// language).
type classifyInput struct {
	Source     string
	Output     string
	Finish     string
	TargetLang string
	// NonProseReply says this completion is a TABLE the engine specified, not prose in the target language,
	// and it turns off the two rules that only make sense over prose: the source-echo share and the
	// target-language screen.
	//
	// Both would be not merely wrong here but INVERTED. A bank role answers `<source term><TAB><answer>`
	// per line, so its source-script share is the specified format (the echo rule would flag every healthy
	// call — and did: `verdict=cjk_artifact` with `bad_lines=0` on a 6/6 clean probe), and its letters are
	// engine identifiers plus source terms, so its target-script share is near zero (the screen would flag
	// every healthy call a SECOND way). Left on, both poison the one signal that says a provider is
	// misbehaving. Every OTHER check — refusal, content-filter, empty, truncation — still runs.
	//
	// ⚠ It names the reply's FORM rather than a role, and it used to be true for the terminologist only
	// (`role == roleTerminologist`), leaving the classifier — whose reply has the same shape — flagged on
	// every batch. That was latent while no shipping config ran the classifier; backlog row 140 turns it
	// on, so the omission would have gone live with it (13-tech-debt-anchors §Б-105).
	NonProseReply bool
	// TargetScripts is the TARGET language's declared writing script(s) (lang.LangScripts) — the data the
	// off-target screen judges against. Empty (a target nobody has authored a row for) ⇒ the screen
	// abstains and the whole rule is inert BY ABSENCE, the layer-7 discipline. Adding a pair is a row in
	// `internal/lang/data/lang-script.txt` and no Go edit, which is the review question this file answers.
	TargetScripts []*unicode.RangeTable
	// SourceScripts is the SOURCE language's declared writing script(s) (lang.LangScripts), the alphabet the
	// echo detector measures the share of in the output. Data-driven (D39.60 §5 G3): zh→Han, ja→Han+kana,
	// ko→Hangul, en→Latin — so a ko or en echo is no longer a blind spot (the old hard-coded Han+kana was).
	// nil (an unknown source) → the echo share is 0, i.e. the detector measures nothing rather than guessing.
	SourceScripts []*unicode.RangeTable
}

// classification is classify's verdict.
type classification struct {
	Reason FlagReason
	Detail string
}

func (c classification) ok() bool { return c.Reason == reasonOK }

// classify is the single verdict function. ORDER MATTERS (D2.1/D2.4): the
// deterministic hard signals (decode / content_filter / refusal finish) and the
// text refusal-blacklist / CJK-echo are checked BEFORE length/empty, so a
// truncated refusal is flagged as a refusal (deterministic, no paid re-attack)
// rather than as a length cut (retryable). Pure: no time, no randomness, no map
// iteration — resume reproduces the identical verdict from the identical
// checkpoint.
func classify(in classifyInput) classification {
	// Strip a <think>…</think> block the model may have leaked into content
	// (mirrors refusal_bench) before judging emptiness/length.
	out := strings.TrimSpace(checks.StripThink(in.Output))
	finish := in.Finish

	// 1) Deterministic finish-reason signals — never retried.
	switch finish {
	case decodeErrorFinish:
		return classification{FlagDecodeError, "billed 2xx with an unreadable body"}
	case attemptTimeoutFinish:
		// Live and resume must agree, exactly as for a decode checkpoint: the row was written by the
		// live path with this finish and no text, and re-reading it has to resolve to the same verdict
		// instead of falling through to «empty completion», which is retryable and would re-buy the
		// generation this whole class exists to stop buying twice.
		//
		// ⚠ It does NOT move classifierVersion, and that is provable rather than hoped: the constant
		// guards a change that could RE-VERDICT a stored checkpoint, and no checkpoint written before
		// this line can carry this finish_reason — the string did not exist and no provider emits it.
		// Bumping it would re-snapshot every book in the world to change the verdict of nothing.
		// ⚠ ONLY OVER AN EMPTY OUTPUT. The engine writes this finish_reason with no text at all
		// (cutcall.go); the string itself, though, shares a namespace with whatever a vendor prints —
		// the adapter already normalises invented values — and a provider answering 200 with a real
		// translation under this name would otherwise have its answer thrown away as a lost call.
		if out == "" {
			return classification{FlagAttemptTimeout, "our own deadline cut a delivered call; the provider generated and billed it, so it is not retried"}
		}
	case llm.FinishContentFilter:
		return classification{FlagContentFilter, "provider finish_reason=content_filter"}
	case llm.FinishRefusal:
		return classification{FlagHardRefusal, "provider finish_reason=refusal"}
	}

	// 2) Text refusal-blacklist / CJK-echo — BEFORE length/empty (only meaningful on
	//    non-empty text; a truncated refusal must not become a paid length retry).
	if out != "" {
		if isRefusal(out, in.Source) {
			return classification{FlagSoftRefusal, "refusal-blacklist match on a short output"}
		}
		if !in.NonProseReply {
			// ⚠ THE CJK GATE BELONGS TO THE ECHO RULE ALONE, and it used to hold both. «Is the output in the
			// SOURCE's script» is a meaningless question for a CJK TARGET — a correct →ja rendering of a zh
			// book is full of han — so the echo rule must not run there. The TARGET-side screen asks the
			// opposite question and is perfectly answerable: an English completion for a →ja book is
			// off-target and nothing about the target's script prevents saying so. Holding both under one
			// gate left →ja/→zh/→ko with NO language screen at all, while langscreen's own doc comment
			// promised the opposite — and it made the answer to the standing review question («does a pair
			// the repo has never seen work without editing Go») a straight NO for every CJK target.
			if !isCJKTarget(in.TargetLang) {
				if share := sourceScriptShare(out, in.SourceScripts); share > cjkEchoThreshold {
					return classification{FlagCJKArtifact, fmt.Sprintf("CJK share %.0f%% in output (untranslated echo)", share*100)}
				}
			}
			// The TARGET-side screen, AFTER the echo rule and never instead of it (backlog row 46, design
			// frozen by D39.92 п.1 + D39.93). The two answer different questions and the order is the
			// diagnosis: an output full of source script is an ECHO and must be named one, and only what
			// survives that is asked «then what language IS this». The residue is small and real — two
			// fluent English drafts in 1162 stand completions — and it is the one failure here that can
			// ship SILENTLY, because a bilingual editor may render Russian from an English draft and
			// deliver a relay translation nobody ordered.
			if scr := langscreen.Screen(out, in.TargetScripts); scr.Verdict == langscreen.OffTarget {
				return classification{FlagOffTargetLang, fmt.Sprintf("only %.0f%% of %d letters are in the target's script — the answer is not in the language that was asked for", scr.Share*100, scr.Letters)}
			}
		}
	}

	// 3) Empty completion — retryable (thinking usually ate the entire budget).
	if out == "" {
		return classification{FlagEmpty, "empty completion (finish=" + finish + ")"}
	}

	// 4) Length cut — retryable, UNLESS the output is a degenerate repetition
	//    loop, in which case a bigger budget only buys more loop (D2.3): flag now.
	if finish == llm.FinishLength {
		if degenerateLoop(out) {
			return classification{FlagLoopDegenerate, "repetition loop at a length cut"}
		}
		return classification{FlagLength, "truncated at max_tokens"}
	}

	// 5) A usable completion.
	return classification{reasonOK, ""}
}

// --- refusal blacklist (en/ru/zh/ja) — ported 1:1 from eval/refusal_bench.py ---
//
// The patterns are UNIVERSAL engine safety data (a model can refuse in any language regardless of the
// book's pair), so they live in internal/lang as an embedded, per-language-sectioned file (pair-14 §3), NOT
// a book pack — a no-langpack book (the ja→ru golden) still flags a refusal. Joined here into one
// case-insensitive regex, byte-identically to the ported reference.
var refusalRE = regexp.MustCompile("(?im)" + strings.Join(lang.RefusalPatterns(), "|"))

// isRefusal ports refusal_bench: a refusal-blacklist match AND an anomalously
// short output (shorter than max(400 runes, half the source), so a long faithful
// translation that merely contains a refusal-like phrase does not flag).
func isRefusal(out, src string) bool {
	if !refusalRE.MatchString(out) {
		return false
	}
	threshold := utf8.RuneCountInString(src) / 2
	if threshold < 400 {
		threshold = 400
	}
	return utf8.RuneCountInString(out) < threshold
}

// --- CJK-echo (untranslated source returned instead of a translation, D3.4) ---

// cjkEchoThreshold: >15% of the output in the SOURCE's declared script, in a non-source-script target, means
// the model echoed the source instead of translating (refusal_bench cjk_share > 0.15). The name keeps the
// flag's «cjk» vocabulary (FlagCJKArtifact), but the measurement is now source-script-general.
const cjkEchoThreshold = 0.15

// sourceScriptShare is the fraction of runes in s that belong to the SOURCE language's declared script(s)
// (data-driven, D39.60 §5 G3): an untranslated echo leaves the source script in a →other-script output. With
// no declared scripts (an unknown source) it is 0 — the detector measures nothing rather than guessing. It
// replaces the old cjkShare, which hard-coded Han+kana and was therefore BLIND to a ko (Hangul) or en (Latin)
// echo (the live ko blind-spot, D39.60 §3.4 / bug row 75).
func sourceScriptShare(s string, scripts []*unicode.RangeTable) float64 {
	if s == "" || len(scripts) == 0 {
		return 0
	}
	inScript, total := 0, 0
	for _, r := range s {
		total++
		if unicode.In(r, scripts...) {
			inScript++
		}
	}
	return float64(inScript) / float64(total)
}

// isCJKTarget reports whether the translation TARGET is a CJK-SCRIPT language, in which case source-script
// runes in the output ARE the translation and the echo check is skipped. Data-driven (lang.IsCJKScriptLang);
// phase-1 scope is →ru, so this is normally false. Replaces the old isCJKLang Go switch (which also drove the
// echo-exposure warning): ONE data-derived predicate, so the source and target queries can never drift apart.
func isCJKTarget(targetLang string) bool { return lang.IsCJKScriptLang(targetLang) }

// The READABILITY gate set (output-sanitizer + cheap style flaggers + Latin-residue repair) used to be
// guarded by an isRuTarget Go predicate. It is now gated BY DATA on *checks.Checkers: TargetActive() (the
// target ships readability data) and TargetScriptNonLatin() (the Latin-residue class). A →en book runs the
// whole of layer 7 inert BY ABSENCE of target data, with no Go branch (D39.62/П1, layer-7 target seam).

// --- degeneration detector (n-gram loop) — runs BEFORE the length retry (D2.3) ---

// degenerateLoop reports whether the text is a repetition loop: the word-level
// trigram distinctness collapses when a phrase/sentence repeats to fill the
// budget. Pure and deterministic (only len(distinct) is used, never map order).
// Below a floor of words a length cut of genuinely short text is not judged a
// loop. CJK output (no spaces → few "words") is left to the echo check.
func degenerateLoop(text string) bool {
	const n = 3
	const minWords = 30
	const distinctRatio = 0.25 // <25% distinct trigrams ⇒ ≥75% are repeats ⇒ loop
	words := strings.Fields(text)
	if len(words) < minWords {
		return false
	}
	total := len(words) - n + 1
	seen := make(map[string]struct{}, total)
	for i := 0; i+n <= len(words); i++ {
		seen[strings.Join(words[i:i+n], "\x00")] = struct{}{}
	}
	return float64(len(seen))/float64(total) < distinctRatio
}

// --- max_tokens on the attempt axis (D2.3) ---

// maxTokensForAttempt is the PURE output-token budget for a retry attempt: no doublings = base, each
// regeneration DOUBLES it (D2.3 remedy for a length cut — the previous budget was too small). Purity
// is load-bearing: the value enters request_hash, so resume must reproduce the identical budget. Its
// FORMULA is versioned into the snapshot (maxTokensPolicyVersion) so a change is a loud --resnapshot,
// not a silent checkpoint miss on retried chunks (the same discipline as estimatorVersion).
//
// ⛔ IT COUNTS DOUBLINGS, NOT ATTEMPTS, and the two used to be the same number by accident. Every
// regeneration is a new attempt, so «attempt index» read as «times the budget was doubled» and both
// callers agreed — until a call the ENGINE cut short had to be re-done. That re-do must NOT be paid
// for with a doubled budget: nothing was wrong with the answer, nobody saw it, and doubling would buy
// twice the call for a health nobody lost. Since a checkpoint cannot be written twice under one key,
// the re-do has to be a NEW attempt index — so the two dimensions had to come apart, and this
// parameter is the one that is about money.
//
// ⚠ THE FORMULA AND ITS OUTPUT ARE UNCHANGED FOR EVERY PATH A SHIPPING CONFIG CAN REACH, and the
// qualifier is load-bearing. Three regeneration paths call this: the content retry, the echo re-roll,
// and — since the empty-reply remedy — a retry that lowers the thinking level INSTEAD of doubling. The
// first two increment the doubling count with the attempt index, so on a run with no cut call the two
// are identical and the snapshot does not move. The third does not increment it, deliberately: that is
// what makes the remedy cheaper than the disease.
//
// ⛔ SO THE REASON maxTokensPolicyVersion NEED NOT MOVE IS CONDITIONAL, not absolute: it holds while
// Retries.LowerEffortOnEmpty is off in every shipping config, which is pinned
// (config.TestShippingPipelinesDoNotLowerEffortOnEmpty) and carried as a decision by backlog row 433.
// Turning that key on does NOT move this version either — attempt 0 is untouched and the snapshot does
// not fold Retries — but it does re-key attempt ≥ 1 for the books that already hold one, which is a
// cost named at the key itself. A reader who removes that pin has removed this paragraph's premise.
func maxTokensForAttempt(base, escalations int) int {
	if escalations <= 0 {
		return base
	}
	if escalations > 20 { // defensive: never shift by a runaway amount (overflow guard)
		escalations = 20
	}
	return base << uint(escalations)
}