package pipeline import ( "fmt" "regexp" "strings" "unicode" "unicode/utf8" "textmachine/backend/internal/llm" ) // disposition.go: the per-chunk×stage verdict machinery of the Веха-2 runner // (D2). It answers three questions about a completion — is it usable, and if not // WHY, and may we re-attack it — WITHOUT ever touching the wire (classify is a // pure function of the completion text + finish reason + a little context, so a // resume reproduces the same verdict from the same checkpoint, no re-billing). // // This is deliberately SEPARATE from the configurable coverage QA-gate // (Gates.Coverage, шаг 6): that gate is opt-in and threshold-driven; the // classifier here is intrinsic runner robustness that is always on, so a bad // chunk is flagged-and-skipped instead of poisoning the pipeline (empty draft → // empty edit → empty export) or wedging the whole book on a single call. // Disposition is the resolved state of a chunk×stage over its checkpoints. type Disposition string const ( // DispOK — a usable completion; its text feeds the next stage and is the // authoritative checkpoint. DispOK Disposition = "ok" // DispFlagged — the completion is unusable (refusal / echo / truncation loop // / empty / decode error) and retries (if any) are exhausted; the chunk is // flagged for a human and later stages are skipped. Money is still accounted. DispFlagged Disposition = "flagged" // DispSkipped — this stage was NOT attempted because an earlier stage of the // same chunk was flagged (no garbage draft → no paid edit over garbage). DispSkipped Disposition = "skipped" ) // FlagReason is the tagged cause of a flag — a contract enum like the Finish* // constants (D2.2: tagging drives retry policy). The runner re-attacks ONLY the // retryable subset; the rest are deterministic per model/channel, so a same-model // retry would just re-refuse and re-bill (routing to channel B / escalation is // Фаза 2). Some constants are DEFINED here for contract completeness but are not // yet emitted by Веха 2 — see the notes — they belong to later steps. type FlagReason string const ( reasonOK FlagReason = "" // sentinel: not a flag // Emitted by classify() — the retryable pair (D2: bigger max_tokens on the // attempt axis, up to the regenerate cap, then flag). FlagLength FlagReason = "length" // truncated at max_tokens (a genuine cut, not a loop) FlagEmpty FlagReason = "empty" // 2xx with no usable text (thinking likely ate the whole budget) // Emitted by classify() — deterministic, NOT retryable on the same model. FlagLoopDegenerate FlagReason = "loop_degenerate" // repetition loop at a length cut (bigger budget just buys more loop, D2.3) FlagHardRefusal FlagReason = "hard_refusal" // provider finish_reason=refusal FlagSoftRefusal FlagReason = "soft_refusal" // refusal-blacklist match on a short output (the real insurance, D2.4) FlagContentFilter FlagReason = "content_filter" // provider finish_reason=content_filter (best-effort — emission unverified across DS/GLM/Kimi/grok, D2.4) FlagCJKArtifact FlagReason = "cjk_artifact" // output echoed the CJK source instead of translating (D3.4; фолбэк-черновик — шаг 7, пока просто флаг) // Emitted by the runner's billed-decode path. FlagDecodeError FlagReason = "decode_error" // 2xx with an unreadable body — billed, conservatively settled, flagged // Reserved for later steps — DEFINED for contract stability, NOT emitted by // Веха 2. coverage_fail / excision_suspect are verdicts of the configurable // coverage gate (шаг 6); hard_block / upstream_not_ok are for HTTP-level // content blocks handled when escalation lands (шаг 7). FlagCoverageFail FlagReason = "coverage_fail" FlagExcisionSuspect FlagReason = "excision_suspect" FlagHardBlock FlagReason = "hard_block" FlagUpstreamNotOK FlagReason = "upstream_not_ok" ) // decodeErrorFinish is the finish_reason the runner stores on a billed-but- // unreadable 2xx (BilledDecodeError). classify() recognises it so a RESUMED // decode checkpoint re-resolves to the same FlagDecodeError verdict the live // path assigned — live and resume must agree (determinism of the resolve). const decodeErrorFinish = "decode_error" // retryable reports whether a flag may be re-attacked on the SAME model along the // attempt axis. Only length/empty: both are budget symptoms a bigger max_tokens // can cure. Everything else is deterministic (refusal/filter/echo/loop/decode) — // re-attacking burns money on a guaranteed repeat (D2.2/D2.3). func (r FlagReason) retryable() bool { return r == FlagLength || r == FlagEmpty } // escalatable reports whether a flag is a DETERMINISTIC content-failure that a // DIFFERENT model might fix — the single-hop escalation trigger (D12, the // deterministic-content-failure class). These arrive as HTTP 200 (the provider // billed a "translation" that is echo / excision / a refusal), so a same-model // retry just re-produces them (which is why they are non-retryable) — escalation // routes them to another model ONCE. length/empty are excluded (retryable on the // same model with a bigger budget); decode_error is transport-shaped (a billed // unreadable body), not a content verdict another model would reliably avoid. func (r FlagReason) escalatable() bool { switch r { case FlagCJKArtifact, FlagExcisionSuspect, FlagLoopDegenerate, FlagHardRefusal, FlagSoftRefusal, FlagContentFilter: return true } return false } // disposition maps a reason to the resolved chunk×stage state. func (r FlagReason) disposition() Disposition { if r == reasonOK { return DispOK } return DispFlagged } // classifyInput is everything classify needs. It is more than the "(text, // finish)" shorthand because a faithful 1:1 port of eval/refusal_bench.py // (03-implementation-notes §3.7) needs the source length (soft-refusal is a // refusal pattern on an ANOMALOUSLY SHORT output — a long translation that // merely quotes "I'm sorry, but…" as dialogue must NOT flag) and the target // language (CJK-echo detection is meaningless when translating INTO a CJK // language). type classifyInput struct { Source string Output string Finish string TargetLang string } // classification is classify's verdict. type classification struct { Reason FlagReason Detail string } func (c classification) ok() bool { return c.Reason == reasonOK } // classify is the single verdict function. ORDER MATTERS (D2.1/D2.4): the // deterministic hard signals (decode / content_filter / refusal finish) and the // text refusal-blacklist / CJK-echo are checked BEFORE length/empty, so a // truncated refusal is flagged as a refusal (deterministic, no paid re-attack) // rather than as a length cut (retryable). Pure: no time, no randomness, no map // iteration — resume reproduces the identical verdict from the identical // checkpoint. func classify(in classifyInput) classification { // Strip a block the model may have leaked into content // (mirrors refusal_bench) before judging emptiness/length. out := strings.TrimSpace(stripThink(in.Output)) finish := in.Finish // 1) Deterministic finish-reason signals — never retried. switch finish { case decodeErrorFinish: return classification{FlagDecodeError, "billed 2xx with an unreadable body"} case llm.FinishContentFilter: return classification{FlagContentFilter, "provider finish_reason=content_filter"} case llm.FinishRefusal: return classification{FlagHardRefusal, "provider finish_reason=refusal"} } // 2) Text refusal-blacklist / CJK-echo — ДО length/empty (only meaningful on // non-empty text; a truncated refusal must not become a paid length retry). if out != "" { if isRefusal(out, in.Source) { return classification{FlagSoftRefusal, "refusal-blacklist match on a short output"} } if !isCJKTarget(in.TargetLang) { if share := cjkShare(out); share > cjkEchoThreshold { return classification{FlagCJKArtifact, fmt.Sprintf("CJK share %.0f%% in output (untranslated echo)", share*100)} } } } // 3) Empty completion — retryable (thinking usually ate the entire budget). if out == "" { return classification{FlagEmpty, "empty completion (finish=" + finish + ")"} } // 4) Length cut — retryable, UNLESS the output is a degenerate repetition // loop, in which case a bigger budget only buys more loop (D2.3): flag now. if finish == llm.FinishLength { if degenerateLoop(out) { return classification{FlagLoopDegenerate, "repetition loop at a length cut"} } return classification{FlagLength, "truncated at max_tokens"} } // 5) A usable completion. return classification{reasonOK, ""} } // --- refusal blacklist (en/ru/zh/ja) — ported 1:1 from eval/refusal_bench.py --- var refusalPatterns = []string{ `I (?:can(?:'|no)t|cannot|won'?t|am (?:not able|unable)) (?:to )?(?:assist|help|translate|continue|provide|fulfill)`, `I'?m sorry,? but`, `against (?:my|our) (?:content )?polic`, `content polic(?:y|ies)`, `не могу (?:помочь|перевести|продолжить|выполнить)`, `я не буду переводить`, `противоречит (?:политике|правилам)`, `としてお答えできません`, `申し訳ありませんが`, `我(?:不能|无法)(?:协助|翻译|继续)`, `抱歉[,,]`, `^\s*\[?(?:REFUSED|ОТКАЗ)\]?\s*$`, } var refusalRE = regexp.MustCompile("(?im)" + strings.Join(refusalPatterns, "|")) var thinkRE = regexp.MustCompile(`(?s).*?\s*`) func stripThink(s string) string { return thinkRE.ReplaceAllString(s, "") } // isRefusal ports refusal_bench: a refusal-blacklist match AND an anomalously // short output (shorter than max(400 runes, half the source), so a long faithful // translation that merely contains a refusal-like phrase does not flag). func isRefusal(out, src string) bool { if !refusalRE.MatchString(out) { return false } threshold := utf8.RuneCountInString(src) / 2 if threshold < 400 { threshold = 400 } return utf8.RuneCountInString(out) < threshold } // --- CJK-echo (untranslated source returned instead of a translation, D3.4) --- // cjkEchoThreshold: >15% Han+kana in a non-CJK-target output means the model // echoed the source (refusal_bench cjk_share > 0.15). const cjkEchoThreshold = 0.15 func cjkShare(s string) float64 { if s == "" { return 0 } cjk, total := 0, 0 for _, r := range s { total++ if unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana) { cjk++ } } if total == 0 { return 0 } return float64(cjk) / float64(total) } // isCJKTarget reports whether the translation TARGET is a CJK language, in which // case CJK in the output is expected and the echo check is skipped. Phase-1 // scope is →ru, so this is normally false. func isCJKTarget(lang string) bool { switch strings.ToLower(strings.TrimSpace(lang)) { case "zh", "ja", "ko", "zh-cn", "zh-tw": return true } return false } // --- degeneration detector (n-gram loop) — runs BEFORE the length retry (D2.3) --- // degenerateLoop reports whether the text is a repetition loop: the word-level // trigram distinctness collapses when a phrase/sentence repeats to fill the // budget. Pure and deterministic (only len(distinct) is used, never map order). // Below a floor of words a length cut of genuinely short text is not judged a // loop. CJK output (no spaces → few "words") is left to the echo check. func degenerateLoop(text string) bool { const n = 3 const minWords = 30 const distinctRatio = 0.25 // <25% distinct trigrams ⇒ ≥75% are repeats ⇒ loop words := strings.Fields(text) if len(words) < minWords { return false } total := len(words) - n + 1 seen := make(map[string]struct{}, total) for i := 0; i+n <= len(words); i++ { seen[strings.Join(words[i:i+n], "\x00")] = struct{}{} } return float64(len(seen))/float64(total) < distinctRatio } // --- max_tokens on the attempt axis (D2.3) --- // maxTokensForAttempt is the PURE output-token budget for a retry attempt: // attempt 0 = base, each regeneration DOUBLES it (D2.3 remedy for a length cut — // the previous budget was too small). Purity is load-bearing: the value enters // request_hash, so resume must reproduce the identical per-attempt budget. Its // FORMULA is versioned into the snapshot (maxTokensPolicyVersion) so a change is // a loud --resnapshot, not a silent checkpoint miss on retried chunks (the same // discipline as estimatorVersion). func maxTokensForAttempt(base, attempt int) int { if attempt <= 0 { return base } if attempt > 20 { // defensive: never shift by a runaway amount (overflow guard) attempt = 20 } return base << uint(attempt) }