package pipeline import ( "encoding/json" "maps" "slices" "strings" "textmachine/backend/internal/store" ) // chunkrun.go: disposition-петля уровня ЧАНКА (D2) — стадии по порядку, первый флаг // останавливает чанк (дальше skipped, платной редактуры мусора нет), плюс горячий // путь памяти v2 (Select→инъекция per-role→post-check E1) и дешёвые стиль-флаггеры. // Новые роли-потребители инъекции Ф2 (annotator, voice-слой) добавляются здесь. // classifyOutput resolves a completion's disposition: FIRST the always-on intrinsic // classifier (echo/empty/refusal/length — Веха 2), then, only if that is ok AND the // configurable coverage gate is enabled, the excision gate (шаг 6 / D12 Q3). Both are // pure and deterministic over (source, output, finish), so a resumed checkpoint // reproduces the identical verdict for free — the intrinsic part unconditionally, the // gate part under the same coverage config (folded into the snapshot, so a gate change // re-pins loudly). The coverage gate runs ONLY on the TRANSLATOR role's output (vs the // original source): a monolingual editor legitimately restructures sentences, so gating // it against the source would false-flag a correct edit (see the role check below). // classifyOutput returns the disposition AND, for a cosmetic sanitizer strip, the resolved EXPORT // text (the strip remainder) computed ONCE here (T3.4 / L8-stripcosmetic-recomputed-twice): the // second return travels back to runStage as the authoritative export, so no site re-derives // stripCosmetic and the "non-empty & clean" invariant is asserted over the very bytes that ship. // The string is "" for every disposition except FlagSanitizerStripped. func (r *Runner) classifyOutput(role, source, output, finish string, isFinal bool) (classification, string) { cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang}) if !cls.ok() { return cls, "" } // Output-sanitizer gate (D30.3): a deterministic verdict-axis check for "instant unreadability" // defects no other gate catches — a leaked service preamble, a trailing note/edit block, a markdown // ### header, a Latin-script insertion, a contentless CJK-leak run. Opt-in (Gates.Sanitizer). Runs // ONLY on the FINAL stage's output (isFinal) — the text that SHIPS: an intermediate translator draft // is legitimately rough and the EDITOR's job is to clean it, so sanitizing the draft would wrongly // skip the recovering editor and drop the chunk to a placeholder (адверсариальное ревью). It is a // READABILITY gate whose patterns bake in Cyrillic/Russian conventions, so it is gated behind a // Russian target (D39 слой 7, L8-readability-gates-target-blind) — a no-op for any other pair, which // would false-flag (prod zh→ru is unaffected). Runs on the completion text like the intrinsic // classifier, so a resumed checkpoint re-derives the identical verdict; its rule version folds into // the snapshot only when enabled (sanitizerSnapshot). if isFinal && r.Pipeline.Gates.Sanitizer.Enabled && isRuTarget(r.Book.TargetLang) { if san := sanitizeOutput(output); san.total() > 0 { // Cosmetic-only leak (leading markdown header and/or contentless CJK-leak) → strip it and // export the remainder flagged, not drop the whole chunk (D35.4a). The guard is // load-bearing: we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that // is now clean, and we RETURN that stripped text so runStage/resume ship the exact bytes the // guard validated. Anything else falls through to the substantive skip. Pure over `output`, // so a resumed checkpoint re-derives the identical verdict. if san.cosmeticOnly() { if stripped := stripCosmetic(output); strings.TrimSpace(stripped) != "" && sanitizeOutput(stripped).total() == 0 { return classification{FlagSanitizerStripped, san.summary()}, stripped } } return classification{FlagSanitizerDefect, san.summary()}, "" } } if !r.Pipeline.Gates.Coverage.Enabled { return cls, "" } // The coverage gate compares the output against the ORIGINAL source, which is a // source→target translation only for the TRANSLATOR role. A monolingual editor // legitimately restructures/merges sentences, so gating ITS output against the // source with the translation corridor false-flags a correct edit as excision // (self-review finding). Editor/other-role fidelity is a Phase-2 concern (a // bilingual judge over the draft, §04 mode 2), not this excision gate. if role != roleTranslator { return cls, "" } if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() { return cov.cls, "" } return cls, "" } // injectionRenderer serializes a chunk's selected memory records into a role's injection message. type injectionRenderer func(injected []pickedEntry) string // roleInjectionRenderers maps a stage ROLE to the memory-injection renderer it consumes — a registry // instead of a hand-edited switch (D39 слой 7, L8-role-injection-hardcoded-switch), so a new Ф2 // consumer role (annotator, voice-layer — README) is DATA + one renderer, not surgery on a switch. // A role absent from the map gets no injection. Renderers are pure, so the per-role result is // deterministic regardless of map iteration order (no map-order in OUTPUT — invariant #6). var roleInjectionRenderers = map[string]injectionRenderer{ roleTranslator: renderGlossaryBlock, // src→dst glossary block roleEditor: renderEditorConstraintBlock, // CONFIRMED dst forms (target-consistency constraints) } // persistRetrievalState writes the per-chunk observability record from the deterministic // selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED // records (what the model saw); spoiler/eviction are the dropped-and-logged totals; // post-check misses are recorded only when the translator actually produced text. func (r *Runner) persistRetrievalState(snapID string, ch Chunk, sel memorySelection, misses []postcheckMiss, outputChecked bool, cheap cheapGateResult, bank bankFlags) error { rs := store.RetrievalState{ BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, SnapshotID: snapID, NStyleFlags: cheap.total(), NBanknoteLines: bank.NLines, BanknoteParseFail: boolToStoreInt(bank.ParseFail), BanknoteTruncated: boolToStoreInt(bank.Truncated), } if cheap.total() > 0 { if b, err := json.Marshal(cheap); err == nil { rs.StyleDetail = string(b) } } for _, p := range sel.injected { if p.via == "sticky" { rs.NSticky++ } else { rs.NExactHits++ } if p.disp == memAmbiguous { rs.NAmbiguousFlagged++ } } rs.NSpoilerBlocked = len(sel.rejected) rs.NEvicted = len(sel.evicted) // Loud record of the disposition-gated suppressor (D39 слой 4): a longer lower-trust key refused // from eating a nested higher-trust one — the term-drift code root, now visible instead of a // silent drop (research/13 §7). Detail carries the refused suppressor→protected pairs for a human. rs.NTrustGatedSuppress = len(sel.trustGated) if len(sel.trustGated) > 0 { if b, err := json.Marshal(sel.trustGated); err == nil { rs.TrustGateDetail = string(b) } } if outputChecked { // n_postcheck_miss is the CONFIRMED-miss count (the actionable consistency-failure // signal, external-review #1); the detail carries ALL misses (confirmed AND the // ambiguous forced-post-check ones, each disp-tagged) for the human. rs.NPostcheckMiss = countConfirmedMisses(misses) if len(misses) > 0 { if b, err := json.Marshal(misses); err == nil { rs.PostcheckDetail = string(b) } } } ids := slices.Sorted(maps.Keys(sel.activeIDs)) if ids == nil { // slices.Sorted даёт NIL на пустой карте → json.Marshal рендерил бы "null", // а исторический формат персистентной колонки — "[]" (чанк без точных // матчей — обычный кейс); держим байты стабильными (находка селфревью №4). ids = []string{} } if b, err := json.Marshal(ids); err == nil { rs.InjectedIDs = string(b) } return r.Store.UpsertRetrievalState(rs) } // boolToStoreInt maps a bool telemetry flag to the store's 0/1 integer column form. func boolToStoreInt(b bool) int { if b { return 1 } return 0 } // cheapGateConfig builds the cheap-gate knobs from the book brief: the ё-policy and the // lower-cased per-project interjection allowlist. Pure, no store access. func (r *Runner) cheapGateConfig() cheapGateConfig { allow := make(map[string]bool, len(r.Book.StyleAllowlist)) for _, s := range r.Book.StyleAllowlist { allow[s] = true } return cheapGateConfig{ yoPolicy: r.Book.YoPolicy, allowlist: allow, regressionEnabled: r.Pipeline.Gates.RegressionGuard.Enabled, } }