175 lines
9 KiB
Go
175 lines
9 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"encoding/json"
|
||
"maps"
|
||
"slices"
|
||
"strings"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// chunkrun.go: disposition-петля уровня ЧАНКА (D2) — стадии по порядку, первый флаг
|
||
// останавливает чанк (дальше skipped, платной редактуры мусора нет), плюс горячий
|
||
// путь памяти v2 (Select→инъекция per-role→post-check E1) и дешёвые стиль-флаггеры.
|
||
// Новые роли-потребители инъекции Ф2 (annotator, voice-слой) добавляются здесь.
|
||
|
||
// classifyOutput resolves a completion's disposition: FIRST the always-on intrinsic
|
||
// classifier (echo/empty/refusal/length — Веха 2), then, only if that is ok AND the
|
||
// configurable coverage gate is enabled, the excision gate (шаг 6 / D12 Q3). Both are
|
||
// pure and deterministic over (source, output, finish), so a resumed checkpoint
|
||
// reproduces the identical verdict for free — the intrinsic part unconditionally, the
|
||
// gate part under the same coverage config (folded into the snapshot, so a gate change
|
||
// re-pins loudly). The coverage gate runs ONLY on the TRANSLATOR role's output (vs the
|
||
// original source): a monolingual editor legitimately restructures sentences, so gating
|
||
// it against the source would false-flag a correct edit (see the role check below).
|
||
// classifyOutput returns the disposition AND, for a cosmetic sanitizer strip, the resolved EXPORT
|
||
// text (the strip remainder) computed ONCE here (T3.4 / L8-stripcosmetic-recomputed-twice): the
|
||
// second return travels back to runStage as the authoritative export, so no site re-derives
|
||
// stripCosmetic and the "non-empty & clean" invariant is asserted over the very bytes that ship.
|
||
// The string is "" for every disposition except FlagSanitizerStripped.
|
||
func (r *Runner) classifyOutput(role, source, output, finish string, isFinal bool) (classification, string) {
|
||
cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang})
|
||
if !cls.ok() {
|
||
return cls, ""
|
||
}
|
||
// Output-sanitizer gate (D30.3): a deterministic verdict-axis check for "instant unreadability"
|
||
// defects no other gate catches — a leaked service preamble, a trailing note/edit block, a markdown
|
||
// ### header, a Latin-script insertion, a contentless CJK-leak run. Opt-in (Gates.Sanitizer). Runs
|
||
// ONLY on the FINAL stage's output (isFinal) — the text that SHIPS: an intermediate translator draft
|
||
// is legitimately rough and the EDITOR's job is to clean it, so sanitizing the draft would wrongly
|
||
// skip the recovering editor and drop the chunk to a placeholder (адверсариальное ревью). It is a
|
||
// READABILITY gate whose patterns bake in Cyrillic/Russian conventions, so it is gated behind a
|
||
// Russian target (D39 слой 7, L8-readability-gates-target-blind) — a no-op for any other pair, which
|
||
// would false-flag (prod zh→ru is unaffected). Runs on the completion text like the intrinsic
|
||
// classifier, so a resumed checkpoint re-derives the identical verdict; its rule version folds into
|
||
// the snapshot only when enabled (sanitizerSnapshot).
|
||
if isFinal && r.Pipeline.Gates.Sanitizer.Enabled && isRuTarget(r.Book.TargetLang) {
|
||
if san := sanitizeOutput(output); san.total() > 0 {
|
||
// Cosmetic-only leak (leading markdown header and/or contentless CJK-leak) → strip it and
|
||
// export the remainder flagged, not drop the whole chunk (D35.4a). The guard is
|
||
// load-bearing: we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that
|
||
// is now clean, and we RETURN that stripped text so runStage/resume ship the exact bytes the
|
||
// guard validated. Anything else falls through to the substantive skip. Pure over `output`,
|
||
// so a resumed checkpoint re-derives the identical verdict.
|
||
if san.cosmeticOnly() {
|
||
if stripped := stripCosmetic(output); strings.TrimSpace(stripped) != "" && sanitizeOutput(stripped).total() == 0 {
|
||
return classification{FlagSanitizerStripped, san.summary()}, stripped
|
||
}
|
||
}
|
||
return classification{FlagSanitizerDefect, san.summary()}, ""
|
||
}
|
||
}
|
||
if !r.Pipeline.Gates.Coverage.Enabled {
|
||
return cls, ""
|
||
}
|
||
// The coverage gate compares the output against the ORIGINAL source, which is a
|
||
// source→target translation only for the TRANSLATOR role. A monolingual editor
|
||
// legitimately restructures/merges sentences, so gating ITS output against the
|
||
// source with the translation corridor false-flags a correct edit as excision
|
||
// (self-review finding). Editor/other-role fidelity is a Phase-2 concern (a
|
||
// bilingual judge over the draft, §04 mode 2), not this excision gate.
|
||
if role != roleTranslator {
|
||
return cls, ""
|
||
}
|
||
if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() {
|
||
return cov.cls, ""
|
||
}
|
||
return cls, ""
|
||
}
|
||
|
||
// injectionRenderer serializes a chunk's selected memory records into a role's injection message.
|
||
type injectionRenderer func(injected []pickedEntry) string
|
||
|
||
// roleInjectionRenderers maps a stage ROLE to the memory-injection renderer it consumes — a registry
|
||
// instead of a hand-edited switch (D39 слой 7, L8-role-injection-hardcoded-switch), so a new Ф2
|
||
// consumer role (annotator, voice-layer — README) is DATA + one renderer, not surgery on a switch.
|
||
// A role absent from the map gets no injection. Renderers are pure, so the per-role result is
|
||
// deterministic regardless of map iteration order (no map-order in OUTPUT — invariant #6).
|
||
var roleInjectionRenderers = map[string]injectionRenderer{
|
||
roleTranslator: renderGlossaryBlock, // src→dst glossary block
|
||
roleEditor: renderEditorConstraintBlock, // CONFIRMED dst forms (target-consistency constraints)
|
||
}
|
||
|
||
// persistRetrievalState writes the per-chunk observability record from the deterministic
|
||
// selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED
|
||
// records (what the model saw); spoiler/eviction are the dropped-and-logged totals;
|
||
// post-check misses are recorded only when the translator actually produced text.
|
||
func (r *Runner) persistRetrievalState(snapID string, ch Chunk, sel memorySelection, misses []postcheckMiss, outputChecked bool, cheap cheapGateResult, bank bankFlags) error {
|
||
rs := store.RetrievalState{
|
||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, SnapshotID: snapID,
|
||
NStyleFlags: cheap.total(),
|
||
NBanknoteLines: bank.NLines,
|
||
BanknoteParseFail: boolToStoreInt(bank.ParseFail),
|
||
BanknoteTruncated: boolToStoreInt(bank.Truncated),
|
||
}
|
||
if cheap.total() > 0 {
|
||
if b, err := json.Marshal(cheap); err == nil {
|
||
rs.StyleDetail = string(b)
|
||
}
|
||
}
|
||
for _, p := range sel.injected {
|
||
if p.via == "sticky" {
|
||
rs.NSticky++
|
||
} else {
|
||
rs.NExactHits++
|
||
}
|
||
if p.disp == memAmbiguous {
|
||
rs.NAmbiguousFlagged++
|
||
}
|
||
}
|
||
rs.NSpoilerBlocked = len(sel.rejected)
|
||
rs.NEvicted = len(sel.evicted)
|
||
// Loud record of the disposition-gated suppressor (D39 слой 4): a longer lower-trust key refused
|
||
// from eating a nested higher-trust one — the term-drift code root, now visible instead of a
|
||
// silent drop (research/13 §7). Detail carries the refused suppressor→protected pairs for a human.
|
||
rs.NTrustGatedSuppress = len(sel.trustGated)
|
||
if len(sel.trustGated) > 0 {
|
||
if b, err := json.Marshal(sel.trustGated); err == nil {
|
||
rs.TrustGateDetail = string(b)
|
||
}
|
||
}
|
||
if outputChecked {
|
||
// n_postcheck_miss is the CONFIRMED-miss count (the actionable consistency-failure
|
||
// signal, external-review #1); the detail carries ALL misses (confirmed AND the
|
||
// ambiguous forced-post-check ones, each disp-tagged) for the human.
|
||
rs.NPostcheckMiss = countConfirmedMisses(misses)
|
||
if len(misses) > 0 {
|
||
if b, err := json.Marshal(misses); err == nil {
|
||
rs.PostcheckDetail = string(b)
|
||
}
|
||
}
|
||
}
|
||
ids := slices.Sorted(maps.Keys(sel.activeIDs))
|
||
if ids == nil {
|
||
// slices.Sorted даёт NIL на пустой карте → json.Marshal рендерил бы "null",
|
||
// а исторический формат персистентной колонки — "[]" (чанк без точных
|
||
// матчей — обычный кейс); держим байты стабильными (находка селфревью №4).
|
||
ids = []string{}
|
||
}
|
||
if b, err := json.Marshal(ids); err == nil {
|
||
rs.InjectedIDs = string(b)
|
||
}
|
||
return r.Store.UpsertRetrievalState(rs)
|
||
}
|
||
|
||
// boolToStoreInt maps a bool telemetry flag to the store's 0/1 integer column form.
|
||
func boolToStoreInt(b bool) int {
|
||
if b {
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|
||
|
||
// cheapGateConfig builds the cheap-gate knobs from the book brief: the ё-policy and the
|
||
// lower-cased per-project interjection allowlist. Pure, no store access.
|
||
func (r *Runner) cheapGateConfig() cheapGateConfig {
|
||
allow := make(map[string]bool, len(r.Book.StyleAllowlist))
|
||
for _, s := range r.Book.StyleAllowlist {
|
||
allow[s] = true
|
||
}
|
||
return cheapGateConfig{
|
||
yoPolicy: r.Book.YoPolicy,
|
||
allowlist: allow,
|
||
regressionEnabled: r.Pipeline.Gates.RegressionGuard.Enabled,
|
||
}
|
||
}
|