234 lines
11 KiB
Go
234 lines
11 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"maps"
|
||
"slices"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// chunkrun.go: disposition-петля уровня ЧАНКА (D2) — стадии по порядку, первый флаг
|
||
// останавливает чанк (дальше skipped, платной редактуры мусора нет), плюс горячий
|
||
// путь памяти v2 (Select→инъекция per-role→post-check E1) и дешёвые стиль-флаггеры.
|
||
// Новые роли-потребители инъекции Ф2 (annotator, voice-слой) добавляются здесь.
|
||
|
||
// classifyOutput resolves a completion's disposition: FIRST the always-on intrinsic
|
||
// classifier (echo/empty/refusal/length — Веха 2), then, only if that is ok AND the
|
||
// configurable coverage gate is enabled, the excision gate (шаг 6 / D12 Q3). Both are
|
||
// pure and deterministic over (source, output, finish), so a resumed checkpoint
|
||
// reproduces the identical verdict for free — the intrinsic part unconditionally, the
|
||
// gate part under the same coverage config (folded into the snapshot, so a gate change
|
||
// re-pins loudly). The coverage gate runs ONLY on the TRANSLATOR role's output (vs the
|
||
// original source): a monolingual editor legitimately restructures sentences, so gating
|
||
// it against the source would false-flag a correct edit (see the role check below).
|
||
func (r *Runner) classifyOutput(role, source, output, finish string) classification {
|
||
cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang})
|
||
if !cls.ok() || !r.Pipeline.Gates.Coverage.Enabled {
|
||
return cls
|
||
}
|
||
// The coverage gate compares the output against the ORIGINAL source, which is a
|
||
// source→target translation only for the TRANSLATOR role. A monolingual editor
|
||
// legitimately restructures/merges sentences, so gating ITS output against the
|
||
// source with the translation corridor false-flags a correct edit as excision
|
||
// (self-review finding). Editor/other-role fidelity is a Phase-2 concern (a
|
||
// bilingual judge over the draft, §04 mode 2), not this excision gate.
|
||
if role != roleTranslator {
|
||
return cls
|
||
}
|
||
if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() {
|
||
return cov.cls
|
||
}
|
||
return cls
|
||
}
|
||
|
||
// translateChunk drives ONE chunk through the stage list. Stages run in order,
|
||
// each feeding the next; the FIRST flagged stage stops the chunk — later stages
|
||
// are recorded `skipped` (no paid edit over a garbage draft, D2). It also runs the
|
||
// memory bank v2 hot path: it Selects the glossary injection once ($0, deterministic),
|
||
// injects it into the TRANSLATOR stage, post-checks the translator output (E1), and
|
||
// persists the per-chunk retrieval-state (observability). Returns the chunk outcome, the
|
||
// exact-matched entity ids (the next chunk's sticky_prev, A5), and an error only on an
|
||
// infra failure. stickyPrev is the prior chunks' exact matches in this chapter.
|
||
func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, stickyPrev map[string]injectionDisposition) (*ChunkOutcome, map[string]injectionDisposition, error) {
|
||
out := &ChunkOutcome{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: DispOK}
|
||
prev := ""
|
||
flagged := false
|
||
var flagReason FlagReason
|
||
|
||
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and
|
||
// serialize the translator injection block. Recomputed on every run (resumed chunks
|
||
// included) so the sticky window and the injected bytes are resume-stable.
|
||
var memSel memorySelection
|
||
var translatorInjection, editorInjection string
|
||
activeIDs := map[string]injectionDisposition{}
|
||
if r.memory != nil {
|
||
memSel = r.memory.Select(ch.Text, ch.Chapter, stickyPrev, r.Pipeline.Context.GlossaryTokenBudget)
|
||
translatorInjection = renderGlossaryBlock(memSel.injected)
|
||
editorInjection = renderEditorConstraintBlock(memSel.injected)
|
||
activeIDs = memSel.activeIDs
|
||
}
|
||
|
||
for stageIdx, st := range r.Pipeline.Stages {
|
||
if flagged {
|
||
// An earlier stage flagged → this stage is not attempted or billed.
|
||
detail := fmt.Sprintf("skipped: an upstream stage was flagged (%s)", flagReason)
|
||
if err := r.Store.UpsertChunkStatus(store.ChunkStatus{
|
||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: st.Name,
|
||
SnapshotID: snapID, Disposition: string(DispSkipped), FlagReason: string(flagReason),
|
||
Detail: detail,
|
||
}); err != nil {
|
||
return out, activeIDs, fmt.Errorf("pipeline: record skipped chunk_status: %w", err)
|
||
}
|
||
out.Stages = append(out.Stages, StageResult{
|
||
Stage: st.Name, Role: st.Role, Model: st.Model,
|
||
Disposition: DispSkipped, FlagReason: flagReason, Detail: detail,
|
||
})
|
||
continue
|
||
}
|
||
|
||
// Inject the glossary per role. The TRANSLATOR gets the src→dst block (its keys are
|
||
// matched against the source chunk); the monolingual EDITOR gets the CONFIRMED dst
|
||
// forms as target-consistency constraints (no source — decisions-log D1); every other
|
||
// stage gets none. Empty injection → the plain 2-message layout. The injection is a
|
||
// message, so it enters this stage's request_hash automatically (a resumed chunk
|
||
// reproduces it deterministically from the frozen bank).
|
||
injection := ""
|
||
switch st.Role {
|
||
case roleTranslator:
|
||
injection = translatorInjection
|
||
case roleEditor:
|
||
injection = editorInjection
|
||
}
|
||
sr, err := r.runStage(ctx, st, stageIdx, snapID, ch, prev, injection)
|
||
if err != nil {
|
||
return out, activeIDs, err
|
||
}
|
||
out.Stages = append(out.Stages, *sr)
|
||
out.CostUSD += sr.CostUSD
|
||
if sr.Disposition == DispFlagged {
|
||
flagged = true
|
||
flagReason = sr.FlagReason
|
||
continue
|
||
}
|
||
prev = sr.Text
|
||
}
|
||
|
||
// Post-check the FINAL, exported output (E1): the reader sees the last stage's text
|
||
// (the editor's), not the translator draft — the monolingual editor can drift an
|
||
// approved term the translator got right, so checking the draft alone would miss it.
|
||
// Runs on the fresh OR fully-resumed chunk (both carry the final text through `prev`),
|
||
// so it is resume-reproducible. In the default FLAGGER mode a miss is recorded only in
|
||
// the retrieval-state (observability); with the opt-in gate a miss flags the chunk
|
||
// (glossary_miss). Skipped when a stage already flagged (no usable output to check).
|
||
var postMisses []postcheckMiss
|
||
outputChecked := false
|
||
if r.memory != nil && !flagged && prev != "" {
|
||
outputChecked = true
|
||
postMisses = r.memory.postcheck(memSel.injected, prev)
|
||
// The gate flips ONLY on CONFIRMED (approved) misses — an AMBIGUOUS miss is an
|
||
// unverified candidate the model may legitimately reject, so it must not discard a
|
||
// correct translation (external-review major #1).
|
||
if r.Pipeline.Gates.Glossary.PostcheckGate && countConfirmedMisses(postMisses) > 0 {
|
||
flagged = true
|
||
flagReason = FlagGlossaryMiss
|
||
r.Log.WarnContext(ctx, "glossary post-check gate flagged the chunk",
|
||
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "confirmed_misses", countConfirmedMisses(postMisses))
|
||
}
|
||
}
|
||
|
||
// Cheap deterministic style/number flaggers on the FINAL text (cheapgates.go): observability
|
||
// only, never a disposition. Runs on the same fresh-or-resumed output as the post-check, so it
|
||
// is resume-reproducible; skipped when a stage flagged (no usable output). Source is ch.Text
|
||
// (the 万/億 magnitude gate compares source↔output).
|
||
var cheap cheapGateResult
|
||
if !flagged && prev != "" {
|
||
cheap = runCheapGates(ch.Text, prev, r.cheapGateConfig())
|
||
if cheap.total() > 0 {
|
||
r.Log.InfoContext(ctx, "cheap style gates flagged the chunk (observability, not a gate)",
|
||
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "style_flags", cheap.total(),
|
||
"dialogue_dash", cheap.DialogueDash, "yo", cheap.YoInconsistent,
|
||
"translit_interj", cheap.TranslitInterj, "number_magnitude", cheap.NumberMagnitude)
|
||
}
|
||
}
|
||
|
||
// Persist the per-chunk retrieval-state (registry gate #4: convert silent memory
|
||
// degradation into a loud, visible record) plus the cheap style-flag counts. Deterministic +
|
||
// idempotent, so a resume re-derives the identical row. Only when a glossary is materialized
|
||
// (always true in a normal run — seedGlossary sets an at-least-empty bank).
|
||
if r.memory != nil {
|
||
if err := r.persistRetrievalState(snapID, ch, memSel, postMisses, outputChecked, cheap); err != nil {
|
||
return out, activeIDs, err
|
||
}
|
||
}
|
||
|
||
if flagged {
|
||
out.Disposition = DispFlagged
|
||
out.FlagReason = flagReason
|
||
out.FinalText = "" // garbage/refusal/glossary-miss never propagates to the next chunk or export
|
||
} else {
|
||
out.FinalText = prev
|
||
}
|
||
return out, activeIDs, nil
|
||
}
|
||
|
||
// persistRetrievalState writes the per-chunk observability record from the deterministic
|
||
// selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED
|
||
// records (what the model saw); spoiler/eviction are the dropped-and-logged totals;
|
||
// post-check misses are recorded only when the translator actually produced text.
|
||
func (r *Runner) persistRetrievalState(snapID string, ch Chunk, sel memorySelection, misses []postcheckMiss, outputChecked bool, cheap cheapGateResult) error {
|
||
rs := store.RetrievalState{
|
||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, SnapshotID: snapID,
|
||
NStyleFlags: cheap.total(),
|
||
}
|
||
if cheap.total() > 0 {
|
||
if b, err := json.Marshal(cheap); err == nil {
|
||
rs.StyleDetail = string(b)
|
||
}
|
||
}
|
||
for _, p := range sel.injected {
|
||
if p.via == "sticky" {
|
||
rs.NSticky++
|
||
} else {
|
||
rs.NExactHits++
|
||
}
|
||
if p.disp == memAmbiguous {
|
||
rs.NAmbiguousFlagged++
|
||
}
|
||
}
|
||
rs.NSpoilerBlocked = len(sel.rejected)
|
||
rs.NEvicted = len(sel.evicted)
|
||
if outputChecked {
|
||
// n_postcheck_miss is the CONFIRMED-miss count (the actionable consistency-failure
|
||
// signal, external-review #1); the detail carries ALL misses (confirmed AND the
|
||
// ambiguous forced-post-check ones, each disp-tagged) for the human.
|
||
rs.NPostcheckMiss = countConfirmedMisses(misses)
|
||
if len(misses) > 0 {
|
||
if b, err := json.Marshal(misses); err == nil {
|
||
rs.PostcheckDetail = string(b)
|
||
}
|
||
}
|
||
}
|
||
ids := slices.Sorted(maps.Keys(sel.activeIDs))
|
||
if ids == nil {
|
||
// slices.Sorted даёт NIL на пустой карте → json.Marshal рендерил бы "null",
|
||
// а исторический формат персистентной колонки — "[]" (чанк без точных
|
||
// матчей — обычный кейс); держим байты стабильными (находка селфревью №4).
|
||
ids = []string{}
|
||
}
|
||
if b, err := json.Marshal(ids); err == nil {
|
||
rs.InjectedIDs = string(b)
|
||
}
|
||
return r.Store.UpsertRetrievalState(rs)
|
||
}
|
||
|
||
// cheapGateConfig builds the cheap-gate knobs from the book brief: the ё-policy and the
|
||
// lower-cased per-project interjection allowlist. Pure, no store access.
|
||
func (r *Runner) cheapGateConfig() cheapGateConfig {
|
||
allow := make(map[string]bool, len(r.Book.StyleAllowlist))
|
||
for _, s := range r.Book.StyleAllowlist {
|
||
allow[s] = true
|
||
}
|
||
return cheapGateConfig{yoPolicy: r.Book.YoPolicy, allowlist: allow}
|
||
}
|