535 lines
24 KiB
Go
535 lines
24 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"strings"
|
||
"sync"
|
||
|
||
"textmachine/backend/internal/config"
|
||
"textmachine/backend/internal/store"
|
||
)
|
||
|
||
// waverun.go: the R1 WAVE EXECUTOR — the driver switch from the sequential per-chunk-all-stages loop
|
||
// (bookrun.go/chunkrun.go, retired) to the precompute pass ($0) → the draft wave (draft stage ∥ over draft CHUNKS) →
|
||
// the bank-mining stop → the edit wave (edit stage ∥ over edit UNITS). The load-bearing decoupling (plan §1/§2):
|
||
// the DRAFT is the fine per-chunk unit (COGS/coverage/alignment), the EDIT is the coarse per-unit unit
|
||
// (a chapter-scale grouping of whole draft chunks — cross-chunk cohesion, D39 п.4). the edit wave reads a unit's draft
|
||
// as the concatenation of its members' draft-wave outputs and does a FRESH memory.Select over the whole-unit source;
|
||
// it NEVER re-renders the draft stage (the несущий invariant — a re-render over the enriched bank would move
|
||
// the draft request_hash and re-bill the draft wave). Money stays single-writer-safe (Reserve/Settle serialize); the
|
||
// waves add only read-only shared state (clients/templates/memory/rate-guards built in the precompute pass). Per-wave
|
||
// snapshots (draft-wave snapshot base-bank, edit-wave snapshot enriched) keep «переоплата ОДНА»: a the bank-mining stop enrichment moves
|
||
// only edit-wave snapshot, so draft-wave checkpoints stay valid.
|
||
|
||
// wavedStage carries a wave stage with its GLOBAL index in Pipeline.Stages — runStage needs the global
|
||
// index for isFinal (the LAST stage is the shipping stage the sanitizer runs on) and for the >0 sizing
|
||
// path (a later stage sizes max_tokens from its input draft, not the source, D2.5).
|
||
type wavedStage struct {
|
||
st config.Stage
|
||
idx int
|
||
}
|
||
|
||
// waveStagesIndexed partitions Pipeline.Stages by role for a wave, carrying each stage's global index
|
||
// (mirrors waveStages but retains the index runStage needs). the draft wave = translator (draft) stages; the edit wave = every
|
||
// non-translator (editor/other) stage. Order preserved.
|
||
func (r *Runner) waveStagesIndexed(w wave) []wavedStage {
|
||
var out []wavedStage
|
||
for i, st := range r.Pipeline.Stages {
|
||
isDraft := st.Role == roleTranslator
|
||
if (w == waveDraft) == isDraft {
|
||
out = append(out, wavedStage{st: st, idx: i})
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// stageNameSet is the set of a wave's stage names — used by the read-models to classify a stored
|
||
// chunk_status row by wave (its snapshot belongs to that wave's per-wave snapshot).
|
||
func stageNameSet(staged []wavedStage) map[string]bool {
|
||
m := make(map[string]bool, len(staged))
|
||
for _, ws := range staged {
|
||
m[ws.st.Name] = true
|
||
}
|
||
return m
|
||
}
|
||
|
||
// outputUnits is the SHIPPING granularity the read-models (export/status) project — matching the per-unit
|
||
// BookResult: edit units (member draft chunks grouped) for an edit pipeline, or one singleton unit per
|
||
// draft chunk for a draft-only pipeline (the draft itself ships). One helper so export and status agree.
|
||
func (r *Runner) outputUnits(chunks []Chunk) []editUnit {
|
||
if r.finalStageWave() == waveEdit {
|
||
return buildEditUnits(chunks)
|
||
}
|
||
out := make([]editUnit, len(chunks))
|
||
for i, ch := range chunks {
|
||
out[i] = editUnit{EditUnitID: ch.EditUnitID, Chapter: ch.Chapter, FirstChunkIdx: ch.ChunkIdx, Members: []Chunk{ch}}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// WaveSignatureStop is the typed sentinel the CLI surfaces when bank-mining proposes a non-empty seed-delta:
|
||
// the run completes the draft wave, writes the owner signature map, and STOPS before the edit wave (plan §1(в) default — an explicit
|
||
// stop boundary, not a mid-run append). The owner reviews the signature map, curates the mined-delta file
|
||
// (promoting terms to approved with a dst), and re-runs: the draft wave resumes at $0, the bank-mining stop re-mines (the now-seeded
|
||
// terms are excluded → the delta empties → auto-continue), and the edit wave runs under edit-wave snapshot. NOT an infra
|
||
// failure — a deliberate human-in-the-loop pause, carried up like CompletedWithFlags.
|
||
type WaveSignatureStop struct {
|
||
Terms int
|
||
SignaturePath string
|
||
}
|
||
|
||
func (e *WaveSignatureStop) Error() string {
|
||
return fmt.Sprintf("bank-mining proposed %d new term(s) — review + sign %s, then resume (the edit wave)", e.Terms, e.SignaturePath)
|
||
}
|
||
|
||
// translateBookWaves is the wave driver (R1). It runs the draft wave (draft ∥), the bank-mining stop, the edit wave (edit ∥) and
|
||
// assembles the BookResult. chunks + stickySel come from the precompute pass (SplitChunks + precomputeSticky), computed by
|
||
// the caller (TranslateBook) after ingest/seed/eager-build. Returns a *WaveSignatureStop when the bank-mining stop stops.
|
||
func (r *Runner) translateBookWaves(ctx context.Context, chunks []Chunk, stickySel []memorySelection) (*BookResult, error) {
|
||
draftStages := r.waveStagesIndexed(waveDraft)
|
||
editStages := r.waveStagesIndexed(waveEdit)
|
||
workers := r.Pipeline.Waves.Workers
|
||
editWave := len(editStages) > 0 // a real editor wave exists; else the draft IS the shipping output (draft-only)
|
||
|
||
// --- the draft wave: draft stage(s) ∥ over draft chunks under draft-wave snapshot (base-bank version) ---
|
||
draftSnapshot, draftPayload, err := r.snapshotIDForWave(waveDraft)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if err := r.Store.UpsertSnapshot(draftSnapshot, r.Book.BriefHash(), draftPayload); err != nil {
|
||
return nil, fmt.Errorf("pipeline: upsert draft-wave snapshot %.12s: %w", draftSnapshot, err)
|
||
}
|
||
r.Log.InfoContext(ctx, "draft wave started", "snapshot", draftSnapshot[:12],
|
||
"chunks", len(chunks), "workers", workers, "draft_stages", len(draftStages))
|
||
draftResults := make([]stageSeqResult, len(chunks))
|
||
if err := r.runWave(ctx, workers, len(chunks), func(ctx context.Context, i int) error {
|
||
d, err := r.runDraftChunk(ctx, draftSnapshot, chunks[i], stickySel[i], draftStages, !editWave)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
draftResults[i] = d
|
||
return nil
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// --- the bank-mining stop: bank-mining stop boundary (auto-continues when mining is not configured) ---
|
||
if stopped, err := r.runBankMiningStop(ctx, chunks); err != nil {
|
||
return nil, err
|
||
} else if stopped {
|
||
return nil, &WaveSignatureStop{Terms: r.lastMinedCount, SignaturePath: r.signatureMapPath()}
|
||
}
|
||
|
||
res := &BookResult{BookID: r.Book.BookID}
|
||
|
||
// --- Draft-only pipeline: the draft IS the shipping output; assemble per draft chunk (no edit units) ---
|
||
if !editWave {
|
||
for i, ch := range chunks {
|
||
oc := draftOnlyOutcome(ch, draftResults[i])
|
||
res.Chunks = append(res.Chunks, oc)
|
||
res.TotalUSD += oc.CostUSD
|
||
if oc.Disposition == DispFlagged {
|
||
res.Flagged++
|
||
}
|
||
}
|
||
r.Log.InfoContext(ctx, "book run finished (draft-only)", "book", r.Book.BookID,
|
||
"chunks", len(res.Chunks), "flagged", res.Flagged, "run_usd", fmt.Sprintf("%.6f", res.TotalUSD))
|
||
return res, nil
|
||
}
|
||
|
||
// --- the edit wave: edit stage(s) ∥ over edit units under edit-wave snapshot (enriched-bank version) ---
|
||
editSnapshot, editPayload, err := r.snapshotIDForWave(waveEdit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if err := r.Store.UpsertSnapshot(editSnapshot, r.Book.BriefHash(), editPayload); err != nil {
|
||
return nil, fmt.Errorf("pipeline: upsert edit-wave snapshot %.12s: %w", editSnapshot, err)
|
||
}
|
||
units := buildEditUnits(chunks)
|
||
draftByKey := make(map[chunkKey]stageSeqResult, len(chunks))
|
||
for i, ch := range chunks {
|
||
draftByKey[chunkKey{ch.Chapter, ch.ChunkIdx}] = draftResults[i]
|
||
}
|
||
r.Log.InfoContext(ctx, "edit wave started", "snapshot", editSnapshot[:12],
|
||
"units", len(units), "workers", workers, "edit_stages", len(editStages))
|
||
unitOutcomes := make([]*ChunkOutcome, len(units))
|
||
if err := r.runWave(ctx, workers, len(units), func(ctx context.Context, i int) error {
|
||
oc, err := r.runEditUnit(ctx, editSnapshot, units[i], draftByKey, editStages)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
unitOutcomes[i] = oc
|
||
return nil
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
for _, oc := range unitOutcomes {
|
||
res.Chunks = append(res.Chunks, *oc)
|
||
res.TotalUSD += oc.CostUSD
|
||
if oc.Disposition == DispFlagged {
|
||
res.Flagged++
|
||
}
|
||
}
|
||
r.Log.InfoContext(ctx, "book run finished", "book", r.Book.BookID,
|
||
"units", len(res.Chunks), "flagged", res.Flagged, "run_usd", fmt.Sprintf("%.6f", res.TotalUSD))
|
||
return res, nil
|
||
}
|
||
|
||
// runWave fans `n` items out over `workers` goroutines, calling work(ctx, i) for each index. The first
|
||
// error cancels the rest (via a derived context) and is returned; work items are independent (a wave has no
|
||
// shared mutable input), and money/store writes serialize through the single-writer pool, so this is
|
||
// race-clean. Results are written by the callback into a caller-owned slice indexed by i, so the result
|
||
// ORDER is deterministic (item index) regardless of completion order — the golden capture sorts the wire/log
|
||
// side-effects separately. workers ≤ 0 is treated as 1 (LoadPipeline already clamps).
|
||
func (r *Runner) runWave(parent context.Context, workers, n int, work func(ctx context.Context, i int) error) error {
|
||
if workers < 1 {
|
||
workers = 1
|
||
}
|
||
if n == 0 {
|
||
return nil
|
||
}
|
||
ctx, cancel := context.WithCancel(parent)
|
||
defer cancel()
|
||
idxCh := make(chan int)
|
||
var wg sync.WaitGroup
|
||
var mu sync.Mutex
|
||
var firstErr error
|
||
fail := func(err error) {
|
||
mu.Lock()
|
||
if firstErr == nil {
|
||
firstErr = err
|
||
cancel()
|
||
}
|
||
mu.Unlock()
|
||
}
|
||
for w := 0; w < workers; w++ {
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for i := range idxCh {
|
||
if err := work(ctx, i); err != nil {
|
||
fail(err)
|
||
return
|
||
}
|
||
}
|
||
}()
|
||
}
|
||
for i := 0; i < n; i++ {
|
||
select {
|
||
case idxCh <- i:
|
||
case <-ctx.Done():
|
||
}
|
||
}
|
||
close(idxCh)
|
||
wg.Wait()
|
||
return firstErr
|
||
}
|
||
|
||
// stageSeqResult is the outcome of running one chunk/unit through an ORDERED list of stages (one wave):
|
||
// the per-stage results, the final OK stage's text, and the terminal flag state.
|
||
type stageSeqResult struct {
|
||
stages []StageResult
|
||
finalText string // the last OK stage's Text (empty until a stage runs; unused when flagged)
|
||
flagged bool
|
||
flagReason FlagReason
|
||
recovered string // sanitizer-stripped export text from the flagging stage (D35.4a)
|
||
cost float64 // THIS run's spend across the sequence
|
||
bankTelem bankFlags // the translator-role stage's banknote telemetry (WS4 point 10)
|
||
}
|
||
|
||
// runStageSequence runs `staged` in order over one chunk/unit, feeding each stage's output to the next as
|
||
// `prev` (starting from startPrev), stopping the sequence at the FIRST flagged stage (later stages are
|
||
// recorded `skipped` — no paid edit over a garbage draft, D2). It is the generalized inner loop of the
|
||
// retired translateChunk, reused by BOTH waves (draft over a chunk, the edit-wave editor over a unit). injectionByRole
|
||
// maps a stage role to its already-rendered memory-injection message (the translator's src→dst glossary in
|
||
// the draft wave, the editor's CONFIRMED-dst constraint block in the edit wave). Deterministic; content-verified resume via runStage.
|
||
func (r *Runner) runStageSequence(ctx context.Context, staged []wavedStage, snapID string, ch Chunk, startPrev string, injectionByRole map[string]string) (stageSeqResult, error) {
|
||
var res stageSeqResult
|
||
prev := startPrev
|
||
flagged := false
|
||
var flagReason FlagReason
|
||
recovered := ""
|
||
for _, ws := range staged {
|
||
st := ws.st
|
||
if flagged {
|
||
detail := fmt.Sprintf("skipped: an upstream stage was flagged (%s)", flagReason)
|
||
if err := r.Store.UpsertChunkStatus(store.ChunkStatus{
|
||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: st.Name,
|
||
SnapshotID: snapID, Disposition: string(DispSkipped), FlagReason: string(flagReason), Detail: detail,
|
||
}); err != nil {
|
||
return res, fmt.Errorf("pipeline: record skipped chunk_status: %w", err)
|
||
}
|
||
res.stages = append(res.stages, StageResult{
|
||
Stage: st.Name, Role: st.Role, Model: st.Model,
|
||
Disposition: DispSkipped, FlagReason: flagReason, Detail: detail,
|
||
})
|
||
continue
|
||
}
|
||
sr, err := r.runStage(ctx, st, ws.idx, snapID, ch, prev, injectionByRole[st.Role])
|
||
if err != nil {
|
||
return res, err
|
||
}
|
||
res.stages = append(res.stages, *sr)
|
||
res.cost += sr.CostUSD
|
||
if st.Role == roleTranslator {
|
||
res.bankTelem = sr.BankFlags
|
||
// FL-2: the resume fast-path serves the banknote-cleaned final_hash and leaves BankFlags zero;
|
||
// restore the telemetry the fresh run persisted so persistRetrievalState does not zero the three
|
||
// banknote columns on every resume of a ready chunk (guarded on an empty BankFlags so the
|
||
// attempt-loop path — which re-derives telemetry from the raw checkpoint — is never overwritten).
|
||
if res.bankTelem == (bankFlags{}) && sr.FromResume && r.Pipeline.Gates.Banknote.Enabled {
|
||
if prevRS, err := r.Store.GetRetrievalState(r.Book.BookID, ch.Chapter, ch.ChunkIdx); err == nil && prevRS != nil {
|
||
res.bankTelem = bankFlags{
|
||
NLines: prevRS.NBanknoteLines, ParseFail: prevRS.BanknoteParseFail != 0, Truncated: prevRS.BanknoteTruncated != 0,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if sr.Disposition == DispFlagged {
|
||
flagged = true
|
||
flagReason = sr.FlagReason
|
||
recovered = sr.RecoveredText
|
||
continue
|
||
}
|
||
prev = sr.Text
|
||
}
|
||
res.finalText = prev
|
||
res.flagged = flagged
|
||
res.flagReason = flagReason
|
||
res.recovered = recovered
|
||
return res, nil
|
||
}
|
||
|
||
// runDraftChunk runs the draft stage(s) over one chunk under the draft-wave snapshot (plan §1). It renders
|
||
// the translator's src→dst glossary injection from the precomputed per-chunk selection over the BASE bank
|
||
// (baseMemory — mined-excluded, so the injection is stable across a bank-mining enrichment), runs the
|
||
// sequence, and persists the draft retrieval_state (injection counts + banknote telemetry). When this is the
|
||
// FINAL wave (a draft-only pipeline with no editor), the draft IS the shipping text, so the post-check +
|
||
// cheap gates run here over the draft; otherwise they belong to the edit wave (the edit unit's final output).
|
||
func (r *Runner) runDraftChunk(ctx context.Context, draftSnapshot string, ch Chunk, memSel memorySelection, draftStages []wavedStage, isFinalWave bool) (stageSeqResult, error) {
|
||
injectionByRole := map[string]string{}
|
||
if r.baseMemory != nil {
|
||
// Serialize the per-role injection blocks via the role→renderer registry (D39 слой 7) from the
|
||
// precomputed base-bank selection: the translator gets its src→dst glossary block; any other
|
||
// role's block is rendered too but consumed only if a stage of that role runs in this wave.
|
||
for role, render := range roleInjectionRenderers {
|
||
injectionByRole[role] = render(memSel.injected) // pure → map-order-independent
|
||
}
|
||
if len(memSel.trustGated) > 0 {
|
||
r.Log.WarnContext(ctx, "memory: lower-trust longer key refused from suppressing a higher-trust nested key (approved term preserved; reconcile the seed)",
|
||
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "trust_gated", len(memSel.trustGated),
|
||
"first", memSel.trustGated[0].Suppressor+"⊃"+memSel.trustGated[0].Protected)
|
||
}
|
||
}
|
||
seq, err := r.runStageSequence(ctx, draftStages, draftSnapshot, ch, "", injectionByRole)
|
||
if err != nil {
|
||
return seq, err
|
||
}
|
||
|
||
var postMisses []postcheckMiss
|
||
outputChecked := false
|
||
var cheap cheapGateResult
|
||
if isFinalWave && r.baseMemory != nil && !seq.flagged && seq.finalText != "" {
|
||
// Draft-only pipeline: the draft is the shipping output → run the FINAL-output checks here, over the
|
||
// SAME base bank whose terms the draft was injected with (post-check parity with the injection).
|
||
outputChecked = true
|
||
postMisses = r.baseMemory.postcheck(memSel.injected, seq.finalText)
|
||
if r.Pipeline.Gates.Glossary.PostcheckGate && countConfirmedMisses(postMisses) > 0 {
|
||
seq.flagged = true
|
||
seq.flagReason = FlagGlossaryMiss
|
||
r.Log.WarnContext(ctx, "glossary post-check gate flagged the chunk",
|
||
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "confirmed_misses", countConfirmedMisses(postMisses))
|
||
}
|
||
}
|
||
if isFinalWave && !seq.flagged && seq.finalText != "" && isRuTarget(r.Book.TargetLang) {
|
||
cheap = runCheapGates(ch.Text, seq.finalText, seq.finalText, r.cheapGateConfig())
|
||
}
|
||
if r.baseMemory != nil {
|
||
if err := r.persistRetrievalState(draftSnapshot, ch, memSel, postMisses, outputChecked, cheap, seq.bankTelem); err != nil {
|
||
return seq, err
|
||
}
|
||
}
|
||
return seq, nil
|
||
}
|
||
|
||
// runEditUnit runs the editor stage(s) over one edit unit under the edit-wave snapshot (plan §1/§2). It reads
|
||
// the unit's draft as the concatenation of its member chunks' draft outputs (NEVER re-rendering the draft
|
||
// stage — the несущий invariant), does a FRESH memory.Select over the WHOLE-unit source over the ENRICHED
|
||
// bank (sticky degenerate at unit scope → nil), and runs the editor over (unit source, unit draft). A unit whose
|
||
// ANY member draft flagged is flagged (skip edit — no paid edit over a partial draft, D2 flag+skip); the
|
||
// good members' draft-wave spend stays honestly committed. Post-check + cheap gates run over the edit output at
|
||
// unit granularity, merged into the LEADER chunk's retrieval_state row.
|
||
func (r *Runner) runEditUnit(ctx context.Context, editSnapshot string, unit editUnit, draftByKey map[chunkKey]stageSeqResult, editStages []wavedStage) (*ChunkOutcome, error) {
|
||
leader := Chunk{Chapter: unit.Chapter, ChunkIdx: unit.FirstChunkIdx, Text: unit.sourceText()}
|
||
out := &ChunkOutcome{Chapter: unit.Chapter, ChunkIdx: unit.FirstChunkIdx, Disposition: DispOK}
|
||
|
||
var draftStages []StageResult
|
||
var draftCost float64
|
||
var draftParts []string
|
||
memberFlagged := false
|
||
var memberFlagReason FlagReason
|
||
memberRecovered := ""
|
||
for _, m := range unit.Members {
|
||
d, ok := draftByKey[chunkKey{m.Chapter, m.ChunkIdx}]
|
||
if !ok {
|
||
return nil, fmt.Errorf("pipeline: edit unit ch%d chunk%d: missing draft result (wave sequencing bug)", m.Chapter, m.ChunkIdx)
|
||
}
|
||
draftStages = append(draftStages, d.stages...)
|
||
draftCost += d.cost
|
||
if d.flagged {
|
||
if !memberFlagged {
|
||
memberFlagged = true
|
||
memberFlagReason = d.flagReason
|
||
memberRecovered = d.recovered
|
||
}
|
||
continue
|
||
}
|
||
draftParts = append(draftParts, d.finalText)
|
||
}
|
||
out.CostUSD = draftCost
|
||
|
||
if memberFlagged {
|
||
// A member draft flagged → the unit's edit is skipped (no paid edit over a partial draft). Record a
|
||
// skipped edit chunk_status at the leader and export the recovered draft (or "" for a dropped flag).
|
||
skipped, err := r.recordSkippedStages(ctx, editStages, editSnapshot, leader, memberFlagReason)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out.Stages = append(draftStages, skipped...)
|
||
out.Disposition = DispFlagged
|
||
out.FlagReason = memberFlagReason
|
||
out.FinalText = exportNormalize(memberRecovered)
|
||
return out, nil
|
||
}
|
||
|
||
unitDraft := strings.Join(draftParts, unitJoinSeparator)
|
||
var editSel memorySelection
|
||
injectionByRole := map[string]string{}
|
||
if r.memory != nil {
|
||
// FRESH Select over the WHOLE-unit source over the ENRICHED bank (§1(б)/F3): NOT the the precompute pass per-chunk
|
||
// memSel (that was over the base bank). Sticky is degenerate at unit scope (a unit == a chapter or a
|
||
// sub-chapter split, so the intra-chapter sticky window does not apply) → nil sticky_prev.
|
||
editSel = r.memory.Select(leader.Text, unit.Chapter, nil, r.Pipeline.Context.GlossaryTokenBudget)
|
||
// Render the per-role injection from the ENRICHED unit selection via the registry (D39 слой 7): the
|
||
// editor gets its CONFIRMED-dst constraint block; other roles' blocks are rendered but consumed
|
||
// only if a stage of that role runs in the edit wave.
|
||
for role, render := range roleInjectionRenderers {
|
||
injectionByRole[role] = render(editSel.injected)
|
||
}
|
||
}
|
||
seq, err := r.runStageSequence(ctx, editStages, editSnapshot, leader, unitDraft, injectionByRole)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out.Stages = append(draftStages, seq.stages...)
|
||
out.CostUSD += seq.cost
|
||
|
||
finalText, flagged, flagReason, recovered := seq.finalText, seq.flagged, seq.flagReason, seq.recovered
|
||
var postMisses []postcheckMiss
|
||
outputChecked := false
|
||
if r.memory != nil && !flagged && finalText != "" {
|
||
outputChecked = true
|
||
postMisses = r.memory.postcheck(editSel.injected, finalText)
|
||
if r.Pipeline.Gates.Glossary.PostcheckGate && countConfirmedMisses(postMisses) > 0 {
|
||
flagged = true
|
||
flagReason = FlagGlossaryMiss
|
||
r.Log.WarnContext(ctx, "glossary post-check gate flagged the edit unit",
|
||
"chapter", unit.Chapter, "leader_chunk", unit.FirstChunkIdx, "confirmed_misses", countConfirmedMisses(postMisses))
|
||
}
|
||
}
|
||
var cheap cheapGateResult
|
||
if !flagged && finalText != "" && isRuTarget(r.Book.TargetLang) {
|
||
cheap = runCheapGates(leader.Text, unitDraft, finalText, r.cheapGateConfig())
|
||
if cheap.total() > 0 {
|
||
r.Log.InfoContext(ctx, "cheap style gates flagged the edit unit (observability, not a gate)",
|
||
"chapter", unit.Chapter, "leader_chunk", unit.FirstChunkIdx, "style_flags", cheap.total())
|
||
}
|
||
}
|
||
if r.memory != nil {
|
||
if err := r.mergeUnitRetrievalState(unit.Chapter, unit.FirstChunkIdx, postMisses, outputChecked, cheap); err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
|
||
if flagged {
|
||
out.Disposition = DispFlagged
|
||
out.FlagReason = flagReason
|
||
out.FinalText = exportNormalize(recovered)
|
||
} else {
|
||
out.FinalText = exportNormalize(finalText)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// recordSkippedStages writes a `skipped` chunk_status row (+ a skipped StageResult) for each stage in a
|
||
// wave, at chunk `ch` under `snapID` — used when a unit's member draft flagged, so the unit's edit is
|
||
// skipped (mirrors runStageSequence's flagged-skip path but for a pre-decided skip).
|
||
func (r *Runner) recordSkippedStages(ctx context.Context, staged []wavedStage, snapID string, ch Chunk, flagReason FlagReason) ([]StageResult, error) {
|
||
detail := fmt.Sprintf("skipped: a member draft chunk of this edit unit was flagged (%s)", flagReason)
|
||
var out []StageResult
|
||
for _, ws := range staged {
|
||
st := ws.st
|
||
if err := r.Store.UpsertChunkStatus(store.ChunkStatus{
|
||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stage: st.Name,
|
||
SnapshotID: snapID, Disposition: string(DispSkipped), FlagReason: string(flagReason), Detail: detail,
|
||
}); err != nil {
|
||
return nil, fmt.Errorf("pipeline: record skipped edit chunk_status: %w", err)
|
||
}
|
||
out = append(out, StageResult{
|
||
Stage: st.Name, Role: st.Role, Model: st.Model,
|
||
Disposition: DispSkipped, FlagReason: flagReason, Detail: detail,
|
||
})
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// mergeUnitRetrievalState folds a edit unit's post-check + cheap-gate observability into the LEADER chunk's
|
||
// retrieval_state row — the row the draft wave already wrote with the leader chunk's draft-injection counts + banknote
|
||
// telemetry. A read-modify-write (single-writer safe; each unit owns a distinct leader chunk_idx) preserves
|
||
// the draft fields and adds the unit-level post-check/style fields, so the read-models aggregate injection
|
||
// per draft chunk and post-check/style per unit. A missing row (r.memory nil path) is a no-op.
|
||
func (r *Runner) mergeUnitRetrievalState(chapter, firstChunkIdx int, misses []postcheckMiss, outputChecked bool, cheap cheapGateResult) error {
|
||
rs, err := r.Store.GetRetrievalState(r.Book.BookID, chapter, firstChunkIdx)
|
||
if err != nil {
|
||
return fmt.Errorf("pipeline: read leader retrieval_state ch%d/chunk%d: %w", chapter, firstChunkIdx, err)
|
||
}
|
||
if rs == nil {
|
||
return nil // no draft row (no memory / no draft) → nothing to merge onto
|
||
}
|
||
rs.NStyleFlags = cheap.total()
|
||
rs.StyleDetail = ""
|
||
if cheap.total() > 0 {
|
||
if b, err := json.Marshal(cheap); err == nil {
|
||
rs.StyleDetail = string(b)
|
||
}
|
||
}
|
||
rs.NPostcheckMiss = 0
|
||
rs.PostcheckDetail = ""
|
||
if outputChecked {
|
||
rs.NPostcheckMiss = countConfirmedMisses(misses)
|
||
if len(misses) > 0 {
|
||
if b, err := json.Marshal(misses); err == nil {
|
||
rs.PostcheckDetail = string(b)
|
||
}
|
||
}
|
||
}
|
||
return r.Store.UpsertRetrievalState(*rs)
|
||
}
|
||
|
||
// draftOnlyOutcome assembles a ChunkOutcome for a draft-only pipeline (no editor): the draft is the shipping
|
||
// output, so the per-chunk draft result maps straight to the outcome (post-check/cheap already ran in
|
||
// runDraftChunk as the final wave). Mirrors the export contract: a cosmetic strip exports its recovered
|
||
// text; every other flag exports "".
|
||
func draftOnlyOutcome(ch Chunk, d stageSeqResult) ChunkOutcome {
|
||
oc := ChunkOutcome{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Stages: d.stages, CostUSD: d.cost, Disposition: DispOK}
|
||
if d.flagged {
|
||
oc.Disposition = DispFlagged
|
||
oc.FlagReason = d.flagReason
|
||
oc.FinalText = exportNormalize(d.recovered)
|
||
} else {
|
||
oc.FinalText = exportNormalize(d.finalText)
|
||
}
|
||
return oc
|
||
}
|