package pipeline import ( "context" "fmt" "textmachine/backend/internal/llm" ) // bookrun.go: цикл уровня КНИГИ (Веха 2) — ingest → сид/материализация памяти → // снапшот → последовательный обход чанков с D2-дисциплиной (флаг не роняет прогон) // и sticky-окном сцены (A5). Плюс типы результатов прогона и exit-контракт CLI. // StageResult reports one executed stage of a chunk. type StageResult struct { Stage string Role string Model string // фактически ответившая модель FromResume bool // no provider call was made THIS run (fully served from checkpoints) Usage llm.Usage CostUSD float64 // THIS run's spend (0 when fully served from checkpoints) CumCostUSD float64 // sum across ALL attempts of this chunk×stage (F3-honest, incl. retries) LatencyMS int FinishReason string Text string // the usable output (only on an ok disposition) // RecoveredText is the cosmetic-stripped export text (D35.4a): non-empty ONLY for a // FlagSanitizerStripped final stage, where the sanitizer removed a leading markdown header / // CJK-leak and committed the cleaned remainder. It carries the export forward on BOTH the // fresh path (runStage) and resume (resumeFromChunkStatus reads the derived checkpoint), so // translateChunk assembles the identical FinalText either way. "" for every other disposition. RecoveredText string Disposition Disposition FlagReason FlagReason // "" when ok Detail string Attempts int // Escalated — a single-hop fallback draft was tried this stage (D12); when the // fallback passed the re-gate, Model above is the fallback (it answered). Escalated bool EscalationModel string // the fallback model when Escalated ("" otherwise) // BankFlags carries the translator draft's banknote telemetry (WS4 point 10): accepted line count + // parse-fail / truncation flags. Zero-valued on every non-translator stage and every channel-off run. BankFlags bankFlags } // ChunkOutcome is one chunk's result across the stage list. type ChunkOutcome struct { Chapter int ChunkIdx int Stages []StageResult // FinalText is the exported text: the last stage's output on ok; the cosmetic-stripped // remainder on a FlagSanitizerStripped flag (D35.4a — the chunk is exported flagged, not lost // to an empty placeholder); "" on any other flag (the contaminated output never exports). FinalText string Disposition Disposition // ok | flagged (a chunk has no "skipped" — that is a per-later-stage state) FlagReason FlagReason // the flagging stage's reason ("" when ok) CostUSD float64 // THIS run's spend on this chunk } // BookResult aggregates a whole run over every chapter×chunk. type BookResult struct { BookID string Chunks []ChunkOutcome TotalUSD float64 // THIS run's spend across all chunks Flagged int // number of flagged chunks (acceptance допускает N) } // ExitCode is the run's shell disposition: 0 clean, 2 completed-with-flags // (acceptance допускает N флагов — приёмка это разрешает). An infra failure is // an error from TranslateBook, which the CLI maps to 1. func (b *BookResult) ExitCode() int { if b.Flagged > 0 { return 2 } return 0 } // CompletedWithFlags is the typed sentinel the CLI maps to exit code 2: the book // finished end-to-end but N chunks were flagged for a human. It is NOT an infra // failure (a plain error → exit 1); it is a "clean run, attention needed" signal // carried up through `func run() error` in the idiomatic Go way. type CompletedWithFlags struct { Flagged int Total int } func (e *CompletedWithFlags) Error() string { return fmt.Sprintf("completed with %d/%d chunk(s) flagged for review", e.Flagged, e.Total) } // TranslateBook runs the whole book: split the normalized source into // chapter/chunk units (chunker.go) and drive each chunk through the stage list. // A bad chunk is flagged and the loop CONTINUES (D2); only an infra failure // (ceiling, config change without --resnapshot, unbilled call failure, store // error) aborts with an error, from which resume continues. func (r *Runner) TranslateBook(ctx context.Context) (*BookResult, error) { // Ingest reads + normalizes the source (txt/epub) into ordered per-chapter text // and captures ruby readings (ingest.go), decoding the txt source per book.encoding // (auto/utf8/gb18030). Offline and deterministic ($0, no LLM). doc, err := IngestEncoded(r.Book.SourceFile, r.Book.Encoding, r.Book.SourceLang) if err != nil { return nil, err } // the precompute pass: eager-build every reachable client BEFORE any wave goroutine, so the clients map is // read-only in the waves (r.client is lock-free, a miss is loud — closes the D12 lazy-init race). if err := r.buildClients(); err != nil { return nil, err } // the precompute pass: build the per-model rate-guards (transport axis, read-only in the waves; a no-op until a // model configures rate_limit — WS1 §1б, the mistral-arm precondition WS6). r.buildRateGuards() // Persist captured ruby readings (idempotent; consumed by seedGlossary below into // auto glossary candidates — шаг 4). Never injected into a prompt here (§7d); a // resume re-persists the same rows at $0. if err := r.persistRuby(doc.Ruby); err != nil { return nil, fmt.Errorf("pipeline: persist ruby readings: %w", err) } // Seed + materialize the glossary BEFORE the snapshot: the frozen approved rows feed // memoryVersion() → the snapshot (F1), so the snapshot must be computed with the bank // already materialized. Deterministic and $0 (seed file + classified ruby, no LLM). if err := r.seedGlossary(ctx); err != nil { return nil, err } chunks := SplitChunks(doc.Chapters, r.segBudget()) if len(chunks) == 0 { return nil, fmt.Errorf("pipeline: source file %s produced no chunks after normalization", r.Book.SourceFile) } // Масштаб прогона — одной строкой на stderr (боль smoke-прогона: N/M и знаменатель // прогресса не появлялись в логах вообще, только в stdout/status). r.Log.InfoContext(ctx, "book run started", "book", r.Book.BookID, "chapters", len(doc.Chapters), "chunks", len(chunks), "stages", len(r.Pipeline.Stages)) // the precompute pass: pre-compute the sticky-chain memory selection for EVERY chunk (WS1 §1б, precomputeSticky). The // sticky window (A5 scene-inertia — the recent chunks' exact matches in the current chapter, reset at // a chapter boundary) is a CROSS-CHUNK sequential dependency, so it is computed once here in the precompute pass and // consumed by the parallel draft-wave workers (it cannot be recomputed inside a wave). Byte-identical to the // retired inline computation (Select is $0 and pure — the golden test proves the injected bytes hold). // The draft wave selects over the BASE bank (mined-excluded) so its injection is byte-identical across // a bank-mining enrichment (the review-confirmed «переоплата ОДНА» fix); the editor uses the enriched bank. stickySel := precomputeSticky(chunks, r.baseMemory, r.Pipeline.Context.GlossaryTokenBudget) // The wave executor (R1, waverun.go): the draft wave (draft ∥) → the bank-mining stop → the edit wave (edit ∥). It computes + // upserts the per-wave snapshots (draft-wave snapshot base-bank / edit-wave snapshot enriched) itself and pins each // wave's jobs to its own — a the bank-mining stop enrichment moves only edit-wave snapshot, keeping draft-wave checkpoints valid // («переоплата ОДНА»). Returns a *WaveSignatureStop when the bank-mining stop stops for owner sign. return r.translateBookWaves(ctx, chunks, stickySel) } // unionSticky merges the recent chunks' exact-matched ids into one sticky_prev, carrying // each id's DISPOSITION (D16.2). When an id fired in several recent chunks, the MOST RECENT // firing wins (win is ordered oldest→newest), so a re-established CONFIRMED match overrides // an older collision-prone AMBIGUOUS one — and vice-versa, never silently upgrading. func unionSticky(win []map[string]injectionDisposition) map[string]injectionDisposition { if len(win) == 0 { return nil } out := map[string]injectionDisposition{} for _, s := range win { for id, disp := range s { out[id] = disp } } return out }