package pipeline import ( "fmt" "textmachine/backend/internal/store" ) // export.go: the read-only EXPORT projection (D39 слой 6 / D39.2-open №1; the deferred `tmctl export` // of D15.2-этап-Б in its MINIMAL read form). It emits every chunk's FINAL export text EXACTLY by prod // semantics, so the polygon extractor (records.json / exp12_extract) and reader-samples read the SAME // export-normalised bytes `tmctl translate` ships — instead of the RAW stored checkpoint, which skips // the export contract for a clean chunk. The owner chose a BACKEND surface over a Python mirror of // exportNormalize (which would drift against the Go contract). It is a PURE READ-ONLY projection like // Status/QualityReport: $0, no LLM, no keys, no snapshot touch — it reads the persisted chunk_status + // the $0 final checkpoint and applies the very same exportNormalize the ChunkOutcome does. Only the READ // half is built here; the annotation/override half of the D15.2 `export` command is deliberately NOT. // exportPending marks a manifest chunk that has not (yet) reached the final stage — no exported text. const exportPending = "pending" // ChunkExport is one chunk's final export record: the metadata plus the export-normalised final text. type ChunkExport struct { Chapter int `json:"chapter"` ChunkIdx int `json:"chunk_idx"` SnapshotID string `json:"snapshot_id,omitempty"` // Disposition is the CHUNK-level verdict mirroring ChunkOutcome: "ok" only when the final stage is // ok; "flagged" for a flagged OR upstream-skipped final row; "pending" for a manifest chunk with no // final-stage row (not yet translated). Disposition string `json:"disposition"` FlagReason string `json:"flag_reason,omitempty"` Detail string `json:"detail,omitempty"` // FinalText is the export-normalised final text EXACTLY as `tmctl translate` ships it: // exportNormalize(final checkpoint response) for an ok or cosmetic-stripped chunk, "" for a // substantive flag / upstream-skip / pending chunk (exportNormalize("")=="", so the paths coincide // byte-for-byte with ChunkOutcome.FinalText). FinalText string `json:"final_text"` } // BookExport is the whole-book export projection (chunks in manifest — (chapter, chunk_idx) — order, so // the JSON is deterministic across runs). The counters make a partial/empty/drifted book EXPLICIT // instead of silently exporting as complete (F4/F3). type BookExport struct { BookID string `json:"book_id"` // TotalChunks is the current source manifest size (the honest denominator); ExportedChunks + // PendingChunks == TotalChunks. GhostRows are stored final rows dropped as OUTSIDE the manifest. TotalChunks int `json:"total_chunks"` PendingChunks int `json:"pending_chunks"` GhostRows int `json:"ghost_rows,omitempty"` // ConfigDrift is true when the CURRENT config renders a snapshot different from the one the stored // rows carry (a gate flip / prompt bump / stage rename since the run) — the gate/stage re-derivation // below may then not match what translate actually did. CurrentSnapshot is that projected id. ConfigDrift bool `json:"config_drift"` CurrentSnapshot string `json:"current_snapshot,omitempty"` // Chunks is always non-nil (an empty book exports [] not null). Chunks []ChunkExport `json:"chunks"` } // Export builds the read-only per-chunk export projection. It opens no jobs, reserves nothing, makes no // LLM call — it joins the $0 source manifest against the persisted chunk_status (final-stage rows) and, // for a chunk that produced exported text, the $0 final checkpoint, applying exportNormalize exactly as // translateChunk does. Safe to run whenever `report`/`status` are (the same exclusive-lock rule). // Deterministic over the store + source. // // CAVEATS (surfaced, never silent): // - CONFIG/GATE DRIFT (F3): the gate flag AND the final-stage NAME are read from the CURRENT config. // A config edit since the run (a glossary post-check gate FLIP, a prompt bump, a final-stage rename) // changes the projected snapshot; Export sets ConfigDrift + WARNs so the gate re-derivation / row // lookup mismatch is visible (a run under the same config reads correctly; `status` shows it too). // - MANIFEST (F4): a chunk absent from the current source is a GHOST row (dropped + counted + WARNed); // a manifest chunk with no final row is PENDING; both keep a partial/edited book from exporting as // if complete (the D37-skew the polygon extractor must not inherit). func (r *Runner) Export() (*BookExport, error) { statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID) if err != nil { return nil, err } // The glossary post-check GATE (opt-in) flips a chunk to flagged/glossary_miss at the CHUNK level // AFTER the stage loop, so it is NEVER written to chunk_status — every stage row stays DispOK // (chunkrun.translateChunk records only retrieval_state). Export must re-derive it, exactly as // Status does: a raw chunk_status read would report a gate-flagged chunk as clean ok AND ship the // contaminated text `tmctl translate` deliberately withheld. Read retrieval_state only when the gate // is on (else the map stays empty and the branch is a no-op). gateOn := r.Pipeline.Gates.Glossary.PostcheckGate missByChunk := map[chunkKey]int{} if gateOn { states, err := r.Store.RetrievalStatesForBook(r.Book.BookID) if err != nil { return nil, err } for _, rs := range states { missByChunk[chunkKey{rs.Chapter, rs.ChunkIdx}] = rs.NPostcheckMiss // CONFIRMED-miss count } } lastStage := "" if n := len(r.Pipeline.Stages); n > 0 { lastStage = r.Pipeline.Stages[n-1].Name } // Index the final-stage rows by chunk key (the exported text + verdict live there) and record the // snapshot ids the rows carry (for the drift check). finalRow := map[chunkKey]store.ChunkStatus{} snapSeen := map[string]bool{} for _, cs := range statuses { if cs.Stage != lastStage { continue // the exported text and the chunk's final verdict live on the final stage's row } finalRow[chunkKey{cs.Chapter, cs.ChunkIdx}] = cs if cs.SnapshotID != "" { snapSeen[cs.SnapshotID] = true } } // F4: the $0 source manifest is the authoritative chunk set — join it so a PENDING chunk is explicit // and a GHOST (deleted-source) row is not exported as if live. manifest, err := r.bookChunks() if err != nil { return nil, err } exp := &BookExport{BookID: r.Book.BookID, Chunks: []ChunkExport{}} inManifest := map[chunkKey]bool{} for _, ch := range manifest { k := chunkKey{ch.Chapter, ch.ChunkIdx} inManifest[k] = true exp.TotalChunks++ cs, ok := finalRow[k] if !ok { exp.PendingChunks++ exp.Chunks = append(exp.Chunks, ChunkExport{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: exportPending}) continue } ce, err := r.chunkExport(cs, gateOn, missByChunk[k]) if err != nil { return nil, err } exp.Chunks = append(exp.Chunks, ce) } // F4: GHOST rows — a stored final row whose chunk is NOT in the current manifest (source shrunk since // the run). Dropped (never exported as if live) but counted + WARNed so the drop is loud. for k := range finalRow { if !inManifest[k] { exp.GhostRows++ } } if exp.GhostRows > 0 { r.Log.Warn("export: dropped chunk_status rows outside the current source manifest (source shrunk since the run?)", "book", r.Book.BookID, "ghost_rows", exp.GhostRows) } // F3: config/gate drift — only meaningful on a single stored snapshot (multi-snapshot drift is // already visible in the per-chunk snapshot_id). Compute the CURRENT config's projected snapshot // read-only and compare; surface a mismatch as a flag + WARN, never silently. if len(snapSeen) == 1 { var stored string for s := range snapSeen { stored = s } if cur, serr := r.currentSnapshotProjected(); serr != nil { r.Log.Warn("export: config-drift check failed; drift state unknown (reported as none)", "err", serr) } else if cur != stored { exp.ConfigDrift = true exp.CurrentSnapshot = cur r.Log.Warn("export: CONFIG-DRIFT — current config renders a different snapshot than the stored rows; the gate/stage re-derivation may not match the run (translate would require --resnapshot)", "book", r.Book.BookID, "stored", stored, "current", cur) } } return exp, nil } // chunkExport computes one chunk's export record from its final-stage row, exactly by prod FinalText // semantics (chunkrun.translateChunk): an ok chunk exports exportNormalize(primary checkpoint), a // cosmetic sanitizer-strip exports exportNormalize(its derived stripped checkpoint) — both via // final_hash — the glossary gate override exports "", and every other flag / upstream skip exports "". func (r *Runner) chunkExport(cs store.ChunkStatus, gateOn bool, gateMiss int) (ChunkExport, error) { ce := ChunkExport{Chapter: cs.Chapter, ChunkIdx: cs.ChunkIdx, SnapshotID: cs.SnapshotID, Detail: cs.Detail} // Post-check gate override (mirrors translateChunk): an otherwise-ok chunk (DispOK ⇒ every stage ok) // with a CONFIRMED miss is flagged glossary_miss and exports NOTHING — the withheld text never ships. if gateOn && cs.Disposition == string(DispOK) && gateMiss > 0 { ce.Disposition = string(DispFlagged) ce.FlagReason = string(FlagGlossaryMiss) return ce, nil } if cs.Disposition == string(DispOK) { ce.Disposition = string(DispOK) if cs.FinalHash == "" { // F9: an ok chunk MUST carry a final_hash — an empty one is an inconsistent store. Fail LOUD // (parity with the missing-checkpoint case), never silently export "". return ce, fmt.Errorf("pipeline: export ch%d/chunk%d: DispOK row has an empty final_hash (inconsistent store)", cs.Chapter, cs.ChunkIdx) } } else { // A flagged final row OR a skipped one (an upstream stage flagged) is a flagged CHUNK; the final // row carries the propagated flag reason (chunkrun records it on the skipped row too). ce.Disposition = string(DispFlagged) ce.FlagReason = cs.FlagReason } if cs.FinalHash != "" && (cs.Disposition == string(DispOK) || cs.FlagReason == string(FlagSanitizerStripped)) { cp, err := r.Store.GetCheckpoint(cs.FinalHash) if err != nil { return ce, err } if cp == nil { return ce, fmt.Errorf("pipeline: export ch%d/chunk%d: chunk_status.final_hash %.12s has no checkpoint (inconsistent store)", cs.Chapter, cs.ChunkIdx, cs.FinalHash) } ce.FinalText = exportNormalize(cp.ResponseText) } return ce, nil }