218 lines
11 KiB
Go
218 lines
11 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"textmachine/backend/internal/store"
|
|
)
|
|
|
|
// export.go: the read-only EXPORT projection (D39 слой 6 / D39.2-open №1; the deferred `tmctl export`
|
|
// of D15.2-этап-Б in its MINIMAL read form). It emits every chunk's FINAL export text EXACTLY by prod
|
|
// semantics, so the polygon extractor (records.json / exp12_extract) and reader-samples read the SAME
|
|
// export-normalised bytes `tmctl translate` ships — instead of the RAW stored checkpoint, which skips
|
|
// the export contract for a clean chunk. The owner chose a BACKEND surface over a Python mirror of
|
|
// exportNormalize (which would drift against the Go contract). It is a PURE READ-ONLY projection like
|
|
// Status/QualityReport: $0, no LLM, no keys, no snapshot touch — it reads the persisted chunk_status +
|
|
// the $0 final checkpoint and applies the very same exportNormalize the ChunkOutcome does. Only the READ
|
|
// half is built here; the annotation/override half of the D15.2 `export` command is deliberately NOT.
|
|
|
|
// exportPending marks a manifest chunk that has not (yet) reached the final stage — no exported text.
|
|
const exportPending = "pending"
|
|
|
|
// ChunkExport is one chunk's final export record: the metadata plus the export-normalised final text.
|
|
type ChunkExport struct {
|
|
Chapter int `json:"chapter"`
|
|
ChunkIdx int `json:"chunk_idx"`
|
|
SnapshotID string `json:"snapshot_id,omitempty"`
|
|
// Disposition is the CHUNK-level verdict mirroring ChunkOutcome: "ok" only when the final stage is
|
|
// ok; "flagged" for a flagged OR upstream-skipped final row; "pending" for a manifest chunk with no
|
|
// final-stage row (not yet translated).
|
|
Disposition string `json:"disposition"`
|
|
FlagReason string `json:"flag_reason,omitempty"`
|
|
Detail string `json:"detail,omitempty"`
|
|
// FinalText is the export-normalised final text EXACTLY as `tmctl translate` ships it:
|
|
// exportNormalize(final checkpoint response) for an ok or cosmetic-stripped chunk, "" for a
|
|
// substantive flag / upstream-skip / pending chunk (exportNormalize("")=="", so the paths coincide
|
|
// byte-for-byte with ChunkOutcome.FinalText).
|
|
FinalText string `json:"final_text"`
|
|
// Source is the chunk's normalized source text — populated ONLY under `tmctl export --pairs` (WS5
|
|
// §5(д) / research/20 §E-3), the src↔target column the polygon needs to measure the DC1 (时辰) / DC2
|
|
// (千万) source-vs-target checkers' false-positive rate. Omitted by default (the export contract is
|
|
// target-only); the source is the $0 manifest re-chunk, never a stored/billed artifact.
|
|
Source string `json:"source,omitempty"`
|
|
}
|
|
|
|
// BookExport is the whole-book export projection (chunks in manifest — (chapter, chunk_idx) — order, so
|
|
// the JSON is deterministic across runs). The counters make a partial/empty/drifted book EXPLICIT
|
|
// instead of silently exporting as complete (F4/F3).
|
|
type BookExport struct {
|
|
BookID string `json:"book_id"`
|
|
// TotalChunks is the current source manifest size (the honest denominator); ExportedChunks +
|
|
// PendingChunks == TotalChunks. GhostRows are stored final rows dropped as OUTSIDE the manifest.
|
|
TotalChunks int `json:"total_chunks"`
|
|
PendingChunks int `json:"pending_chunks"`
|
|
GhostRows int `json:"ghost_rows,omitempty"`
|
|
// ConfigDrift is true when the CURRENT config renders a snapshot different from the one the stored
|
|
// rows carry (a gate flip / prompt bump / stage rename since the run) — the gate/stage re-derivation
|
|
// below may then not match what translate actually did. CurrentSnapshot is that projected id.
|
|
ConfigDrift bool `json:"config_drift"`
|
|
CurrentSnapshot string `json:"current_snapshot,omitempty"`
|
|
// Chunks is always non-nil (an empty book exports [] not null).
|
|
Chunks []ChunkExport `json:"chunks"`
|
|
}
|
|
|
|
// Export builds the read-only per-chunk export projection. It opens no jobs, reserves nothing, makes no
|
|
// LLM call — it joins the $0 source manifest against the persisted chunk_status (final-stage rows) and,
|
|
// for a chunk that produced exported text, the $0 final checkpoint, applying exportNormalize exactly as
|
|
// translateChunk does. Safe to run whenever `report`/`status` are (the same exclusive-lock rule).
|
|
// Deterministic over the store + source.
|
|
//
|
|
// CAVEATS (surfaced, never silent):
|
|
// - CONFIG/GATE DRIFT (F3): the gate flag AND the final-stage NAME are read from the CURRENT config.
|
|
// A config edit since the run (a glossary post-check gate FLIP, a prompt bump, a final-stage rename)
|
|
// changes the projected snapshot; Export sets ConfigDrift + WARNs so the gate re-derivation / row
|
|
// lookup mismatch is visible (a run under the same config reads correctly; `status` shows it too).
|
|
// - MANIFEST (F4): a chunk absent from the current source is a GHOST row (dropped + counted + WARNed);
|
|
// a manifest chunk with no final row is PENDING; both keep a partial/edited book from exporting as
|
|
// if complete (the D37-skew the polygon extractor must not inherit).
|
|
func (r *Runner) Export(pairs bool) (*BookExport, error) {
|
|
statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// The glossary post-check GATE (opt-in) flips a chunk to flagged/glossary_miss at the CHUNK level
|
|
// AFTER the stage loop, so it is NEVER written to chunk_status — every stage row stays DispOK
|
|
// (chunkrun.translateChunk records only retrieval_state). Export must re-derive it, exactly as
|
|
// Status does: a raw chunk_status read would report a gate-flagged chunk as clean ok AND ship the
|
|
// contaminated text `tmctl translate` deliberately withheld. Read retrieval_state only when the gate
|
|
// is on (else the map stays empty and the branch is a no-op).
|
|
gateOn := r.Pipeline.Gates.Glossary.PostcheckGate
|
|
missByChunk := map[chunkKey]int{}
|
|
if gateOn {
|
|
states, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, rs := range states {
|
|
missByChunk[chunkKey{rs.Chapter, rs.ChunkIdx}] = rs.NPostcheckMiss // CONFIRMED-miss count
|
|
}
|
|
}
|
|
lastStage := ""
|
|
if n := len(r.Pipeline.Stages); n > 0 {
|
|
lastStage = r.Pipeline.Stages[n-1].Name
|
|
}
|
|
// Index the final-stage rows by chunk key (the exported text + verdict live there) and record the
|
|
// snapshot ids the rows carry (for the drift check).
|
|
finalRow := map[chunkKey]store.ChunkStatus{}
|
|
snapSeen := map[string]bool{}
|
|
for _, cs := range statuses {
|
|
if cs.Stage != lastStage {
|
|
continue // the exported text and the chunk's final verdict live on the final stage's row
|
|
}
|
|
finalRow[chunkKey{cs.Chapter, cs.ChunkIdx}] = cs
|
|
if cs.SnapshotID != "" {
|
|
snapSeen[cs.SnapshotID] = true
|
|
}
|
|
}
|
|
|
|
// F4: the $0 source manifest is the authoritative chunk set — join it so a PENDING chunk is explicit
|
|
// and a GHOST (deleted-source) row is not exported as if live.
|
|
manifest, err := r.bookChunks()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
exp := &BookExport{BookID: r.Book.BookID, Chunks: []ChunkExport{}}
|
|
inManifest := map[chunkKey]bool{}
|
|
for _, ch := range manifest {
|
|
k := chunkKey{ch.Chapter, ch.ChunkIdx}
|
|
inManifest[k] = true
|
|
exp.TotalChunks++
|
|
cs, ok := finalRow[k]
|
|
if !ok {
|
|
pend := ChunkExport{Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, Disposition: exportPending}
|
|
if pairs {
|
|
pend.Source = ch.Text
|
|
}
|
|
exp.PendingChunks++
|
|
exp.Chunks = append(exp.Chunks, pend)
|
|
continue
|
|
}
|
|
ce, err := r.chunkExport(cs, gateOn, missByChunk[k])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if pairs {
|
|
ce.Source = ch.Text // the src↔target column for the DC1/DC2 FP-measure (--pairs)
|
|
}
|
|
exp.Chunks = append(exp.Chunks, ce)
|
|
}
|
|
// F4: GHOST rows — a stored final row whose chunk is NOT in the current manifest (source shrunk since
|
|
// the run). Dropped (never exported as if live) but counted + WARNed so the drop is loud.
|
|
for k := range finalRow {
|
|
if !inManifest[k] {
|
|
exp.GhostRows++
|
|
}
|
|
}
|
|
if exp.GhostRows > 0 {
|
|
r.Log.Warn("export: dropped chunk_status rows outside the current source manifest (source shrunk since the run?)",
|
|
"book", r.Book.BookID, "ghost_rows", exp.GhostRows)
|
|
}
|
|
|
|
// F3: config/gate drift — only meaningful on a single stored snapshot (multi-snapshot drift is
|
|
// already visible in the per-chunk snapshot_id). Compute the CURRENT config's projected snapshot
|
|
// read-only and compare; surface a mismatch as a flag + WARN, never silently.
|
|
if len(snapSeen) == 1 {
|
|
var stored string
|
|
for s := range snapSeen {
|
|
stored = s
|
|
}
|
|
if cur, serr := r.currentSnapshotProjected(); serr != nil {
|
|
r.Log.Warn("export: config-drift check failed; drift state unknown (reported as none)", "err", serr)
|
|
} else if cur != stored {
|
|
exp.ConfigDrift = true
|
|
exp.CurrentSnapshot = cur
|
|
r.Log.Warn("export: CONFIG-DRIFT — current config renders a different snapshot than the stored rows; the gate/stage re-derivation may not match the run (translate would require --resnapshot)",
|
|
"book", r.Book.BookID, "stored", stored, "current", cur)
|
|
}
|
|
}
|
|
return exp, nil
|
|
}
|
|
|
|
// chunkExport computes one chunk's export record from its final-stage row, exactly by prod FinalText
|
|
// semantics (chunkrun.translateChunk): an ok chunk exports exportNormalize(primary checkpoint), a
|
|
// cosmetic sanitizer-strip exports exportNormalize(its derived stripped checkpoint) — both via
|
|
// final_hash — the glossary gate override exports "", and every other flag / upstream skip exports "".
|
|
func (r *Runner) chunkExport(cs store.ChunkStatus, gateOn bool, gateMiss int) (ChunkExport, error) {
|
|
ce := ChunkExport{Chapter: cs.Chapter, ChunkIdx: cs.ChunkIdx, SnapshotID: cs.SnapshotID, Detail: cs.Detail}
|
|
// Post-check gate override (mirrors translateChunk): an otherwise-ok chunk (DispOK ⇒ every stage ok)
|
|
// with a CONFIRMED miss is flagged glossary_miss and exports NOTHING — the withheld text never ships.
|
|
if gateOn && cs.Disposition == string(DispOK) && gateMiss > 0 {
|
|
ce.Disposition = string(DispFlagged)
|
|
ce.FlagReason = string(FlagGlossaryMiss)
|
|
return ce, nil
|
|
}
|
|
if cs.Disposition == string(DispOK) {
|
|
ce.Disposition = string(DispOK)
|
|
if cs.FinalHash == "" {
|
|
// F9: an ok chunk MUST carry a final_hash — an empty one is an inconsistent store. Fail LOUD
|
|
// (parity with the missing-checkpoint case), never silently export "".
|
|
return ce, fmt.Errorf("pipeline: export ch%d/chunk%d: DispOK row has an empty final_hash (inconsistent store)", cs.Chapter, cs.ChunkIdx)
|
|
}
|
|
} else {
|
|
// A flagged final row OR a skipped one (an upstream stage flagged) is a flagged CHUNK; the final
|
|
// row carries the propagated flag reason (chunkrun records it on the skipped row too).
|
|
ce.Disposition = string(DispFlagged)
|
|
ce.FlagReason = cs.FlagReason
|
|
}
|
|
if cs.FinalHash != "" && (cs.Disposition == string(DispOK) || cs.FlagReason == string(FlagSanitizerStripped)) {
|
|
cp, err := r.Store.GetCheckpoint(cs.FinalHash)
|
|
if err != nil {
|
|
return ce, err
|
|
}
|
|
if cp == nil {
|
|
return ce, fmt.Errorf("pipeline: export ch%d/chunk%d: chunk_status.final_hash %.12s has no checkpoint (inconsistent store)", cs.Chapter, cs.ChunkIdx, cs.FinalHash)
|
|
}
|
|
ce.FinalText = exportNormalize(cp.ResponseText)
|
|
}
|
|
return ce, nil
|
|
}
|