112 lines
4.3 KiB
Go
112 lines
4.3 KiB
Go
package ingest
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
)
|
|
|
|
// export.go: the ALLOWLISTED reading of `tmctl export --json --pairs`.
|
|
//
|
|
// The ONE channel that carries a pair's TEXT: the manifest has structure without text, frames never
|
|
// carry text, and the engine's database is not readable from here (D39.85).
|
|
//
|
|
// $0 but slow in CPU — `--pairs` demands the full re-chunk — so it runs at the BOUNDARIES of the
|
|
// work rather than per request, which is also the freshness the contract declares for `Unit.target`.
|
|
//
|
|
// What is deliberately NOT read is most of the record: `disposition` and `flag_reason` are the
|
|
// engine's vocabulary, `detail` reads like pipeline internals ("CJK leak in the ru output: 第一节"),
|
|
// and `snapshot_id`, `config_drift` and the ghost counters are operator facts. The two that ARE read
|
|
// are read as a DERIVATION and not as a value — see UnitText.State.
|
|
|
|
// exportDisposition is the engine's per-unit verdict, and it stays inside this file: what leaves is
|
|
// the contract's own three-valued state.
|
|
const (
|
|
exportOK = "ok"
|
|
exportFlagged = "flagged"
|
|
)
|
|
|
|
// Export is one book's export projection, keyed the way the rest of the seam is keyed.
|
|
type Export struct {
|
|
// TotalUnits is the engine's honest denominator — the current manifest size in output units.
|
|
TotalUnits int
|
|
// Units is one record per output unit, addressed by (chapter ordinal, leader chunk index).
|
|
Units []UnitText
|
|
}
|
|
|
|
// UnitText is one pair as the reader sees it.
|
|
type UnitText struct {
|
|
// Chapter is the engine's dense chapter ordinal, Unit the LEADER chunk index — together the join
|
|
// key the manifest publishes as `first_chunk_idx`.
|
|
Chapter int
|
|
Unit int
|
|
Source string
|
|
Target string
|
|
// State is the contract's UnitState, DERIVED here rather than mapped one-to-one: a flagged unit
|
|
// legally ships with text (a cosmetic sanitizer strip, a c-lite member drop), and the contract
|
|
// derives the state from the PAIR — text present → `translated` (with a note attached), flagged
|
|
// and empty → `withheld`, nothing yet → `pending` (canon §UnitState, companion §2.7).
|
|
State string
|
|
}
|
|
|
|
// The contract's UnitState values.
|
|
const (
|
|
StateTranslated = "translated"
|
|
StateWithheld = "withheld"
|
|
StatePending = "pending"
|
|
)
|
|
|
|
// wireExport is the document as the engine prints it.
|
|
type wireExport struct {
|
|
BookID string `json:"book_id"`
|
|
TotalUnits int `json:"total_units"`
|
|
Chunks []struct {
|
|
Chapter int `json:"chapter"`
|
|
ChunkIdx int `json:"chunk_idx"`
|
|
Disposition string `json:"disposition"`
|
|
FinalText string `json:"final_text"`
|
|
Source string `json:"source"`
|
|
} `json:"chunks"`
|
|
}
|
|
|
|
// DecodeExport parses an export document, and refuses anything that does not identify itself as
|
|
// one.
|
|
//
|
|
// The identity check is the same guard DecodeManifest carries and for a weaker but real version of
|
|
// the same reason: `{}`, `null` and any object of fields this build has never heard of all decode
|
|
// into an Export of zeros, and a materializer would then replace a book's whole text with nothing.
|
|
// A document with no `book_id` is not an export.
|
|
func DecodeExport(b []byte) (Export, error) {
|
|
var doc wireExport
|
|
if err := json.Unmarshal(b, &doc); err != nil {
|
|
return Export{}, fmt.Errorf("ingest: decode export: %w", err)
|
|
}
|
|
if doc.BookID == "" {
|
|
return Export{}, fmt.Errorf("ingest: decode export: the document carries no book_id, so it is not an export")
|
|
}
|
|
out := Export{TotalUnits: doc.TotalUnits, Units: make([]UnitText, 0, len(doc.Chunks))}
|
|
for _, c := range doc.Chunks {
|
|
out.Units = append(out.Units, UnitText{
|
|
Chapter: c.Chapter, Unit: c.ChunkIdx,
|
|
Source: c.Source, Target: c.FinalText,
|
|
State: unitState(c.Disposition, c.FinalText),
|
|
})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// unitState derives the contract's state from the pair.
|
|
//
|
|
// The text decides, and the verdict only decides what an ABSENT text means. `pending` and a flagged
|
|
// empty unit are different facts — one is work not done, the other is work that produced nothing —
|
|
// and a disposition this build has never heard of is read as `pending` rather than as either: an
|
|
// unknown verdict is not evidence that anything shipped, and `pending` is the state that promises
|
|
// the client the least.
|
|
func unitState(disposition, text string) string {
|
|
if text != "" {
|
|
return StateTranslated
|
|
}
|
|
if disposition == exportOK || disposition == exportFlagged {
|
|
return StateWithheld
|
|
}
|
|
return StatePending
|
|
}
|