159 lines
8.5 KiB
Go
159 lines
8.5 KiB
Go
package ingest
|
|
|
|
import (
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
)
|
|
|
|
// Manifest is the ALLOWLISTED reading of the engine's chapter manifest (`tmctl manifest --json`,
|
|
// unified backlog row 100, form ratified with D39.122 as `tm-manifest-v2`).
|
|
//
|
|
// It carries what intake decides with — how many chapters the book has, and which cut of which bytes
|
|
// produced that number — and, since the reading surface landed (P7), the TREE: the engine's stable
|
|
// chapter identities, its dense ordinals and the units inside each chapter.
|
|
//
|
|
// ⚠ `heading` is deliberately NOT read, and that is the allowlist doing its job rather than an
|
|
// omission. The engine's field is a deterministic render of the pair's heading rule («Глава N») and
|
|
// its own comment forbids presenting it as a label carried by the book; the contract forbids a
|
|
// deployment to put a rendered ordinal in `Chapter.heading` (canon §Chapter). A reader that decoded
|
|
// it would have somewhere to put it, which is the whole distance between an allowlist and a habit.
|
|
type Manifest struct {
|
|
// Version is `manifest_version`. Its VALUE is recorded and logged rather than gated — the fields
|
|
// below are counts and identities whose meaning is stable, and pinning the value here would make
|
|
// every engine release a platform release. Its PRESENCE is required (DecodeManifest), which is a
|
|
// different question: not "which manifest is this" but "is this a manifest at all".
|
|
Version string `json:"manifest_version"`
|
|
ChaptersTotal int `json:"chapters_total"`
|
|
UnitsTotal int `json:"units_total"`
|
|
// Key is the engine's validity key: everything whose change moves a chunk boundary or the chapter
|
|
// set is hashed into it. The platform stores it and compares one string to answer "was this book
|
|
// cut again", which is what moves `structure_version` on the wire.
|
|
Key string `json:"key"`
|
|
// SourceSHA256 is the digest of the ingested source, hex. It answers "is the file on disk still
|
|
// the one that was cut" without the platform reading the file again.
|
|
SourceSHA256 string `json:"source_sha256"`
|
|
SourceBytes int64 `json:"source_bytes"`
|
|
// ChunkerVersion is one of the inputs of the manifest's validity key: a change re-numbers
|
|
// chapters, so a stored tree is only comparable within one of these (register row PD-166).
|
|
ChunkerVersion string `json:"chunker_version"`
|
|
Chapters []ManifestChapter `json:"chapters"`
|
|
// Artifacts is the engine's own answer to where its file channels live (row 213). Taking the
|
|
// paths from here is what lets the platform stop re-deriving them from the book's configuration
|
|
// — knowing the `<project_db>.bank.json` spelling was a copy of the engine's convention that a
|
|
// changed default would have silently walked away from.
|
|
Artifacts StatusArtifacts `json:"artifacts"`
|
|
}
|
|
|
|
// StatusArtifacts is the allowlisted half of the engine's artifact envelope, published by both
|
|
// `manifest --json` and `status --json` (backend/internal/pipeline/status.go, StatusArtifacts).
|
|
// Only the channel the platform reads is taken: the project database is the engine's own and must
|
|
// not be opened here (D39.85), and the two decision files have exactly one writer, `tmctl
|
|
// bank-apply`.
|
|
type StatusArtifacts struct {
|
|
// BankExport is the whole-bank read-out — the one channel through which the bank leaves the
|
|
// engine. The path names a PLACE, not a presence: a book that has never run has no file there.
|
|
BankExport string `json:"bank_export"`
|
|
}
|
|
|
|
// ManifestChapter is one chapter of the tree.
|
|
type ManifestChapter struct {
|
|
// ID is the engine's content-derived identity: stable across a re-cut and across edits to OTHER
|
|
// chapters, which is exactly the stability the contract promises for a chapter id.
|
|
ID string `json:"id"`
|
|
// Number is the DISPLAY ordinal, dense and 1-based. Not a key — editing the source shifts every
|
|
// later one — and it is also the ordinal `unit_done` events address a chapter by.
|
|
Number int `json:"number"`
|
|
UnitsTotal int `json:"units_total"`
|
|
Units []ManifestUnit `json:"units"`
|
|
}
|
|
|
|
// ManifestUnit is one output unit — the pair the client reads.
|
|
type ManifestUnit struct {
|
|
// ID carries the cut it belongs to, so it dies when the book is cut differently. That is the
|
|
// contract's promise about a pair id, and it is the engine's property rather than ours.
|
|
ID string `json:"id"`
|
|
// FirstChunkIdx is the LEADER chunk index: the join key for both other channels — the `unit` of
|
|
// every `unit_done` event and the `chunk_idx` of every export record.
|
|
FirstChunkIdx int `json:"first_chunk_idx"`
|
|
}
|
|
|
|
// Whole reports whether the document DESCRIBES ITSELF: every list is as long as the count printed
|
|
// beside it.
|
|
//
|
|
// It is the engine's own rule, mirrored rather than invented — `BookManifest.selfConsistent`
|
|
// (backend/internal/pipeline/manifest.go) refuses a sidecar whose counters do not describe its
|
|
// contents and falls back to a full re-chunk. What it protects HERE is different and is why it is
|
|
// checked again on this side: the reader above is an ALLOWLIST, so a field the engine renames
|
|
// decodes into a zero-valued list while the counts beside it, read from other keys, still say how
|
|
// long that list should have been. The counts are the only independent witness this build has that
|
|
// it read the document at all.
|
|
//
|
|
// The bound the engine also carries (a chunk count no book could produce) has no counterpart here:
|
|
// this reader does not take chunk counts, so there is nothing for it to over-allocate from.
|
|
func (m Manifest) Whole() error {
|
|
if m.ChaptersTotal != len(m.Chapters) {
|
|
return fmt.Errorf("ingest: the manifest counts %d chapters and carries %d", m.ChaptersTotal, len(m.Chapters))
|
|
}
|
|
units := 0
|
|
for _, c := range m.Chapters {
|
|
// ⚠ THE IDENTITIES HAVE NO COUNTER BESIDE THEM, so they need a witness of their own — and they
|
|
// are the fields with the most to lose. A reader keyed on them derives the stored row's id from
|
|
// the string verbatim, so a renamed key decodes to "" for every chapter or every pair, every
|
|
// row collapses onto one derived id, and the replacement write then deletes the rest of the
|
|
// book. Measured: a three-pair chapter came out as one row, with the document passing every
|
|
// count it declares. Refusing an empty id costs nothing — the engine never emits one
|
|
// (backend/internal/pipeline/manifest.go, buildManifest) — and it is the only check here that
|
|
// the counts cannot make for us.
|
|
if c.ID == "" {
|
|
return fmt.Errorf("ingest: chapter %d carries no id, so this build is not reading the manifest's identities", c.Number)
|
|
}
|
|
for _, u := range c.Units {
|
|
if u.ID == "" {
|
|
return fmt.Errorf("ingest: a pair of chapter %d carries no id, so this build is not reading the manifest's identities", c.Number)
|
|
}
|
|
}
|
|
if c.Number < 1 {
|
|
return fmt.Errorf("ingest: the manifest carries a chapter numbered %d, and chapter numbering is 1-based", c.Number)
|
|
}
|
|
if c.UnitsTotal != len(c.Units) {
|
|
return fmt.Errorf("ingest: chapter %d counts %d pairs and carries %d", c.Number, c.UnitsTotal, len(c.Units))
|
|
}
|
|
units += len(c.Units)
|
|
}
|
|
if m.UnitsTotal != units {
|
|
return fmt.Errorf("ingest: the manifest counts %d pairs and its chapters carry %d", m.UnitsTotal, units)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SourceSHA256Bytes is the digest as the column stores it. An unparsable value is stored as nothing
|
|
// rather than as garbage: the field is evidence, and evidence that cannot be decoded is absence.
|
|
func (m Manifest) SourceSHA256Bytes() []byte {
|
|
b, err := hex.DecodeString(m.SourceSHA256)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return b
|
|
}
|
|
|
|
// DecodeManifest parses a manifest document, and refuses anything that does not identify itself as
|
|
// one.
|
|
//
|
|
// ⚠ The identity check is not decoration, and what it protects is the one path in this zone that
|
|
// DESTROYS a user's file. `{}`, `null` and any JSON object of fields this build has never heard of
|
|
// all decode into a Manifest of zeros, and intake reads a zero chapter count as "the engine read the
|
|
// source and there is no book in it" — the single verdict that removes the upload (books.Parse,
|
|
// books.reject). So a document that is not a manifest must not be able to arrive as an EMPTY one:
|
|
// the two are opposite facts, and only one of them is destructive. Raised by the seam lens of the
|
|
// dofix review.
|
|
func DecodeManifest(b []byte) (Manifest, error) {
|
|
var m Manifest
|
|
if err := json.Unmarshal(b, &m); err != nil {
|
|
return Manifest{}, fmt.Errorf("ingest: decode manifest: %w", err)
|
|
}
|
|
if m.Version == "" {
|
|
return Manifest{}, fmt.Errorf("ingest: decode manifest: the document carries no manifest_version, so it is not a manifest")
|
|
}
|
|
return m, nil
|
|
}
|