textmachine/platform/internal/ingest/manifest.go

package ingest

import (
	"encoding/hex"
	"encoding/json"
	"fmt"

	"textmachine/platform/internal/money"
)

// Manifest is the ALLOWLISTED reading of the engine's chapter manifest (`tmctl manifest --json`,
// unified backlog row 100, form ratified with D39.122 as `tm-manifest-v2`).
//
// It carries what intake decides with — how many chapters the book has, and which cut of which bytes
// produced that number — and, since the reading surface landed (P7), the TREE: the engine's stable
// chapter identities, its dense ordinals and the units inside each chapter.
//
// ⚠ `heading` is deliberately NOT read, and that is the allowlist doing its job rather than an
// omission. The engine's field is a deterministic render of the pair's heading rule («Глава N») and
// its own comment forbids presenting it as a label carried by the book; the contract forbids a
// deployment to put a rendered ordinal in `Chapter.heading` (canon §Chapter). A reader that decoded
// it would have somewhere to put it, which is the whole distance between an allowlist and a habit.
type Manifest struct {
	// Version is `manifest_version`. Its VALUE is recorded and logged rather than gated — the fields
	// below are counts and identities whose meaning is stable, and pinning the value here would make
	// every engine release a platform release. Its PRESENCE is required (DecodeManifest), which is a
	// different question: not "which manifest is this" but "is this a manifest at all".
	Version       string `json:"manifest_version"`
	ChaptersTotal int    `json:"chapters_total"`
	UnitsTotal    int    `json:"units_total"`
	// Key is the engine's validity key: everything whose change moves a chunk boundary or the chapter
	// set is hashed into it. The platform stores it and compares one string to answer "was this book
	// cut again", which is what moves `structure_version` on the wire.
	Key string `json:"key"`
	// SourceSHA256 is the digest of the ingested source, hex. It answers "is the file on disk still
	// the one that was cut" without the platform reading the file again.
	SourceSHA256 string `json:"source_sha256"`
	SourceBytes  int64  `json:"source_bytes"`
	// ChunkerVersion is one of the inputs of the manifest's validity key: a change re-numbers
	// chapters, so a stored tree is only comparable within one of these (register row PD-166).
	ChunkerVersion string `json:"chunker_version"`
	// Structure says WHICH PATH drew the chapter boundaries: `declared` (the FORMAT drew them — an
	// EPUB spine, a form feed), `detected` (the engine matched header-shaped lines in the prose) or
	// `none` (the whole book is one chapter).
	//
	// ⚠ IT IS PROVENANCE AND NOT TRUST, and the distinction is worth a line because the engine's own
	// file gets it wrong in its opening sentence and right two lines later
	// (backend/internal/chunk/ingest.go, the provenance comment). What the field states is which
	// mechanism produced the boundaries; how much any of those mechanisms is worth is a decision this
	// side makes on top of it — see ChapterOrdersOffered, which trusts `detected` alone and says why.
	// Inheriting the word «trust» from the publisher is how the same error travels a third time.
	//
	// An unknown value is read as untrusted by ChapterOrdersOffered rather than refused: the
	// vocabulary is the engine's and is ratified to GROW, so a word this build has never seen costs a
	// character slider and an honest sentence, never a failure.
	Structure string `json:"structure"`
	// Price is the engine's A PRIORI projection of what this book costs and what ceiling it needs —
	// derived from the text, the pair's own calibration and the engine's own prices, before a single
	// call is made (backend/internal/pipeline/priceprojection.go, landing 81a89e9, act D39.206).
	//
	// A POINTER, and absent is not zero (PD-40): a book the engine could not price at all has no
	// projection, and the platform REFUSES to sell it rather than invent a per-chapter constant. That
	// refusal is the whole reason the constant could be deleted — see pricing.Model.
	Price    *BookPrice        `json:"price"`
	Chapters []ManifestChapter `json:"chapters"`
	// Artifacts is the engine's own answer to where its file channels live (row 213). Taking the
	// paths from here is what lets the platform stop re-deriving them from the book's configuration
	// — knowing the `<project_db>.bank.json` spelling was a copy of the engine's convention that a
	// changed default would have silently walked away from.
	Artifacts StatusArtifacts `json:"artifacts"`
}

// StatusArtifacts is the allowlisted half of the engine's artifact envelope, published by both
// `manifest --json` and `status --json` (backend/internal/pipeline/status.go, StatusArtifacts).
// Only the channel the platform reads is taken: the project database is the engine's own and must
// not be opened here (D39.85), and the two decision files have exactly one writer, `tmctl
// bank-apply`.
type StatusArtifacts struct {
	// BankExport is the whole-bank read-out — the one channel through which the bank leaves the
	// engine. The path names a PLACE, not a presence: a book that has never run has no file there.
	BankExport string `json:"bank_export"`
}

// BookPrice is the book-level half of the engine's price projection, converted to whole micro-USD
// AT THE SEAM — the same rule the committed figure follows (resync.go, PD-15): a JSON decimal bound
// to a float64 would put drift one step before the integer column that exists to prevent drift.
//
// ⚠ THE TWO SUMS ARE DIFFERENT ARITHMETICS AND MUST NOT BE ADDED CASUALLY. `ExpectedUSD` is what the
// book is expected to be BILLED and it ALREADY INCLUDES `BookOnceUSD`; `StepMaxUSD` is the largest
// single INDIVISIBLE reservation and it is worst-case by construction. The engine says so itself and
// says why: publishing one number for both is what produced «this book cannot be translated at any
// price» (priceprojection.go, the two-numbers comment).
type BookPrice struct {
	// ExpectedUSD is the whole book's expected bill, book-level passes INCLUDED.
	ExpectedUSD money.MicroUSD `json:"expected_usd"`
	// BookOnceUSD is the part of ExpectedUSD that belongs to the BOOK rather than to any chapter: the
	// terminology consolidation and its classifier read the whole book's drafts once.
	//
	// ⚠ IT IS A BOUND AND NOT A FORECAST — the engine's own words — and on the shipped arm it is a
	// FLAT $2.00 (gates.terminology.budget_usd $1.00 + classify_budget_usd $1.00, configs/pipeline-c1.yaml)
	// whatever the book's length. That is why the platform prices an ORDER from the chapters' own
	// expected bills and not from this sum: folding a flat two dollars into the hold would make a
	// five-chapter book unbuyable again, which is the PD-440 wall in new clothes. See pricing.Hold.
	BookOnceUSD money.MicroUSD `json:"book_once_usd"`
	// StepMaxUSD is the largest single reservation any one call of this book can ask for — «the number
	// the wall was made of». A ceiling below it admits NOTHING, so it is the floor of every hold and
	// the minimum of the money slider (D39.196 §1, unified backlog row 276).
	StepMaxUSD money.MicroUSD `json:"step_max_usd"`
	// SourceChars is the whole book's ingested text in RUNES, spaces included — a figure a person can
	// reproduce in an editor. It is the honest replacement for the intake's write-stream count, which
	// for an EPUB counts the runes of a ZIP archive (unified backlog row 282).
	SourceChars int64 `json:"source_chars"`
}

// UnitPrice is the size and expected bill of one chapter (the engine rolls its units up) or of one
// unit. Only the two members the platform prices an order with are taken: the dense/sparse split is
// the engine's token-sizing taxonomy and nothing here converts characters to money directly.
type UnitPrice struct {
	SourceChars int64          `json:"source_chars"`
	ExpectedUSD money.MicroUSD `json:"expected_usd"`
}

// ManifestChapter is one chapter of the tree.
type ManifestChapter struct {
	// ID is the engine's content-derived identity: stable across a re-cut and across edits to OTHER
	// chapters, which is exactly the stability the contract promises for a chapter id.
	ID string `json:"id"`
	// Number is the DISPLAY ordinal, dense and 1-based. Not a key — editing the source shifts every
	// later one — and it is also the ordinal `unit_done` events address a chapter by.
	Number     int            `json:"number"`
	UnitsTotal int            `json:"units_total"`
	Units      []ManifestUnit `json:"units"`
	// Price is the chapter's rolled-up size and expected bill. It is read at CHAPTER granularity and
	// not at unit granularity because that is the granularity an order is phrased in (D39.196 §1), and
	// because the engine already did the roll-up — re-deriving it here would be a second arithmetic
	// over one fact.
	Price *UnitPrice `json:"price"`
}

// ManifestUnit is one output unit — the pair the client reads.
type ManifestUnit struct {
	// ID carries the cut it belongs to, so it dies when the book is cut differently. That is the
	// contract's promise about a pair id, and it is the engine's property rather than ours.
	ID string `json:"id"`
	// FirstChunkIdx is the LEADER chunk index: the join key for both other channels — the `unit` of
	// every `unit_done` event and the `chunk_idx` of every export record.
	FirstChunkIdx int `json:"first_chunk_idx"`
	// Price is this unit's own size and expected bill. Read at UNIT granularity as well as at chapter
	// granularity because a book whose structure was not recognised is ONE chapter, and the only
	// partial order such a book can carry is a number of CHARACTERS — which resolves to a prefix of
	// units and needs each unit's own size to resolve at all.
	Price *UnitPrice `json:"price"`
}

// KnownManifestVersion is the shape of the engine's manifest this build was written against. It
// MIRRORS `manifestVersion` in backend/internal/pipeline/manifest.go and moves when that moves.
//
// It is a shape gate and not a value pin, which is the distinction the Version field's own comment
// draws: the counts and identities below are stable, so pinning the value everywhere would make
// every engine release a platform release. What it guards is the ONE place where reading a manifest
// wrong is destructive — see Readable.
const KnownManifestVersion = "tm-manifest-v2"

// Readable is the floor the INTAKE stands on: is this a manifest this build can read at all.
//
// Two questions, and they are one floor because the answer to both is the same non-destructive class.
//
// The version, which is register row PD-213 and a latent mine: `DecodeManifest` does not gate the
// value, `json.Unmarshal` silently ignores unknown fields and leaves missing ones zero — so a shape
// change on the engine's side (`chapters_total` renamed, say) decodes into a perfectly valid Manifest
// with ChaptersTotal 0 and UnitsTotal 0. Zero chapters and zero units is exactly what the intake
// reads as "the engine read the source and there is no book in it", which is the ONE verdict that
// DELETES the user's file after the attempt budget. The engine would have parsed the book perfectly.
//
// The self-consistency, which is register row PD-367: `Whole()` already guarded the materialiser, and
// the intake accepted the very documents the materialiser would refuse — `{chapters_total: 120,
// units_total: 400}` with an empty chapter list founded a book with `chapter_count = 120` and an
// empty tree, over which a run could be STARTED AND PAID FOR. The floor is symmetric now: one
// document, one answer, on both sides of the intake.
//
// The class is the law of the seam's: unknown or inconsistent is answered NON-DESTRUCTIVELY and
// LOUDLY. The caller maps this to `parser_unavailable`, which spends the attempt budget and keeps the
// file — never `source_unreadable`, which is a statement about the USER's text and is the one that
// deletes it.
func (m Manifest) Readable() error {
	if m.Version != KnownManifestVersion {
		return fmt.Errorf("ingest: the manifest identifies as %q and this build reads %s",
			m.Version, KnownManifestVersion)
	}
	return m.Whole()
}

// Whole reports whether the document DESCRIBES ITSELF: every list is as long as the count printed
// beside it.
//
// It is the engine's own rule, mirrored rather than invented — `BookManifest.selfConsistent`
// (backend/internal/pipeline/manifest.go) refuses a sidecar whose counters do not describe its
// contents and falls back to a full re-chunk. What it protects HERE is different and is why it is
// checked again on this side: the reader above is an ALLOWLIST, so a field the engine renames
// decodes into a zero-valued list while the counts beside it, read from other keys, still say how
// long that list should have been. The counts are the only independent witness this build has that
// it read the document at all.
//
// The bound the engine also carries (a chunk count no book could produce) has no counterpart here:
// this reader does not take chunk counts, so there is nothing for it to over-allocate from.
func (m Manifest) Whole() error {
	if m.ChaptersTotal != len(m.Chapters) {
		return fmt.Errorf("ingest: the manifest counts %d chapters and carries %d", m.ChaptersTotal, len(m.Chapters))
	}
	units := 0
	for _, c := range m.Chapters {
		// ⚠ THE IDENTITIES HAVE NO COUNTER BESIDE THEM, so they need a witness of their own — and they
		// are the fields with the most to lose. A reader keyed on them derives the stored row's id from
		// the string verbatim, so a renamed key decodes to "" for every chapter or every pair, every
		// row collapses onto one derived id, and the replacement write then deletes the rest of the
		// book. Measured: a three-pair chapter came out as one row, with the document passing every
		// count it declares. Refusing an empty id costs nothing — the engine never emits one
		// (backend/internal/pipeline/manifest.go, buildManifest) — and it is the only check here that
		// the counts cannot make for us.
		if c.ID == "" {
			return fmt.Errorf("ingest: chapter %d carries no id, so this build is not reading the manifest's identities", c.Number)
		}
		for _, u := range c.Units {
			if u.ID == "" {
				return fmt.Errorf("ingest: a pair of chapter %d carries no id, so this build is not reading the manifest's identities", c.Number)
			}
		}
		if c.Number < 1 {
			return fmt.Errorf("ingest: the manifest carries a chapter numbered %d, and chapter numbering is 1-based", c.Number)
		}
		if c.UnitsTotal != len(c.Units) {
			return fmt.Errorf("ingest: chapter %d counts %d pairs and carries %d", c.Number, c.UnitsTotal, len(c.Units))
		}
		units += len(c.Units)
	}
	if m.UnitsTotal != units {
		return fmt.Errorf("ingest: the manifest counts %d pairs and its chapters carry %d", m.UnitsTotal, units)
	}
	return nil
}

// SourceSHA256Bytes is the digest as the column stores it. An unparsable value is stored as nothing
// rather than as garbage: the field is evidence, and evidence that cannot be decoded is absence.
func (m Manifest) SourceSHA256Bytes() []byte {
	b, err := hex.DecodeString(m.SourceSHA256)
	if err != nil {
		return nil
	}
	return b
}

// DecodeManifest parses a manifest document, and refuses anything that does not identify itself as
// one.
//
// ⚠ The identity check is not decoration, and what it protects is the one path in this zone that
// DESTROYS a user's file. `{}`, `null` and any JSON object of fields this build has never heard of
// all decode into a Manifest of zeros, and intake reads a zero chapter count as "the engine read the
// source and there is no book in it" — the single verdict that removes the upload (books.Parse,
// books.reject). So a document that is not a manifest must not be able to arrive as an EMPTY one:
// the two are opposite facts, and only one of them is destructive. Raised by the seam lens of the
// dofix review.
func DecodeManifest(b []byte) (Manifest, error) {
	var m Manifest
	if err := json.Unmarshal(b, &m); err != nil {
		return Manifest{}, fmt.Errorf("ingest: decode manifest: %w", err)
	}
	if m.Version == "" {
		return Manifest{}, fmt.Errorf("ingest: decode manifest: the document carries no manifest_version, so it is not a manifest")
	}
	return m, nil
}

// StructureDeclared, StructureDetected and StructureNone are the engine's whole vocabulary for where
// a chapter cut came from (backend/internal/pipeline/manifest.go, BookManifest.Structure).
const (
	StructureDeclared = "declared"
	StructureDetected = "detected"
	StructureNone     = "none"
)

// ChapterOrdersOffered answers the only question the order form asks of the cut: may an order be
// phrased in CHAPTERS, or must it be phrased in CHARACTERS with the fact said out loud (D39.196 §1)?
//
// ⛔ ONLY `detected` IS TRUSTED, AND `declared` IS DELIBERATELY NOT — which is the opposite of what
// the word suggests, so the reason is written here rather than remembered. For an EPUB the engine
// cuts the book by SPINE DOCUMENTS — one document, one «chapter» — and honestly labels that
// `declared`, because the format did draw those boundaries. But the spine is the READING ORDER, not
// the table of contents: the real chapter boundaries live in `nav`/NCX, which the engine does not
// read yet (unified backlog rows 160-162, and row 283 is the pack that will). So `declared` today
// means «this many documents», not «this many chapters»: a book shipped as one large document has
// one, a book split by scene has dozens. Offering «through chapter 12» against it sells a buyer
// twelve DOCUMENTS under the name of twelve chapters.
//
// Ratified 05.09 by the orchestrator, on the structure pack's own reading. The vocabulary itself
// stays three-valued and the engine's structure pack will EXTEND it additively rather than replace
// it, so this decision is about trust and not about shape.
//
// An EMPTY or UNKNOWN value is not trusted either, and for the mirror reason: the field is additive,
// so a manifest written before the landing carries no `structure` at all and a later engine may
// carry a word this build has never read. Both decode to something this function does not know, and
// the safe answer to «is this cut trustworthy enough to sell chapters against» is no — the order is
// then offered in characters and the client says so, which is a worse offer and never a wrong one.
func (m Manifest) ChapterOrdersOffered() bool {
	return m.Structure == StructureDetected
}

// Priced returns the book's projection, and FALSE when this build did not read a whole and
// self-consistent one.
//
// ⚠ ABSENT IS NOT ZERO, AND HALF-READ IS NOT ABSENT — the two failures need one answer and it is
// this one. `Price` is `omitempty` on the engine's side, so a book the runner could not price at all
// legitimately carries none; but `json.Unmarshal` also leaves a RENAMED key at its zero value, and a
// projection whose `step_max_usd` decoded as zero is the exact shape of the PD-440 wall coming back
// silently: the hold loses its floor, the run is admitted with a ceiling no single call can clear,
// and it dies at once having spent nothing and moved nothing. So a zero in a load-bearing member is
// read as «not read» and the sale is refused, never as «this book is free».
//
// THE CHAPTERS ARE THE INDEPENDENT WITNESS, for the same reason the counts are the witness of the
// tree (see Whole): the platform prices an ORDER from the chapters' own expected bills, and a
// per-chapter key the engine renamed would decode into a book where every chapter is free while the
// book-level figure, read from another key, still looks like money. The engine emits both halves
// from ONE derivation or neither (priceprojection.go, projectBook), so a document carrying one
// without the other was not read correctly whatever else is true of it.
func (m Manifest) Priced() (BookPrice, bool) {
	p, why := m.priced()
	return p, why == ""
}

// PriceRefusal is Priced's answer in words: "" when the book is priced, and otherwise WHICH of the
// five things was missing.
//
// ⛔ IT EXISTS BECAUSE THE REFUSAL WAS MUTE, and a mute refusal here is a deployment that never
// sells. A book whose projection this build cannot read is materialized normally — its tree, its
// text, its reading surface all land — and only the SALE is refused; nothing logged, no metric, no
// listing. On a deployment whose engine predates the projection every book on the host would arrive
// in exactly that state, and an operator would have no line to grep for. The five reasons are
// distinguished because they mean opposite things: «this engine does not publish prices at all» is a
// deploy to fix, and «one chapter lost its price» is a document this build read wrong.
func (m Manifest) PriceRefusal() string {
	_, why := m.priced()
	return why
}

// priced is the one walk both of the above read, so the answer and its reason cannot disagree.
func (m Manifest) priced() (BookPrice, string) {
	switch {
	case m.Price == nil:
		return BookPrice{}, "the manifest carries no price projection at all"
	case m.Price.StepMaxUSD <= 0:
		return BookPrice{}, "step_max_usd did not arrive, so no hold would have a floor"
	case m.Price.ExpectedUSD <= 0:
		return BookPrice{}, "expected_usd did not arrive, so an order has nothing to be priced from"
	case m.Price.SourceChars <= 0:
		return BookPrice{}, "source_chars did not arrive"
	}
	sum := money.MicroUSD(0)
	for _, c := range m.Chapters {
		if c.Price == nil || c.Price.ExpectedUSD <= 0 {
			return BookPrice{}, fmt.Sprintf("chapter %d carries no price, so no partial order can be quoted", c.Number)
		}
		sum += c.Price.ExpectedUSD
		// ⚠ EVERY UNIT IS WITNESSED, ON BOTH AXES, and neither is redundant with the chapter above it.
		//
		// The SIZE: a renamed `source_chars` decodes to zero on every unit, and a CHARACTER order
		// resolved against zeroes reaches the end of the book without ever meeting the figure it was
		// asked for — a partial order silently becoming a total one.
		//
		// The PRICE: a character order is quoted from the UNITS' own bills, so zeroes there sell a
		// prefix of the book for nothing and hold nothing against it — the chapter roll-up would not
		// notice, because it is read from a different key and stays money. Both are the same
		// «half-read is not absent» class as the guard above, on the two axes a reader of this
		// function does not think to check: one because it is not money, and one because the
		// chapter's figure looks like it already covered it.
		for _, u := range c.Units {
			if u.Price == nil || u.Price.SourceChars <= 0 || u.Price.ExpectedUSD <= 0 {
				return BookPrice{}, fmt.Sprintf("a unit of chapter %d carries no size or no price, "+
					"so a character order over this book cannot be resolved", c.Number)
			}
		}
	}
	// The cross-witness: the chapters' bills plus the book-level pass ARE the book's expected bill
	// (projectBook adds BookOnceUSD to the sum of the units and nothing else). The slack is the
	// conversion's own: every amount is rounded UP to whole micro-USD at this seam, independently, so
	// the chapters can sum one micro-dollar per chapter above their own share. Anything outside that
	// is two numbers from two documents.
	slack := money.MicroUSD(len(m.Chapters) + 1)
	units := m.Price.ExpectedUSD - m.Price.BookOnceUSD
	if sum < units-slack || sum > units+slack {
		return BookPrice{}, "the chapters' own bills and the book's figure describe different books"
	}
	return *m.Price, ""
}