textmachine/backend/internal/pipeline/manifest.go

package pipeline

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"strconv"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/text"
)

// manifest.go: the PERSISTED chapter/chunk manifest (backlog row 100).
//
// Two problems, one artifact. (1) Every $0 read re-ingested and re-cut the whole source to get its
// honest N/M denominator — 1.4–1.5 s of CPU per call on a 23 MB book, paid again on every poll of every
// book. (2) The chapter tree a reader needs has no carrier at all: the engine addresses chapters by a
// DENSE ordinal (chapters that yield no text do not consume a number, chunker.go), so editing the source
// renumbers every later chapter and any bookmark built on the number silently moves to another chapter.
//
// The artifact is a FILE beside the project DB, not a table, for the same reason the bank export is one
// (bankexport.go): the reader is another process which must not open the engine's private store
// (D39.85). It is also a projection and never a source — nothing reads it back into a decision the
// engine makes; a stale one is DETECTED and ignored, never trusted.
//
// STALENESS is decided by a KEY, not by the mtime and not by a flag. Everything whose change moves the
// chunk boundaries or the chapter set is hashed into it, so a manifest written under a different chunker,
// budget, langpack, embedded data set, encoding or source file cannot be mistaken for a current one. A
// write path rebuilds it unconditionally (so `translate --resnapshot`, which is the loud "the chunker
// moved" command, rebuilds it like every other run); a read path that finds a key mismatch falls back to
// the full re-chunk it did before this file existed. There is no state in which a wrong manifest is used.

// manifestVersion versions the DOCUMENT SHAPE. It also rides the key, so a shape change invalidates every
// stored manifest rather than being half-read by a binary expecting the other layout.
//
// v2 (acceptance V2-4): unit ids gained the cut tag — «<chapter>:<cutTag>:<idx>» instead of
// «<chapter>:<idx>». A v1 sidecar is structurally valid and would keep validating, handing a reader ids
// in the OLD form whose whole defect was that they outlive the cut. The bump is what refuses it.
const manifestVersion = "tm-manifest-v2"

// BookManifest is the persisted chapter/chunk structure of a book.
type BookManifest struct {
	Version string `json:"manifest_version"`
	BookID  string `json:"book_id"`
	// Key is the validity key (manifestKey). A reader that only wants "is this still the same book, cut
	// the same way" compares this one string.
	Key string `json:"key"`
	// The key's INPUTS, spelled out beside it. The key alone answers "is it current"; these answer "what
	// changed" when it is not, which is the question an operator staring at an invalidated tree asks.
	ChunkerVersion string `json:"chunker_version"`
	SourceSHA256   string `json:"source_sha256"`
	SourceBytes    int64  `json:"source_bytes"`
	SourceLang     string `json:"source_lang"`
	TargetLang     string `json:"target_lang"`
	Encoding       string `json:"encoding"`

	// Structure says WHERE the chapter boundaries came from — `declared` (the FORMAT drew them: an EPUB
	// spine, a form feed), `detected` (this engine matched header-shaped lines in the prose) or `none`
	// (one chapter). It is a statement about TRUST, not about count, and a consumer needs it before it
	// offers an order phrased in chapters: an order «through chapter 12» against a detected cut is an
	// order against a guess, and a reader has to be told that in words rather than discover it.
	Structure string `json:"structure"`

	Chapters      []ManifestChapter `json:"chapters"`
	ChaptersTotal int               `json:"chapters_total"`
	UnitsTotal    int               `json:"units_total"`
	ChunksTotal   int               `json:"chunks_total"`

	// Price is what this book is expected to cost and what ceiling it needs — derived from the text, the
	// pair's own calibration and the engine's own prices, with no per-chapter constant anywhere in it
	// (priceprojection.go). Absent when the runner could not resolve prices at all.
	//
	// ⚠ ADDITIVE, AND THE DOCUMENT VERSION DELIBERATELY DOES NOT MOVE FOR IT — see Artifacts above for
	// the full reasoning: a version bump makes loadManifest discard every stored sidecar and re-chunk
	// every book once, and it buys nothing for a field that cannot be half-read.
	Price *BookPrice `json:"price,omitempty"`

	// Artifacts are the engine-owned paths a reader outside this process needs — the same envelope
	// `status --json` publishes, on the surface the reader actually calls.
	//
	// It is here rather than only there because of who reads what. `status` is expensive by construction
	// (it re-ingests the book) and the platform's read-model does not call it at all: its engine interface
	// is {Manifest, Export}, and the one place it needs an engine-owned path it DERIVES the path itself,
	// re-implementing the engine's `<workdir>/<book_id>.db` convention (unified backlog row 213). A
	// published envelope that the consumer never reads removes nothing.
	//
	// DERIVED ON EVERY READ, never trusted from the file. The document is persisted, and a path stored in
	// a sidecar that travels with the book directory is a path that is wrong the first time the directory
	// moves — the stand's own book.yaml files carry dead absolute paths for exactly that reason. Stamping
	// it on load as well as on build is what makes the stored copy incapable of lying.
	//
	// ⚠ Why the document VERSION does not move for it, spelled out so the next additive field is not
	// bumped «to be safe»: a version change makes loadManifest discard every stored sidecar and re-chunk
	// every book once (1.4–1.5 s on a 23 MB book), and it buys nothing here — the field cannot be
	// half-read, because it is recomputed on every read rather than trusted. The version exists for
	// changes a reader could MISREAD (v1→v2 moved the unit ids); additive fields the engine derives are
	// the ratified «engine-first» compatibility path of the seam, not a shape break.
	Artifacts StatusArtifacts `json:"artifacts"`
}

// BookPrice is the book-level half of the price projection.
type BookPrice struct {
	// ExpectedUSD is the sum of every unit's expected bill PLUS BookOnceUSD — what translating the whole
	// book is expected to cost, once.
	ExpectedUSD float64 `json:"expected_usd"`
	// BookOnceUSD is the spend that belongs to the BOOK rather than to a unit: the terminology
	// consolidation and its classifier read the whole book's drafts once, so charging them per unit would
	// under-price a short book exactly where the error hurts most. ⚠ It is a BOUND and not a forecast —
	// the passes' input does not exist until the drafts do (see bookOnceUSD).
	BookOnceUSD float64 `json:"book_once_usd"`
	// StepMaxUSD is the largest single INDIVISIBLE reservation this book can ask for. A ceiling below it
	// admits nothing at all — this is the number the wall was made of, and publishing it is the point of
	// the whole projection: a run whose ceiling clears StepMaxUSD can always move, whatever else is true.
	StepMaxUSD float64 `json:"step_max_usd"`
	// SourceChars is the whole book's ingested text in runes, spaces included.
	SourceChars int `json:"source_chars"`
}

// UnitPrice is one output unit's size and expected bill.
type UnitPrice struct {
	SourceChars       int     `json:"source_chars"`
	SourceCharsDense  int     `json:"source_chars_dense"`
	SourceCharsSparse int     `json:"source_chars_sparse"`
	ExpectedUSD       float64 `json:"expected_usd"`
}

// ManifestChapter is one chapter: its stable identity, its DISPLAY ordinal, and its units.
type ManifestChapter struct {
	// ID is the engine's STABLE key for this chapter: 16 hex chars of SHA-256 over the chapter's INGESTED
	// text (see manifestChapterID). It is stable across a chunker/budget/heading-rule change and across
	// edits to OTHER chapters — which is the whole point, since Number is not: numbering is dense, so
	// inserting a chapter shifts every later ordinal while every later id stays put.
	ID string `json:"id"`
	// Number is the dense 1-based ordinal the engine addresses chunk_status rows by. It is a POSITION,
	// not a key — see ID.
	Number int `json:"number"`
	// Heading is the title the engine RENDERS for this chapter from the pair's heading rule («Глава N»),
	// or "" when the book has no rule or the chapter has no structural header. ⚠ It is the engine's own
	// deterministic render — the source marker is stripped from the text the model sees and this string is
	// glued back onto the chapter's first output unit at export (chunk.ApplyHeading). It is NOT a label
	// carried by the book's own data, and a consumer must not present it as one. Which label a reader
	// should see is an open contract question (companion §4, K-2/K-3), not settled here.
	Heading string `json:"heading"`
	// TitleRaw is the chapter's title AS THE SOURCE SPELLS IT — the header line of a txt, the navLabel of an
	// EPUB — in the SOURCE language, and "" when the source names nothing. It rides beside Heading rather
	// than replacing it: Heading is what this engine renders in the TARGET language, and a title that never
	// travels cannot be rendered by a reader at all. Not omitempty: "" is the answer "the source named
	// nothing", and a consumer must not have to tell that from an absent field.
	TitleRaw    string         `json:"title_raw"`
	UnitsTotal  int            `json:"units_total"`
	ChunksTotal int            `json:"chunks_total"`
	Units       []ManifestUnit `json:"units"`
	// Price is the chapter's rolled-up size and expected bill. It is here and not only on the units
	// because CHAPTERS are the unit an order is phrased in when the cut is trusted (D39.196 §1: a slider
	// in chapters when the book was recognised), and a consumer answering «does this cover the whole
	// book» should not have to re-derive the roll-up the engine already did.
	Price *UnitPrice `json:"price,omitempty"`
}

// ManifestUnit is one OUTPUT unit — the granularity the engine ships and every read model counts in.
type ManifestUnit struct {
	// ID is stable only as long as the CUT is, and it SAYS SO in its own bytes: it is the chapter id, the
	// cut tag, and the unit's leader chunk index (manifestUnitID). A chunker/budget/pipeline-shape change
	// re-cuts the chapter into different units, mints a new cut tag, and therefore mints a new id for
	// EVERY unit of the book — including a chapter's first unit, whose leader index is 0 under any cut and
	// which would otherwise carry an id that outlives the text it named.
	//
	// So: chapter identity survives a re-chunk, unit identity does not, and a consumer can tell the two
	// apart by whether the id it holds still appears in the current manifest.
	ID string `json:"id"`
	// FirstChunkIdx is the LEADER chunk index — the (chapter, chunk_idx) the unit's shipping row is
	// addressed at in chunk_status, and therefore the join key for every progress read.
	FirstChunkIdx int `json:"first_chunk_idx"`
	// ChunkCount is how many draft chunks the unit groups; members are the CONSECUTIVE indices
	// [FirstChunkIdx, FirstChunkIdx+ChunkCount) of this chapter (chunk.assignEditUnits groups consecutive
	// chunks, which is what makes storing a count instead of a list lossless).
	ChunkCount int `json:"chunk_count"`
	// EditUnitID is the engine's book-global edit-unit id for this unit.
	EditUnitID int `json:"edit_unit_id"`
	// Price is this unit's size and expected bill (priceprojection.go). SourceChars counts RUNES INCLUDING
	// SPACES so a person can reproduce it in an editor; the dense/sparse split is the engine's own
	// token-sizing taxonomy, and it is reported because the same number of characters is very different
	// money in a Han source and an alphabetic one.
	Price *UnitPrice `json:"price,omitempty"`
}

// manifestPath is where the manifest lands: beside the project DB, like every other book-state sidecar.
func (r *Runner) manifestPath() string { return r.Book.ProjectDB + ".manifest.json" }

// manifestChapterID is a chapter's stable identity: the first 64 bits of SHA-256 over its INGESTED text.
//
// WHY CONTENT AND NOT POSITION. The alternative — hashing the book id and the ordinal — is the ordinal
// with extra steps: inserting a chapter would re-point every later id at a different chapter's content,
// which is precisely the "open tabs and bookmarks quietly move" damage the contract companion names (§5).
// Content-derived, an edit to chapter 5 mints a new id for chapter 5 alone and every other chapter keeps
// the id it had, whatever its number became.
//
// DUPLICATES: two chapters with byte-identical text would hash the same, so the second and later
// occurrences carry an occurrence suffix. Deterministic for a given reading order, and honest about its
// own edge — deleting the FIRST of a duplicate pair promotes the second, changing its id.
func manifestChapterID(ingested string, occurrence int) string {
	sum := sha256.Sum256([]byte(ingested))
	id := hex.EncodeToString(sum[:])[:16]
	if occurrence > 1 {
		id += "-" + strconv.Itoa(occurrence)
	}
	return id
}

// manifestUnitID is a unit's identity: its chapter's id, the CUT it belongs to, and the leader chunk
// index.
//
// The cut tag is load-bearing and was missing in the first version, which made the declared boundary
// FALSE rather than merely coarse. Without it a unit id is «<chapter>:<firstChunkIdx>», and a chapter's
// FIRST unit is always index 0 — so «<chapter>:0» survived every re-chunk unchanged while pointing at a
// different span of text. A consumer that persists unit ids reads "the id is still there" as "my anchor
// is still valid", and lands on the wrong text with no signal. With the tag, ANY re-cut mints new unit
// ids across the whole book, which is exactly what ManifestUnit.ID's boundary says it does.
func manifestUnitID(chapterID, cutTag string, firstChunkIdx int) string {
	return chapterID + ":" + cutTag + ":" + strconv.Itoa(firstChunkIdx)
}

// cutTag identifies the CUT — everything that decides where chunk boundaries fall and how chunks group
// into output units. It is a SHORT hash because it rides inside every unit id; the full identity of the
// manifest is the validity key, which folds these same inputs plus the source bytes and the book identity.
//
// The last three are the acceptance V2-3 catch and are the same class as the defect the tag exists to
// close: the langpack decides the heading rule (a header-only chapter can stop consuming a number), the
// embedded data feeds the CJK numerals and sentence terminators that place boundaries, and the
// normalization decides what the chunker is even splitting. Any of them moving re-cuts the book while
// chunker_version and the budget stand still — and a manifest rebuilt afterwards would stamp the same
// unit ids onto different text.
func (r *Runner) cutTag() string { return r.cutInputs().tag() }

// cutInputs is everything the cut tag folds, as a named type rather than an anonymous literal. Named for a
// reason a reader should not have to guess: the tag is an eight-character hash, so a component silently
// dropped from it is invisible in every output the engine produces — the id still looks like an id. With
// the inputs addressable, a test can hold each component to its claim (cuttag_test.go) instead of
// re-writing the payload beside the code and pinning its own copy.
type cutInputs struct {
	Chunker      string           `json:"chunker_version"`
	Segmentation segmentationSnap `json:"segmentation"`
	ShippingWave string           `json:"shipping_wave"`
	Langpack     string           `json:"langpack_version,omitempty"`
	Embedded     string           `json:"embedded_version"`
	Norm         string           `json:"norm_version"`
	// Structure is the source language's chapter grammar. It decides WHERE chapter boundaries fall, so it
	// belongs here and not merely in the manifest key: without it a grammar edit would re-cut the book while
	// every unit id stood still. Never omitempty — the ladder always resolves to something, and "no grammar
	// declared" is an answer that has to be as visible as any other.
	Structure string `json:"structure_version"`
}

func (r *Runner) cutInputs() cutInputs {
	return cutInputs{
		Chunker: chunkerVersion, Segmentation: r.segmentationSnapshot(), ShippingWave: r.shippingWaveTag(),
		Langpack: r.packVersion(), Embedded: lang.EmbeddedVersion(), Norm: text.NormVersion(),
		Structure: r.structure.Fingerprint(),
	}
}

func (c cutInputs) tag() string {
	data, err := json.Marshal(c)
	if err != nil {
		panic(fmt.Sprintf("pipeline: manifest cut tag marshal: %v", err))
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:])[:8]
}

// shippingWaveTag names which wave owns the shipping stage. The UNIT decomposition depends on it
// (outputUnits groups whole chunks into edit units for an edit pipeline and emits one unit per chunk for
// a draft-only one), so it belongs to the cut for both the validity key and the unit id.
func (r *Runner) shippingWaveTag() string {
	if r.finalStageWave() == waveDraft {
		return "draft"
	}
	return "edit"
}

// sourceFingerprint is the source file's identity at one moment: the SHA-256 of its BYTES as they sit on
// disk — before decoding, before normalization — plus its size. Taken before AND after the split (see
// writeManifest), because a manifest whose STRUCTURE came from one version of the file and whose KEY came
// from another would validate forever while describing a book that no longer exists.
type sourceFingerprint struct {
	SHA   string
	Bytes int64
}

// sourceSHA256 fingerprints the source file. Streamed, so a 23 MB book costs a buffer and not a copy of
// itself (measured: ~20 ms on the acceptance book, against ~1.4 s for the ingest+split it replaces).
func sourceSHA256(path string) (sourceFingerprint, error) {
	f, err := os.Open(path)
	if err != nil {
		return sourceFingerprint{}, err
	}
	defer f.Close()
	h := sha256.New()
	n, err := io.Copy(h, f)
	if err != nil {
		return sourceFingerprint{}, err
	}
	return sourceFingerprint{SHA: hex.EncodeToString(h.Sum(nil)), Bytes: n}, nil
}

// sourceFingerprintBeforeIngest takes the fingerprint a manifest write must be built against, and returns
// nil (with a warning, never an error) when it cannot: the manifest is an accelerator, and a run must not
// die because its accelerator could not be stamped.
func (r *Runner) sourceFingerprintBeforeIngest(ctx context.Context) *sourceFingerprint {
	fp, err := sourceSHA256(r.Book.SourceFile)
	if err != nil {
		r.Log.WarnContext(ctx, "manifest: could not fingerprint the source before reading it; the chapter/chunk manifest will not be written this run",
			"source", r.Book.SourceFile, "err", err)
		return nil
	}
	return &fp
}

// manifestKey hashes everything whose change makes a stored manifest wrong.
//
// The list is the set of inputs the chunk manifest is a function of: the source bytes, how they are
// decoded (encoding) and which language's rules segment them, the chunker's own version, the
// segmentation budget (a re-chunk), the language pack (its heading rule decides both the rendered
// heading and, for a header-only chapter, whether the chapter survives at all) and the embedded language
// data (CJK numerals/terminators feed chapter and sentence boundaries; source abbreviations feed the
// sentence splitter). text.NormVersion rides too — normalization decides what "the chapter's text" even
// is, and the chapter ids are hashes of exactly that.
//
// INGEST CODE ITSELF is covered by convention rather than by a version of its own: the project already
// treats an ingest-rule change as a chunkerVersion bump ("This is an ingest rule → covered by
// chunkerVersion", text/source.go). That convention is inherited here, not invented — but it IS a
// convention, so an ingest edit that forgets the bump leaves a stale manifest looking current, exactly as
// it leaves stale checkpoints looking current today.
func (r *Runner) manifestKey(src sourceFingerprint) string {
	// The UNIT decomposition of the document is a function of the pipeline SHAPE, not only of the cut:
	// outputUnits groups whole chunks into edit units for an edit pipeline and emits one unit per chunk
	// for a draft-only one (waverun.go). Dropping the editor stage moves no other input here, so without
	// this the stored units[] would keep validating while describing groups the run no longer ships.
	payload := struct {
		Version      string           `json:"manifest_version"`
		BookID       string           `json:"book_id"`
		SourceSHA    string           `json:"source_sha256"`
		SourceBytes  int64            `json:"source_bytes"`
		Encoding     string           `json:"encoding"`
		SourceLang   string           `json:"source_lang"`
		TargetLang   string           `json:"target_lang"`
		Chunker      string           `json:"chunker_version"`
		Segmentation segmentationSnap `json:"segmentation"`
		Langpack     string           `json:"langpack_version,omitempty"`
		Embedded     string           `json:"embedded_version"`
		Norm         string           `json:"norm_version"`
		ShippingWave string           `json:"shipping_wave"`
		Structure    string           `json:"structure_version"`
	}{
		Version: manifestVersion, BookID: r.Book.BookID,
		SourceSHA: src.SHA, SourceBytes: src.Bytes,
		Encoding: r.Book.Encoding, SourceLang: r.Book.SourceLang, TargetLang: r.Book.TargetLang,
		Chunker: chunkerVersion, Segmentation: r.segmentationSnapshot(),
		Langpack: r.packVersion(), Embedded: lang.EmbeddedVersion(), Norm: text.NormVersion(),
		ShippingWave: r.shippingWaveTag(), Structure: r.structure.Fingerprint(),
	}
	data, err := json.Marshal(payload)
	if err != nil {
		// A struct of scalars and a fixed nested struct cannot fail to marshal; a key that could silently
		// become "" would make every manifest look valid, so refuse to produce one instead.
		panic(fmt.Sprintf("pipeline: manifest key marshal: %v", err))
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:])
}

// denseChapters is the chapters that CONSUMED a number, with their source titles. The two are held together
// rather than passed side by side because the pairing IS the invariant: selected once from the chunker's own
// kept indices, they cannot fall out of step. A title one chapter late is invisible — every chapter still has
// a title, just the wrong one.
type denseChapters struct {
	texts  []string
	titles []string
}

// denseFrom selects the chapters that took a number, using the chunker's OWN indices rather than re-deriving
// "an empty chapter takes no number" here.
func denseFrom(doc *chunk.Document, keptIdx []int) denseChapters {
	out := denseChapters{texts: make([]string, 0, len(keptIdx)), titles: make([]string, 0, len(keptIdx))}
	for _, i := range keptIdx {
		out.texts = append(out.texts, doc.Chapters[i])
		title := ""
		if i < len(doc.Titles) {
			title = doc.Titles[i]
		}
		out.titles = append(out.titles, title)
	}
	return out
}

// buildManifest projects a fresh split into the document. ch holds the chapters that took a number and
// their source titles, index-aligned to chapter
// numbers (chunk.SplitChunksWithChapters), chunks is the split itself. Pure and deterministic.
func (r *Runner) buildManifest(ch denseChapters, chunks []chunk.Chunk, structure string, src sourceFingerprint) *BookManifest {
	m := &BookManifest{
		Version: manifestVersion, BookID: r.Book.BookID,
		Key:            r.manifestKey(src),
		ChunkerVersion: chunkerVersion,
		SourceSHA256:   src.SHA, SourceBytes: src.Bytes,
		SourceLang: r.Book.SourceLang, TargetLang: r.Book.TargetLang, Encoding: r.Book.Encoding,
		ChunksTotal: len(chunks), Artifacts: r.artifacts(), Structure: structure,
	}
	// The price projection is built from the SAME unit decomposition the ids below are built from, in the
	// same pass, so a unit and its price can never describe different text (priceprojection.go).
	plan := r.pricePlanFor()
	// Chapter ids first, so a unit can name its chapter's id. The occurrence counter makes duplicate
	// chapter texts distinguishable (manifestChapterID).
	occurrence := map[string]int{}
	ids := make([]string, len(ch.texts))
	for i, txt := range ch.texts {
		occurrence[txt]++
		ids[i] = manifestChapterID(txt, occurrence[txt])
	}
	// A chapter's title lives on its FIRST chunk only (chunker.go), so it is collected off the chunk list
	// rather than off the unit list — a unit is a group of chunks and knowing which of them is the
	// chapter's opener is the chunker's business, not this projection's.
	headings := map[int]string{}
	for _, c := range chunks {
		if c.Heading != "" {
			headings[c.Chapter] = c.Heading
		}
	}
	cut := r.cutTag()   // rides every unit id: a re-cut must mint new ones (manifestUnitID)
	at := map[int]int{} // chapter number → index into m.Chapters (units arrive in book order)
	units := r.outputUnits(chunks)
	// ONE pass produces every unit's price and the book's roll-up; the loop below only places them.
	unitPrices, bookPrice := r.projectBook(plan, units)
	for ui, u := range units {
		i, ok := at[u.Chapter]
		if !ok {
			// Chapter numbers are dense and index-aligned to chapterTexts by construction (the chunker
			// appends a text exactly when it consumes a number). If that ever stops holding, fall back to a
			// number-derived id and say so: an EMPTY id would make every unit id of every such chapter
			// collide on ":<idx>", which is worse than an id that is merely position-based.
			id := ""
			if k := u.Chapter - 1; k >= 0 && k < len(ids) {
				id = ids[k]
			} else {
				id = "n" + strconv.Itoa(u.Chapter)
				r.Log.Warn("manifest: no ingested text for a chapter the split emitted — its id falls back to the (unstable) chapter number",
					"book", r.Book.BookID, "chapter", u.Chapter, "chapter_texts", len(ids))
			}
			title := ""
			// The SAME guard the chapter-id lookup six lines above carries, and for the same reason: that
			// branch exists because dense 1-based numbering might one day stop holding, and it degrades with
			// a WARN. Without the lower bound this line turns that warned degradation into an index-out-of-
			// range panic on a paid run.
			if k := u.Chapter - 1; k >= 0 && k < len(ch.titles) {
				title = ch.titles[k]
			}
			m.Chapters = append(m.Chapters, ManifestChapter{ID: id, Number: u.Chapter, Heading: headings[u.Chapter], TitleRaw: title})
			i = len(m.Chapters) - 1
			at[u.Chapter] = i
		}
		c := &m.Chapters[i]
		c.UnitsTotal++
		c.ChunksTotal += len(u.Members)
		mu := ManifestUnit{
			ID:            manifestUnitID(c.ID, cut, u.FirstChunkIdx),
			FirstChunkIdx: u.FirstChunkIdx, ChunkCount: len(u.Members), EditUnitID: u.EditUnitID,
		}
		if unitPrices != nil {
			p := unitPrices[ui]
			mu.Price = &UnitPrice{SourceChars: p.SourceChars, SourceCharsDense: p.Dense, SourceCharsSparse: p.Sparse, ExpectedUSD: p.ExpectedUSD}
			if c.Price == nil {
				c.Price = &UnitPrice{}
			}
			c.Price.SourceChars += p.SourceChars
			c.Price.SourceCharsDense += p.Dense
			c.Price.SourceCharsSparse += p.Sparse
			c.Price.ExpectedUSD += p.ExpectedUSD
		}
		c.Units = append(c.Units, mu)
		m.UnitsTotal++
	}
	m.ChaptersTotal = len(m.Chapters)
	m.Price = bookPrice
	return m
}

// persistManifest writes the manifest for a fresh split. LOUD BUT NOT FATAL: the manifest is a read
// accelerator and a reader's tree, not an input to anything the run decides — failing a paid run over it
// would trade money for an artifact the next run rewrites. A reader that finds no manifest (or a stale
// one) falls back to the full re-chunk, so the failure degrades to the pre-existing behaviour.
func (r *Runner) persistManifest(ctx context.Context, before *sourceFingerprint, ch denseChapters, chunks []chunk.Chunk, structure string) {
	if before == nil {
		return // sourceFingerprintBeforeIngest already said why
	}
	if _, err := r.writeManifest(*before, ch, chunks, structure); err != nil {
		r.Log.WarnContext(ctx, "manifest: could not persist the chapter/chunk manifest; read paths fall back to re-chunking the source and the chapter tree may be stale",
			"path", r.manifestPath(), "err", err)
	}
}

// writeManifest is persistManifest's fallible half — separate so a caller that MUST know (the manifest
// command) can report the failure instead of logging it. It returns the document it wrote, so a caller
// that also wants to render it does not build and hash the book a second time.
//
// `before` is the fingerprint taken BEFORE the source was read, and the re-fingerprint here is what makes
// the validity key mean what it claims. Hashing only afterwards would stamp the structure derived from
// one version of the file with the identity of another: the ingest+split of a 23 MB book takes ~1.4 s,
// and a source that is rewritten inside that window (a re-upload, an operator's edit) would produce a
// document that describes the OLD cut, matches its own key forever, and can never be detected as stale.
// A source that moved under the read is therefore not written at all — the next write path produces a
// consistent one, and until then readers take the full re-chunk.
func (r *Runner) writeManifest(before sourceFingerprint, ch denseChapters, chunks []chunk.Chunk, structure string) (*BookManifest, error) {
	after, err := sourceSHA256(r.Book.SourceFile)
	if err != nil {
		return nil, fmt.Errorf("pipeline: re-hash source %s for the manifest: %w", r.Book.SourceFile, err)
	}
	if after != before {
		return nil, fmt.Errorf("pipeline: the source %s changed while it was being read (%.12s/%d bytes → %.12s/%d bytes): the manifest is NOT written, because a structure cut from the old bytes stamped with the new hash would validate forever",
			r.Book.SourceFile, before.SHA, before.Bytes, after.SHA, after.Bytes)
	}
	m := r.buildManifest(ch, chunks, structure, before)
	body, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return nil, fmt.Errorf("pipeline: marshal the manifest: %w", err)
	}
	if err := writeFileAtomic(r.manifestPath(), append(body, '\n')); err != nil {
		return nil, err
	}
	return m, nil
}

// BuildAndPersistManifest re-derives the split and persists the manifest ($0: ingest + chunker, no LLM,
// no provider key, no wave). It is the producer behind `tmctl manifest` — the command a caller
// runs to get a chapter tree for a book that has never been translated, which is a state the library has
// to be able to show and which no other engine surface produces.
func (r *Runner) BuildAndPersistManifest() (*BookManifest, error) {
	// Fingerprint BEFORE the read, so writeManifest can prove the structure and the identity came from the
	// same bytes. Here a fingerprint failure IS an error: producing the artifact is the command's job.
	before, err := sourceSHA256(r.Book.SourceFile)
	if err != nil {
		return nil, refuseSource(fmt.Errorf("pipeline: hash source %s for the manifest: %w", r.Book.SourceFile, err))
	}
	doc, err := r.ingestSource()
	if err != nil {
		return nil, err
	}
	chunks, _, keptIdx := chunk.SplitChunksWithChapters(doc.Chapters, r.segBudget(), r.chapterRule(), r.sentenceAbbrevs())
	if len(chunks) == 0 {
		return nil, sourceHasNoContent(fmt.Errorf("pipeline: source file %s produced no chunks after normalization", r.Book.SourceFile))
	}
	return r.writeManifest(before, denseFrom(doc, keptIdx), chunks, doc.Structure)
}

// ManifestPath exposes where the artifact lives, for a CLI that has to tell the operator.
func (r *Runner) ManifestPath() string { return r.manifestPath() }

// loadManifest returns the persisted manifest IF it is present and still describes this book, cut this
// way. Every other outcome — absent, unreadable, malformed, wrong shape, stale key — returns nil, and the
// caller re-chunks. A read failure is deliberately not an error to the caller: the manifest is an
// accelerator, and a broken accelerator must degrade to the slow path, never to a wrong answer.
func (r *Runner) loadManifest() *BookManifest {
	raw, err := os.ReadFile(r.manifestPath())
	if err != nil {
		if !errors.Is(err, fs.ErrNotExist) {
			r.Log.Warn("manifest: could not read the manifest; falling back to re-chunking the source", "path", r.manifestPath(), "err", err)
		}
		return nil
	}
	var m BookManifest
	if err := json.Unmarshal(raw, &m); err != nil {
		r.Log.Warn("manifest: the manifest is not readable JSON; falling back to re-chunking the source", "path", r.manifestPath(), "err", err)
		return nil
	}
	if m.Version != manifestVersion {
		r.Log.Warn("manifest: the manifest was written in another document version; falling back to re-chunking the source",
			"path", r.manifestPath(), "stored", m.Version, "want", manifestVersion)
		return nil
	}
	// Internal sanity BEFORE the key: the key is a stored string, so a hand-edited sidecar (they sit in the
	// directory operators open) keeps matching while its counters say anything at all. A negative capacity
	// panics `make`, and an absurd one allocates the machine out of memory — inside a $0 read command.
	if !m.selfConsistent() {
		r.Log.Warn("manifest: the manifest's own counters do not describe its own contents; falling back to re-chunking the source",
			"path", r.manifestPath(), "chapters_total", m.ChaptersTotal, "units_total", m.UnitsTotal, "chunks_total", m.ChunksTotal)
		return nil
	}
	src, err := sourceSHA256(r.Book.SourceFile)
	if err != nil {
		r.Log.Warn("manifest: could not hash the source to validate the manifest; falling back to re-chunking the source", "err", err)
		return nil
	}
	// The paths are the CURRENT truth about this invocation, not whatever the sidecar was written with:
	// the book's directory may have moved since. See BookManifest.Artifacts.
	m.Artifacts = r.artifacts()
	if want := r.manifestKey(src); m.Key != want {
		// This is the normal, expected path after a source edit or a chunker/pack/budget change — say what
		// happened at INFO-level volume rather than as a fault, but say it: a chapter tree whose ids just
		// changed is a fact a reader has to be told about (a re-anchor, not a refresh).
		r.Log.Info("manifest: the stored manifest is stale (source or cut changed); re-chunking the source and rebuilding it on the next write path",
			"path", r.manifestPath(), "stored_key", m.Key, "current_key", want)
		return nil
	}
	return &m
}

// selfConsistent reports whether the document's own counters describe its own contents. It is a guard
// against a HAND-EDITED file, not against a corrupt one (a corrupt one fails the JSON parse or the key):
// the counters are read straight into slice capacities and loop bounds below, so a negative or absurd
// number would panic or exhaust memory inside a read-only command.
func (m *BookManifest) selfConsistent() bool {
	if m.ChaptersTotal != len(m.Chapters) {
		return false
	}
	units, chunks := 0, 0
	for _, c := range m.Chapters {
		if c.Number < 1 || c.UnitsTotal != len(c.Units) {
			return false
		}
		for _, u := range c.Units {
			if u.ChunkCount < 1 || u.ChunkCount > manifestMaxChunks || u.FirstChunkIdx < 0 {
				return false
			}
			chunks += u.ChunkCount
		}
		units += len(c.Units)
	}
	// The UPPER bound matters as much as the lower one: a file edited so that ChunksTotal and the per-unit
	// counts agree at an absurd value passes every equality above and then allocates that many chunks.
	if chunks > manifestMaxChunks {
		return false
	}
	return m.UnitsTotal == units && m.ChunksTotal == chunks
}

// manifestMaxChunks bounds what a manifest may claim to describe. It is a sanity ceiling, not a product
// limit: the 23 MB acceptance book cuts into ~5 000 chunks, so this is three orders of magnitude above
// any real book and far below the value at which the reconstruction would exhaust memory.
const manifestMaxChunks = 10_000_000

// chunks reconstructs the manifest's chunk list. Safe to call only on a document that passed
// selfConsistent (loadManifest is the only producer).
//
// ⚠ THE CHUNKS CARRY NO SOURCE TEXT. The manifest stores STRUCTURE — a 23 MB book's text is the source
// file, and duplicating it into a sidecar would trade the CPU this file saves for the same amount of
// disk and write time. Every field the read models actually join on is here (chapter, chunk index, edit
// unit, the chapter title). Three read-side consumers genuinely need the text and therefore take the full
// re-chunk instead: `export --pairs` (it emits the source column), the re-bill projection's content
// re-render (it re-renders injected bytes), and `QualityReport` (`quality.go` — its residual scan is a
// comparison against the unit's source, so `tmctl report` still re-cuts the book).
func (m *BookManifest) chunks() []chunk.Chunk {
	out := make([]chunk.Chunk, 0, m.ChunksTotal)
	for _, c := range m.Chapters {
		for _, u := range c.Units {
			for i := 0; i < u.ChunkCount; i++ {
				ch := chunk.Chunk{
					Chapter: c.Number, ChunkIdx: u.FirstChunkIdx + i, EditUnitID: u.EditUnitID,
				}
				if ch.ChunkIdx == 0 {
					ch.Heading = c.Heading // only a chapter's first chunk carries the title (chunker.go)
				}
				out = append(out, ch)
			}
		}
	}
	return out
}

// readModelChunks is what the $0 read models join against: the persisted manifest when it is current, the
// full ingest+split otherwise.
//
// The second result is a PROVIDER of the same chunks WITH their source text, not a flag — a flag that
// every caller discarded would document a guard the code does not keep. On the manifest path it is the
// full re-chunk (paid only if some branch actually needs text); on the fallback path it hands back the
// slice that was just materialized, so a read that already re-cut the book never re-cuts it twice.
func (r *Runner) readModelChunks() (chunks []chunk.Chunk, withText func() ([]chunk.Chunk, error), err error) {
	if m := r.loadManifest(); m != nil {
		return m.chunks(), r.bookChunks, nil
	}
	full, err := r.bookChunks()
	return full, func() ([]chunk.Chunk, error) { return full, err }, err
}

// readModelPrice is the a-priori price and the cut's provenance for a $0 read path — from the persisted
// sidecar when it can answer, and from the cut this read has already made when it cannot.
//
// ⚠ «CAN ANSWER» IS THREE STATES, NOT TWO, and the two-state wording this comment used to carry is what
// let the third slip past: a sidecar is absent, or stale, or CURRENT AND OLDER THAN THESE FIELDS. The
// third reads as healthy to every check the loader makes — see the body.
//
// ⛔ THE FALLBACK IS NOT AN EXTRA, IT IS THE INVARIANT. `status` must project the same report whichever
// path served it, and the manifest is an ACCELERATOR: an accelerator that changes the answer is a second
// source of truth. Without this branch a book whose sidecar cannot answer reports no price at all —
// silently, and precisely on the surface a buyer's platform reads before deciding — while the same book
// with an answering sidecar reports one. Both branches end in projectBook, so there is one derivation and
// not two that agree.
//
// The cut's provenance comes with the cut: `structure` is a statement about how the boundaries were
// drawn, which only the ingest knows, so the fallback takes it from the ingest that just ran rather than
// trying to recover it from chunks that no longer remember.
func (r *Runner) readModelPrice(withText func() ([]chunk.Chunk, error)) (*BookPrice, string) {
	// ⛔ A CURRENT SIDECAR CAN STILL PREDATE THESE FIELDS, and that is a THIRD state, not a shade of the
	// other two. `price` and `structure` are ADDITIVE, so the document version deliberately did not move
	// for them (moving it would discard every stored sidecar and re-cut every book — see manifestVersion).
	// The consequence is that a file written by an older build passes the version, passes selfConsistent
	// and passes the validity key, comes back as «current», and carries no price at all — so the branch
	// above returned nil and the fallback below, which this comment calls the invariant, never ran. The
	// book then reported no price SILENTLY, on the surface a buyer's platform reads before deciding,
	// which is precisely the failure the fallback exists to prevent — found by acceptance (V2-3).
	//
	// So the liveness of an additive field is asked ABOUT THE FIELD, not about the document: a sidecar
	// that cannot answer this question is, for this question, no sidecar.
	if m := r.loadManifest(); m != nil && m.Price != nil && m.Structure != "" {
		return m.Price, m.Structure
	}
	full, err := withText()
	if err != nil || len(full) == 0 {
		return nil, ""
	}
	_, bp := r.projectBook(r.pricePlanFor(), r.outputUnits(full))
	return bp, r.cutStructure
}