textmachine/backend/internal/pipeline/manifest.go

package pipeline

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"strconv"

	"textmachine/backend/internal/chunk"
	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/text"
)

// manifest.go: the PERSISTED chapter/chunk manifest (backlog row 100).
//
// Two problems, one artifact. (1) Every $0 read re-ingested and re-cut the whole source to get its
// honest N/M denominator — 1.4–1.5 s of CPU per call on a 23 MB book, paid again on every poll of every
// book. (2) The chapter tree a reader needs has no carrier at all: the engine addresses chapters by a
// DENSE ordinal (chapters that yield no text do not consume a number, chunker.go), so editing the source
// renumbers every later chapter and any bookmark built on the number silently moves to another chapter.
//
// The artifact is a FILE beside the project DB, not a table, for the same reason the bank export is one
// (bankexport.go): the reader is another process which must not open the engine's private store
// (D39.85). It is also a projection and never a source — nothing reads it back into a decision the
// engine makes; a stale one is DETECTED and ignored, never trusted.
//
// STALENESS is decided by a KEY, not by the mtime and not by a flag. Everything whose change moves the
// chunk boundaries or the chapter set is hashed into it, so a manifest written under a different chunker,
// budget, langpack, embedded data set, encoding or source file cannot be mistaken for a current one. A
// write path rebuilds it unconditionally (so `translate --resnapshot`, which is the loud "the chunker
// moved" command, rebuilds it like every other run); a read path that finds a key mismatch falls back to
// the full re-chunk it did before this file existed. There is no state in which a wrong manifest is used.

// manifestVersion versions the DOCUMENT SHAPE. It also rides the key, so a shape change invalidates every
// stored manifest rather than being half-read by a binary expecting the other layout.
//
// v2 (acceptance V2-4): unit ids gained the cut tag — «<chapter>:<cutTag>:<idx>» instead of
// «<chapter>:<idx>». A v1 sidecar is structurally valid and would keep validating, handing a reader ids
// in the OLD form whose whole defect was that they outlive the cut. The bump is what refuses it.
const manifestVersion = "tm-manifest-v2"

// BookManifest is the persisted chapter/chunk structure of a book.
type BookManifest struct {
	Version string `json:"manifest_version"`
	BookID  string `json:"book_id"`
	// Key is the validity key (manifestKey). A reader that only wants "is this still the same book, cut
	// the same way" compares this one string.
	Key string `json:"key"`
	// The key's INPUTS, spelled out beside it. The key alone answers "is it current"; these answer "what
	// changed" when it is not, which is the question an operator staring at an invalidated tree asks.
	ChunkerVersion string `json:"chunker_version"`
	SourceSHA256   string `json:"source_sha256"`
	SourceBytes    int64  `json:"source_bytes"`
	SourceLang     string `json:"source_lang"`
	TargetLang     string `json:"target_lang"`
	Encoding       string `json:"encoding"`

	Chapters      []ManifestChapter `json:"chapters"`
	ChaptersTotal int               `json:"chapters_total"`
	UnitsTotal    int               `json:"units_total"`
	ChunksTotal   int               `json:"chunks_total"`
}

// ManifestChapter is one chapter: its stable identity, its DISPLAY ordinal, and its units.
type ManifestChapter struct {
	// ID is the engine's STABLE key for this chapter: 16 hex chars of SHA-256 over the chapter's INGESTED
	// text (see manifestChapterID). It is stable across a chunker/budget/heading-rule change and across
	// edits to OTHER chapters — which is the whole point, since Number is not: numbering is dense, so
	// inserting a chapter shifts every later ordinal while every later id stays put.
	ID string `json:"id"`
	// Number is the dense 1-based ordinal the engine addresses chunk_status rows by. It is a POSITION,
	// not a key — see ID.
	Number int `json:"number"`
	// Heading is the title the engine RENDERS for this chapter from the pair's heading rule («Глава N»),
	// or "" when the book has no rule or the chapter has no structural header. ⚠ It is the engine's own
	// deterministic render — the source marker is stripped from the text the model sees and this string is
	// glued back onto the chapter's first output unit at export (chunk.ApplyHeading). It is NOT a label
	// carried by the book's own data, and a consumer must not present it as one. Which label a reader
	// should see is an open contract question (companion §4, K-2/K-3), not settled here.
	Heading     string         `json:"heading"`
	UnitsTotal  int            `json:"units_total"`
	ChunksTotal int            `json:"chunks_total"`
	Units       []ManifestUnit `json:"units"`
}

// ManifestUnit is one OUTPUT unit — the granularity the engine ships and every read model counts in.
type ManifestUnit struct {
	// ID is stable only as long as the CUT is, and it SAYS SO in its own bytes: it is the chapter id, the
	// cut tag, and the unit's leader chunk index (manifestUnitID). A chunker/budget/pipeline-shape change
	// re-cuts the chapter into different units, mints a new cut tag, and therefore mints a new id for
	// EVERY unit of the book — including a chapter's first unit, whose leader index is 0 under any cut and
	// which would otherwise carry an id that outlives the text it named.
	//
	// So: chapter identity survives a re-chunk, unit identity does not, and a consumer can tell the two
	// apart by whether the id it holds still appears in the current manifest.
	ID string `json:"id"`
	// FirstChunkIdx is the LEADER chunk index — the (chapter, chunk_idx) the unit's shipping row is
	// addressed at in chunk_status, and therefore the join key for every progress read.
	FirstChunkIdx int `json:"first_chunk_idx"`
	// ChunkCount is how many draft chunks the unit groups; members are the CONSECUTIVE indices
	// [FirstChunkIdx, FirstChunkIdx+ChunkCount) of this chapter (chunk.assignEditUnits groups consecutive
	// chunks, which is what makes storing a count instead of a list lossless).
	ChunkCount int `json:"chunk_count"`
	// EditUnitID is the engine's book-global edit-unit id for this unit.
	EditUnitID int `json:"edit_unit_id"`
}

// manifestPath is where the manifest lands: beside the project DB, like every other book-state sidecar.
func (r *Runner) manifestPath() string { return r.Book.ProjectDB + ".manifest.json" }

// manifestChapterID is a chapter's stable identity: the first 64 bits of SHA-256 over its INGESTED text.
//
// WHY CONTENT AND NOT POSITION. The alternative — hashing the book id and the ordinal — is the ordinal
// with extra steps: inserting a chapter would re-point every later id at a different chapter's content,
// which is precisely the "open tabs and bookmarks quietly move" damage the contract companion names (§5).
// Content-derived, an edit to chapter 5 mints a new id for chapter 5 alone and every other chapter keeps
// the id it had, whatever its number became.
//
// DUPLICATES: two chapters with byte-identical text would hash the same, so the second and later
// occurrences carry an occurrence suffix. Deterministic for a given reading order, and honest about its
// own edge — deleting the FIRST of a duplicate pair promotes the second, changing its id.
func manifestChapterID(ingested string, occurrence int) string {
	sum := sha256.Sum256([]byte(ingested))
	id := hex.EncodeToString(sum[:])[:16]
	if occurrence > 1 {
		id += "-" + strconv.Itoa(occurrence)
	}
	return id
}

// manifestUnitID is a unit's identity: its chapter's id, the CUT it belongs to, and the leader chunk
// index.
//
// The cut tag is load-bearing and was missing in the first version, which made the declared boundary
// FALSE rather than merely coarse. Without it a unit id is «<chapter>:<firstChunkIdx>», and a chapter's
// FIRST unit is always index 0 — so «<chapter>:0» survived every re-chunk unchanged while pointing at a
// different span of text. A consumer that persists unit ids reads "the id is still there" as "my anchor
// is still valid", and lands on the wrong text with no signal. With the tag, ANY re-cut mints new unit
// ids across the whole book, which is exactly what ManifestUnit.ID's boundary says it does.
func manifestUnitID(chapterID, cutTag string, firstChunkIdx int) string {
	return chapterID + ":" + cutTag + ":" + strconv.Itoa(firstChunkIdx)
}

// cutTag identifies the CUT — everything that decides where chunk boundaries fall and how chunks group
// into output units. It is a SHORT hash because it rides inside every unit id; the full identity of the
// manifest is the validity key, which folds these same inputs plus the source bytes and the book identity.
//
// The last three are the acceptance V2-3 catch and are the same class as the defect the tag exists to
// close: the langpack decides the heading rule (a header-only chapter can stop consuming a number), the
// embedded data feeds the CJK numerals and sentence terminators that place boundaries, and the
// normalization decides what the chunker is even splitting. Any of them moving re-cuts the book while
// chunker_version and the budget stand still — and a manifest rebuilt afterwards would stamp the same
// unit ids onto different text.
func (r *Runner) cutTag() string {
	payload := struct {
		Chunker      string           `json:"chunker_version"`
		Segmentation segmentationSnap `json:"segmentation"`
		ShippingWave string           `json:"shipping_wave"`
		Langpack     string           `json:"langpack_version,omitempty"`
		Embedded     string           `json:"embedded_version"`
		Norm         string           `json:"norm_version"`
	}{chunkerVersion, r.segmentationSnapshot(), r.shippingWaveTag(),
		r.packVersion(), lang.EmbeddedVersion(), text.NormVersion()}
	data, err := json.Marshal(payload)
	if err != nil {
		panic(fmt.Sprintf("pipeline: manifest cut tag marshal: %v", err))
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:])[:8]
}

// shippingWaveTag names which wave owns the shipping stage. The UNIT decomposition depends on it
// (outputUnits groups whole chunks into edit units for an edit pipeline and emits one unit per chunk for
// a draft-only one), so it belongs to the cut for both the validity key and the unit id.
func (r *Runner) shippingWaveTag() string {
	if r.finalStageWave() == waveDraft {
		return "draft"
	}
	return "edit"
}

// sourceFingerprint is the source file's identity at one moment: the SHA-256 of its BYTES as they sit on
// disk — before decoding, before normalization — plus its size. Taken before AND after the split (see
// writeManifest), because a manifest whose STRUCTURE came from one version of the file and whose KEY came
// from another would validate forever while describing a book that no longer exists.
type sourceFingerprint struct {
	SHA   string
	Bytes int64
}

// sourceSHA256 fingerprints the source file. Streamed, so a 23 MB book costs a buffer and not a copy of
// itself (measured: ~20 ms on the acceptance book, against ~1.4 s for the ingest+split it replaces).
func sourceSHA256(path string) (sourceFingerprint, error) {
	f, err := os.Open(path)
	if err != nil {
		return sourceFingerprint{}, err
	}
	defer f.Close()
	h := sha256.New()
	n, err := io.Copy(h, f)
	if err != nil {
		return sourceFingerprint{}, err
	}
	return sourceFingerprint{SHA: hex.EncodeToString(h.Sum(nil)), Bytes: n}, nil
}

// sourceFingerprintBeforeIngest takes the fingerprint a manifest write must be built against, and returns
// nil (with a warning, never an error) when it cannot: the manifest is an accelerator, and a run must not
// die because its accelerator could not be stamped.
func (r *Runner) sourceFingerprintBeforeIngest(ctx context.Context) *sourceFingerprint {
	fp, err := sourceSHA256(r.Book.SourceFile)
	if err != nil {
		r.Log.WarnContext(ctx, "manifest: could not fingerprint the source before reading it; the chapter/chunk manifest will not be written this run",
			"source", r.Book.SourceFile, "err", err)
		return nil
	}
	return &fp
}

// manifestKey hashes everything whose change makes a stored manifest wrong.
//
// The list is the set of inputs the chunk manifest is a function of: the source bytes, how they are
// decoded (encoding) and which language's rules segment them, the chunker's own version, the
// segmentation budget (a re-chunk), the language pack (its heading rule decides both the rendered
// heading and, for a header-only chapter, whether the chapter survives at all) and the embedded language
// data (CJK numerals/terminators feed chapter and sentence boundaries; source abbreviations feed the
// sentence splitter). text.NormVersion rides too — normalization decides what "the chapter's text" even
// is, and the chapter ids are hashes of exactly that.
//
// INGEST CODE ITSELF is covered by convention rather than by a version of its own: the project already
// treats an ingest-rule change as a chunkerVersion bump ("This is an ingest rule → covered by
// chunkerVersion", text/source.go). That convention is inherited here, not invented — but it IS a
// convention, so an ingest edit that forgets the bump leaves a stale manifest looking current, exactly as
// it leaves stale checkpoints looking current today.
func (r *Runner) manifestKey(src sourceFingerprint) string {
	// The UNIT decomposition of the document is a function of the pipeline SHAPE, not only of the cut:
	// outputUnits groups whole chunks into edit units for an edit pipeline and emits one unit per chunk
	// for a draft-only one (waverun.go). Dropping the editor stage moves no other input here, so without
	// this the stored units[] would keep validating while describing groups the run no longer ships.
	payload := struct {
		Version      string           `json:"manifest_version"`
		BookID       string           `json:"book_id"`
		SourceSHA    string           `json:"source_sha256"`
		SourceBytes  int64            `json:"source_bytes"`
		Encoding     string           `json:"encoding"`
		SourceLang   string           `json:"source_lang"`
		TargetLang   string           `json:"target_lang"`
		Chunker      string           `json:"chunker_version"`
		Segmentation segmentationSnap `json:"segmentation"`
		Langpack     string           `json:"langpack_version,omitempty"`
		Embedded     string           `json:"embedded_version"`
		Norm         string           `json:"norm_version"`
		ShippingWave string           `json:"shipping_wave"`
	}{
		Version: manifestVersion, BookID: r.Book.BookID,
		SourceSHA: src.SHA, SourceBytes: src.Bytes,
		Encoding: r.Book.Encoding, SourceLang: r.Book.SourceLang, TargetLang: r.Book.TargetLang,
		Chunker: chunkerVersion, Segmentation: r.segmentationSnapshot(),
		Langpack: r.packVersion(), Embedded: lang.EmbeddedVersion(), Norm: text.NormVersion(),
		ShippingWave: r.shippingWaveTag(),
	}
	data, err := json.Marshal(payload)
	if err != nil {
		// A struct of scalars and a fixed nested struct cannot fail to marshal; a key that could silently
		// become "" would make every manifest look valid, so refuse to produce one instead.
		panic(fmt.Sprintf("pipeline: manifest key marshal: %v", err))
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:])
}

// buildManifest projects a fresh split into the document. chapterTexts is index-aligned to chapter
// numbers (chunk.SplitChunksWithChapters), chunks is the split itself. Pure and deterministic.
func (r *Runner) buildManifest(chapterTexts []string, chunks []chunk.Chunk, src sourceFingerprint) *BookManifest {
	m := &BookManifest{
		Version: manifestVersion, BookID: r.Book.BookID,
		Key:            r.manifestKey(src),
		ChunkerVersion: chunkerVersion,
		SourceSHA256:   src.SHA, SourceBytes: src.Bytes,
		SourceLang: r.Book.SourceLang, TargetLang: r.Book.TargetLang, Encoding: r.Book.Encoding,
		ChunksTotal: len(chunks),
	}
	// Chapter ids first, so a unit can name its chapter's id. The occurrence counter makes duplicate
	// chapter texts distinguishable (manifestChapterID).
	occurrence := map[string]int{}
	ids := make([]string, len(chapterTexts))
	for i, txt := range chapterTexts {
		occurrence[txt]++
		ids[i] = manifestChapterID(txt, occurrence[txt])
	}
	// A chapter's title lives on its FIRST chunk only (chunker.go), so it is collected off the chunk list
	// rather than off the unit list — a unit is a group of chunks and knowing which of them is the
	// chapter's opener is the chunker's business, not this projection's.
	headings := map[int]string{}
	for _, c := range chunks {
		if c.Heading != "" {
			headings[c.Chapter] = c.Heading
		}
	}
	cut := r.cutTag()   // rides every unit id: a re-cut must mint new ones (manifestUnitID)
	at := map[int]int{} // chapter number → index into m.Chapters (units arrive in book order)
	for _, u := range r.outputUnits(chunks) {
		i, ok := at[u.Chapter]
		if !ok {
			// Chapter numbers are dense and index-aligned to chapterTexts by construction (the chunker
			// appends a text exactly when it consumes a number). If that ever stops holding, fall back to a
			// number-derived id and say so: an EMPTY id would make every unit id of every such chapter
			// collide on ":<idx>", which is worse than an id that is merely position-based.
			id := ""
			if k := u.Chapter - 1; k >= 0 && k < len(ids) {
				id = ids[k]
			} else {
				id = "n" + strconv.Itoa(u.Chapter)
				r.Log.Warn("manifest: no ingested text for a chapter the split emitted — its id falls back to the (unstable) chapter number",
					"book", r.Book.BookID, "chapter", u.Chapter, "chapter_texts", len(ids))
			}
			m.Chapters = append(m.Chapters, ManifestChapter{ID: id, Number: u.Chapter, Heading: headings[u.Chapter]})
			i = len(m.Chapters) - 1
			at[u.Chapter] = i
		}
		c := &m.Chapters[i]
		c.UnitsTotal++
		c.ChunksTotal += len(u.Members)
		c.Units = append(c.Units, ManifestUnit{
			ID:            manifestUnitID(c.ID, cut, u.FirstChunkIdx),
			FirstChunkIdx: u.FirstChunkIdx, ChunkCount: len(u.Members), EditUnitID: u.EditUnitID,
		})
		m.UnitsTotal++
	}
	m.ChaptersTotal = len(m.Chapters)
	return m
}

// persistManifest writes the manifest for a fresh split. LOUD BUT NOT FATAL: the manifest is a read
// accelerator and a reader's tree, not an input to anything the run decides — failing a paid run over it
// would trade money for an artifact the next run rewrites. A reader that finds no manifest (or a stale
// one) falls back to the full re-chunk, so the failure degrades to the pre-existing behaviour.
func (r *Runner) persistManifest(ctx context.Context, before *sourceFingerprint, chapterTexts []string, chunks []chunk.Chunk) {
	if before == nil {
		return // sourceFingerprintBeforeIngest already said why
	}
	if _, err := r.writeManifest(*before, chapterTexts, chunks); err != nil {
		r.Log.WarnContext(ctx, "manifest: could not persist the chapter/chunk manifest; read paths fall back to re-chunking the source and the chapter tree may be stale",
			"path", r.manifestPath(), "err", err)
	}
}

// writeManifest is persistManifest's fallible half — separate so a caller that MUST know (the manifest
// command) can report the failure instead of logging it. It returns the document it wrote, so a caller
// that also wants to render it does not build and hash the book a second time.
//
// `before` is the fingerprint taken BEFORE the source was read, and the re-fingerprint here is what makes
// the validity key mean what it claims. Hashing only afterwards would stamp the structure derived from
// one version of the file with the identity of another: the ingest+split of a 23 MB book takes ~1.4 s,
// and a source that is rewritten inside that window (a re-upload, an operator's edit) would produce a
// document that describes the OLD cut, matches its own key forever, and can never be detected as stale.
// A source that moved under the read is therefore not written at all — the next write path produces a
// consistent one, and until then readers take the full re-chunk.
func (r *Runner) writeManifest(before sourceFingerprint, chapterTexts []string, chunks []chunk.Chunk) (*BookManifest, error) {
	after, err := sourceSHA256(r.Book.SourceFile)
	if err != nil {
		return nil, fmt.Errorf("pipeline: re-hash source %s for the manifest: %w", r.Book.SourceFile, err)
	}
	if after != before {
		return nil, fmt.Errorf("pipeline: the source %s changed while it was being read (%.12s/%d bytes → %.12s/%d bytes): the manifest is NOT written, because a structure cut from the old bytes stamped with the new hash would validate forever",
			r.Book.SourceFile, before.SHA, before.Bytes, after.SHA, after.Bytes)
	}
	m := r.buildManifest(chapterTexts, chunks, before)
	body, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return nil, fmt.Errorf("pipeline: marshal the manifest: %w", err)
	}
	if err := writeFileAtomic(r.manifestPath(), append(body, '\n')); err != nil {
		return nil, err
	}
	return m, nil
}

// BuildAndPersistManifest re-derives the split and persists the manifest ($0: ingest + chunker, no LLM,
// no provider key, no wave). It is the producer behind `tmctl manifest` — the command a caller
// runs to get a chapter tree for a book that has never been translated, which is a state the library has
// to be able to show and which no other engine surface produces.
func (r *Runner) BuildAndPersistManifest() (*BookManifest, error) {
	// Fingerprint BEFORE the read, so writeManifest can prove the structure and the identity came from the
	// same bytes. Here a fingerprint failure IS an error: producing the artifact is the command's job.
	before, err := sourceSHA256(r.Book.SourceFile)
	if err != nil {
		return nil, refuseSource(fmt.Errorf("pipeline: hash source %s for the manifest: %w", r.Book.SourceFile, err))
	}
	doc, err := r.ingestSource()
	if err != nil {
		return nil, err
	}
	chunks, chapterTexts := chunk.SplitChunksWithChapters(doc.Chapters, r.segBudget(), r.headingRule(), r.sentenceAbbrevs())
	if len(chunks) == 0 {
		return nil, sourceHasNoContent(fmt.Errorf("pipeline: source file %s produced no chunks after normalization", r.Book.SourceFile))
	}
	return r.writeManifest(before, chapterTexts, chunks)
}

// ManifestPath exposes where the artifact lives, for a CLI that has to tell the operator.
func (r *Runner) ManifestPath() string { return r.manifestPath() }

// loadManifest returns the persisted manifest IF it is present and still describes this book, cut this
// way. Every other outcome — absent, unreadable, malformed, wrong shape, stale key — returns nil, and the
// caller re-chunks. A read failure is deliberately not an error to the caller: the manifest is an
// accelerator, and a broken accelerator must degrade to the slow path, never to a wrong answer.
func (r *Runner) loadManifest() *BookManifest {
	raw, err := os.ReadFile(r.manifestPath())
	if err != nil {
		if !errors.Is(err, fs.ErrNotExist) {
			r.Log.Warn("manifest: could not read the manifest; falling back to re-chunking the source", "path", r.manifestPath(), "err", err)
		}
		return nil
	}
	var m BookManifest
	if err := json.Unmarshal(raw, &m); err != nil {
		r.Log.Warn("manifest: the manifest is not readable JSON; falling back to re-chunking the source", "path", r.manifestPath(), "err", err)
		return nil
	}
	if m.Version != manifestVersion {
		r.Log.Warn("manifest: the manifest was written in another document version; falling back to re-chunking the source",
			"path", r.manifestPath(), "stored", m.Version, "want", manifestVersion)
		return nil
	}
	// Internal sanity BEFORE the key: the key is a stored string, so a hand-edited sidecar (they sit in the
	// directory operators open) keeps matching while its counters say anything at all. A negative capacity
	// panics `make`, and an absurd one allocates the machine out of memory — inside a $0 read command.
	if !m.selfConsistent() {
		r.Log.Warn("manifest: the manifest's own counters do not describe its own contents; falling back to re-chunking the source",
			"path", r.manifestPath(), "chapters_total", m.ChaptersTotal, "units_total", m.UnitsTotal, "chunks_total", m.ChunksTotal)
		return nil
	}
	src, err := sourceSHA256(r.Book.SourceFile)
	if err != nil {
		r.Log.Warn("manifest: could not hash the source to validate the manifest; falling back to re-chunking the source", "err", err)
		return nil
	}
	if want := r.manifestKey(src); m.Key != want {
		// This is the normal, expected path after a source edit or a chunker/pack/budget change — say what
		// happened at INFO-level volume rather than as a fault, but say it: a chapter tree whose ids just
		// changed is a fact a reader has to be told about (a re-anchor, not a refresh).
		r.Log.Info("manifest: the stored manifest is stale (source or cut changed); re-chunking the source and rebuilding it on the next write path",
			"path", r.manifestPath(), "stored_key", m.Key, "current_key", want)
		return nil
	}
	return &m
}

// selfConsistent reports whether the document's own counters describe its own contents. It is a guard
// against a HAND-EDITED file, not against a corrupt one (a corrupt one fails the JSON parse or the key):
// the counters are read straight into slice capacities and loop bounds below, so a negative or absurd
// number would panic or exhaust memory inside a read-only command.
func (m *BookManifest) selfConsistent() bool {
	if m.ChaptersTotal != len(m.Chapters) {
		return false
	}
	units, chunks := 0, 0
	for _, c := range m.Chapters {
		if c.Number < 1 || c.UnitsTotal != len(c.Units) {
			return false
		}
		for _, u := range c.Units {
			if u.ChunkCount < 1 || u.ChunkCount > manifestMaxChunks || u.FirstChunkIdx < 0 {
				return false
			}
			chunks += u.ChunkCount
		}
		units += len(c.Units)
	}
	// The UPPER bound matters as much as the lower one: a file edited so that ChunksTotal and the per-unit
	// counts agree at an absurd value passes every equality above and then allocates that many chunks.
	if chunks > manifestMaxChunks {
		return false
	}
	return m.UnitsTotal == units && m.ChunksTotal == chunks
}

// manifestMaxChunks bounds what a manifest may claim to describe. It is a sanity ceiling, not a product
// limit: the 23 MB acceptance book cuts into ~5 000 chunks, so this is three orders of magnitude above
// any real book and far below the value at which the reconstruction would exhaust memory.
const manifestMaxChunks = 10_000_000

// chunks reconstructs the manifest's chunk list. Safe to call only on a document that passed
// selfConsistent (loadManifest is the only producer).
//
// ⚠ THE CHUNKS CARRY NO SOURCE TEXT. The manifest stores STRUCTURE — a 23 MB book's text is the source
// file, and duplicating it into a sidecar would trade the CPU this file saves for the same amount of
// disk and write time. Every field the read models actually join on is here (chapter, chunk index, edit
// unit, the chapter title). Three read-side consumers genuinely need the text and therefore take the full
// re-chunk instead: `export --pairs` (it emits the source column), the re-bill projection's content
// re-render (it re-renders injected bytes), and `QualityReport` (`quality.go` — its residual scan is a
// comparison against the unit's source, so `tmctl report` still re-cuts the book).
func (m *BookManifest) chunks() []chunk.Chunk {
	out := make([]chunk.Chunk, 0, m.ChunksTotal)
	for _, c := range m.Chapters {
		for _, u := range c.Units {
			for i := 0; i < u.ChunkCount; i++ {
				ch := chunk.Chunk{
					Chapter: c.Number, ChunkIdx: u.FirstChunkIdx + i, EditUnitID: u.EditUnitID,
				}
				if ch.ChunkIdx == 0 {
					ch.Heading = c.Heading // only a chapter's first chunk carries the title (chunker.go)
				}
				out = append(out, ch)
			}
		}
	}
	return out
}

// readModelChunks is what the $0 read models join against: the persisted manifest when it is current, the
// full ingest+split otherwise.
//
// The second result is a PROVIDER of the same chunks WITH their source text, not a flag — a flag that
// every caller discarded would document a guard the code does not keep. On the manifest path it is the
// full re-chunk (paid only if some branch actually needs text); on the fallback path it hands back the
// slice that was just materialized, so a read that already re-cut the book never re-cuts it twice.
func (r *Runner) readModelChunks() (chunks []chunk.Chunk, withText func() ([]chunk.Chunk, error), err error) {
	if m := r.loadManifest(); m != nil {
		return m.chunks(), r.bookChunks, nil
	}
	full, err := r.bookChunks()
	return full, func() ([]chunk.Chunk, error) { return full, err }, err
}