textmachine/backend/internal/pipeline/bookbuild.go

package pipeline

import (
	"bytes"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"textmachine/backend/internal/bookfile"
	"textmachine/backend/internal/lang"
)

// bookbuild.go: the translated book as a FILE — the `tmctl build` half of backlog row 236. It turns the
// export projection (export.go) into the reader's copy (internal/bookfile: EPUB 3 and plain text), and
// its whole job beyond that is honesty: a copy of an unfinished book must never look finished.
//
// SIX states the file may not lie about. Three are BookExport's counters (pending, ghost rows, config
// drift); two are per unit and read through UnitHole (withheld, incomplete — holes.go); the sixth is the
// writer's own, read from the store: STALE — a unit whose final row was made for a source that is no
// longer the book's (staleUnits). The rule:
//
//   - pending · withheld · incomplete · stale · ghost are HOLES. By default a book with any hole is
//     REFUSED (RefusalBookIncomplete, listing every hole) — a reader-facing copy is fail-closed
//     (D29.1(б)). With --partial the same book is written with a NOTICE a reader meets on opening it (the
//     first blocks of the first spine document, and dc:description) and a MARK at the place of every
//     hole: instead of the text for a pending/withheld/stale unit, ahead of the text for an incomplete
//     unit, at the end of a chapter for ghost rows (their place inside the chapter is not known). Both
//     behaviours exist because the contract says an unfinished book may be exported and leaves what such
//     a file contains to the engine (14 §createExport); which is the default is this file's decision.
//   - ghost rows are a hole and not a footnote: when the cut coarsened since the run, the leader row the
//     export joins carries the text of a SHORTER unit than the one it now stands for, and the rest of that
//     source's translation sits in the ghost rows — and reaches no file. The acceptance stand book shows
//     it exactly: 0 pending, 20 ghosts, a «complete» export that is not.
//   - stale is a hole because the file would otherwise carry the translation of text that is no longer
//     in the book, byte-identical to the previous build, under `complete: true` — reached by an ordinary
//     sequence (build; fix a typo in the source; build). The engine already holds the disproving fact and
//     acts on it at the next run (the resume fast-path re-buys the unit); the writer reads the same fact.
//     Under config drift the fact is unreadable (see staleUnits) and the report says UNKNOWN, never none.
//   - config drift is NOT a hole: the text in the file is byte-for-byte the text the run shipped (the
//     drift is between the current config and that run — export.go). It is reported (BuildReport,
//     the log), never written into the file: «config drift» is operator vocabulary a reader cannot use.
//
// WORDS. Every word a reader sees at a hole or in the notice is DATA of the TARGET language
// (lang.LoadReaderWords: <langpack_root>/<target>/reader.txt, outside the pack's version — a rendering
// fact must never move a snapshot). A book with no language data gets the non-verbal form: a sign and
// the numbers. No Go source here knows how any language says «chapter» or «not translated».
//
// HEADINGS. A chapter's title is the deterministic heading the engine ALREADY glued onto its first
// unit's export text (ChunkExport.Heading == chunk.ApplyHeading's literal): the writer strips that
// known prefix and gives the same literal to the chapter's <h1> and its nav entry. When the heading is ""
// — a legal state (D39.100 п.1) — the chapter is titled with its bare number, the one label that decides
// nothing (which label a reader should see there is the open K-3 question; it is not settled here). A
// title the MODEL wrote inside the prose is prose and stays prose (backlog row 160 owns that seam).
//
// PARAGRAPHS. A paragraph is a non-blank line of the export text (CR, LF or CRLF all end a line, as on
// ingest). The editor's output separates paragraphs with a blank line in one place and a bare newline in
// the next with no meaning behind the difference (measured on the minirun export: unit 1/0 has 14 of one
// and 7 of the other, unit 2/0 has 56 and 0), so a rule that tells them apart invents a distinction the
// text does not carry. The line is the unit of a paragraph in the web-novel source, and it is what
// ingest reads back (the round-trip proof). A unit whose text yields NO paragraph — control characters
// only — shipped nothing a reader can see and is a hole of the withheld kind, whatever its row says.
//
// FILES. Everything a build writes beside the database is from ONE build: every requested format is
// staged first and committed together, and a format that was NOT requested is removed, so `book_files`
// never points at an older copy next to a newer one — two files of different vintage beside one
// database is the «silently complete» outcome again, reached through a legitimate command sequence
// (build, translate more, build --format epub). A failure to PREPARE a file leaves the previous set
// whole; a failure between two commits (a rename in a directory that was just written to — rare, but
// not impossible) is RefusalWriteIncomplete naming what landed, and a rebuild converges. A copy is still
// a snapshot of the store at build time: a consumer that hands a file to a reader — the platform's export
// door — builds first and serves what it built; it does not serve whatever is lying beside the database.
//
// --out names a NEW file, and it is never replaced: an explicit path can be anything the caller typed —
// the project database, the source, the other format's copy — and the writer cannot tell a previous copy
// of the book from any of those, so it refuses to write over whatever exists there. The engine's own
// place beside the database IS replaced, because there the writer knows what it is replacing.

// buildVersion versions the SHAPE of the build report.
const buildVersion = "tm-build-v1"

// BuildOptions are the caller's choices for one build.
type BuildOptions struct {
	// Formats to write, each one of bookfile.Formats; empty means all of them.
	Formats []string
	// Out is an explicit path for the ONE requested format; "" writes beside the project database. The
	// path must not exist yet (see the file comment) and its directory must.
	Out string
	// Partial writes a book with holes, marked, instead of refusing it.
	Partial bool
}

// BuildReport is what `tmctl build` prints: where the files landed and what the book is.
type BuildReport struct {
	Version string `json:"build_version"`
	BookID  string `json:"book_id"`
	// Files maps each written format to its absolute path.
	Files map[string]string `json:"files"`
	// The six states, so a caller does not have to open the file to learn what it wrote.
	TotalUnits      int `json:"total_units"`
	PendingUnits    int `json:"pending_units"`
	WithheldUnits   int `json:"withheld_units"`
	IncompleteUnits int `json:"incomplete_units"`
	StaleUnits      int `json:"stale_units"`
	// StaleUnknown is true when the stale check could not be made for at least one shipped unit: under
	// config drift (a moved config moves every rendered hash, so a source edit cannot be told from it),
	// or for a row with no content hash (written before the engine kept one). Reported, never folded
	// into «none».
	StaleUnknown bool `json:"stale_unknown"`
	GhostRows    int  `json:"ghost_rows"`
	ConfigDrift  bool `json:"config_drift"`
	// RemovedFiles are copies of formats this build did NOT produce that were found beside the database
	// and DELETED, absolute. The set there is always from one build, so removing them is correct — and
	// doing it in silence was not: `build --format epub` destroyed a neighbouring .book.txt with exit 0
	// and no line anywhere (cold run 31.08). A destructive act the caller did not ask for is named.
	RemovedFiles []string `json:"removed_files,omitempty"`
	// StaleCopies are the same copies when the removal FAILED. The book is written and this report is
	// valid; these paths simply hold files that are not from this build. Reported instead of returned as
	// an error, because an error here would have thrown this whole report away and made a completed build
	// read as an infra failure with nothing on disk.
	StaleCopies []string `json:"stale_copies,omitempty"`
	// Complete is true when the file has no hole of any kind — pending, withheld, incomplete, stale,
	// ghost. A file written under --partial with Complete=false carries the notice and the marks.
	Complete bool `json:"complete"`
	// TextModified is the file's dcterms:modified (BookExport.TextModified, or the epoch for a book that
	// shipped nothing), shown so the determinism of the stamp is visible without unzipping the file.
	TextModified string `json:"text_modified"`
}

// hole is one place the file cannot be whole, in reading order.
type hole struct {
	Kind    HoleKind
	Chapter int
	Unit    int
	Reason  string // the unit's flag reason (withheld) or the dropped member's (incomplete)
	// Dropped is how many members of the unit are missing for good. Filled for `incomplete` (text shipped
	// without a member) AND for `withheld` (a stop mark over a unit that also lost one), because the
	// operator's refusal text has to say both halves: topping up the ceiling finishes the unit and still
	// cannot bring the member back.
	Dropped int
}

// bookIdentifierPrefix is the URN namespace of dc:identifier: the book id under it is deterministic —
// the same book yields the same identifier in every build, which is what a reader's library keys on.
const bookIdentifierPrefix = "urn:textmachine:book:"

// bookFilePath is where format's copy of the book lives: beside the project DB, like every sidecar.
func (r *Runner) bookFilePath(format string) string { return r.Book.ProjectDB + ".book." + format }

// bookFilePaths is the complete map StatusArtifacts publishes — every format, whether written or not.
func (r *Runner) bookFilePaths() map[string]string {
	out := make(map[string]string, len(bookfile.Formats))
	for _, f := range bookfile.Formats {
		out[f] = absPath(r.bookFilePath(f))
	}
	return out
}

// BuildBook writes the reader's copy of the book (see the file comment). It is a $0 read of the store
// plus a file write: no job, no reservation, no provider. Safe whenever `export` is.
//
// It writes through the atomic staging of artifact.go: a reader — the platform's export door — sees
// either the previous copy or the new one, whole. An existing copy beside the database is REPLACED: the
// file is a deterministic function of the store and re-made at will, unlike a backup, which refuses to
// overwrite because it is a restore point nothing can re-make.
func (r *Runner) BuildBook(opts BuildOptions) (*BuildReport, error) {
	formats := opts.Formats
	if len(formats) == 0 {
		formats = bookfile.Formats
	}
	for _, f := range formats {
		if !bookfile.KnownFormat(f) {
			return nil, fmt.Errorf("pipeline: build: unknown format %q (want one of %s)", f, strings.Join(bookfile.Formats, ", "))
		}
	}
	if opts.Out != "" {
		if len(formats) != 1 {
			return nil, fmt.Errorf("pipeline: build: --out names ONE file, but %d formats were asked for (%s) — pass --format with a single format", len(formats), strings.Join(formats, ","))
		}
		if err := r.checkOutPath(opts.Out); err != nil {
			return nil, err
		}
	}
	exp, err := r.Export(false)
	if err != nil {
		return nil, err
	}
	stale, staleUnknown := r.staleUnits(exp)
	return r.buildFromExport(exp, stale, staleUnknown, formats, opts)
}

// checkOutPath is the --out rule of the file comment: a new file, in a directory that exists. Refused
// with the config class — the operator's argument is the thing to fix, and a plain error would put the
// typo on exit 1 among the crashes.
func (r *Runner) checkOutPath(out string) error {
	if _, err := os.Lstat(out); err == nil {
		return refuse(RefusalBadConfig, fmt.Errorf("pipeline: build %s: --out %s already exists and an explicit path is never replaced (it could be anything the caller typed — the project database included); name a new file, or omit --out for the engine's own place beside the database, which is replaced", r.Book.BookID, out))
	} else if !errors.Is(err, fs.ErrNotExist) {
		return refuse(RefusalBadConfig, fmt.Errorf("pipeline: build %s: --out %s: %w", r.Book.BookID, out, err))
	}
	if fi, err := os.Stat(filepath.Dir(out)); err != nil || !fi.IsDir() {
		return refuse(RefusalBadConfig, fmt.Errorf("pipeline: build %s: --out %s: its directory does not exist", r.Book.BookID, out))
	}
	return nil
}

// staleUnits finds the shipped units whose FINAL row was made for a source that is no longer the book's:
// the row's content hash — the signature of the rendered messages, source text included, which the
// resume fast-path compares before serving a row for $0 (stagerun.go) — differs from the hash the run
// would render now (repin.go renderedContentHashes, the same reproduction the money projection uses).
//
// It answers only when the config has NOT drifted: a prompt bump moves every rendered hash, and a
// source edit cannot be told apart from it — under drift the answer is UNKNOWN, reported as such. A unit
// whose hash cannot be reproduced (an old row with no content hash, a template the runner has not
// loaded, a member draft missing) is unknown too, never «not stale». It never fails the build: a check
// that cannot run says so in the report and the log.
func (r *Runner) staleUnits(exp *BookExport) (stale map[UnitRef]bool, unknown bool) {
	if exp.ConfigDrift {
		return nil, true
	}
	// A drift verdict that could not be REACHED is not «no drift». Before the basis existed this branch
	// did not exist either, and a failed drift check let build compute staleness as though the config were
	// clean — publishing `stale: 0` over a state nobody had established, inside a report whose own field
	// comment promises «Reported, never folded into none». The basis is what makes the distinction
	// available here at all.
	if exp.ConfigDriftBasis == DriftBasisUnknown {
		r.Log.Warn("build: the config-drift state is UNKNOWN, so whether the source moved under the shipped rows is UNKNOWN too (reported as unknown, not as none)",
			"book", r.Book.BookID)
		return nil, true
	}
	shipped := 0
	for _, ce := range exp.Chunks {
		if ce.Disposition != exportPending {
			shipped++
		}
	}
	if shipped == 0 {
		return nil, false
	}
	cannot := func(why string, err error) (map[UnitRef]bool, bool) {
		r.Log.Warn("build: the stale check could not run; whether the source moved under the shipped rows is UNKNOWN (reported as unknown, not as none)",
			"book", r.Book.BookID, "step", why, "err", err)
		return nil, true
	}
	statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
	if err != nil {
		return cannot("read chunk_status", err)
	}
	if err := r.projectStoredMemory(); err != nil {
		return cannot("materialize the stored bank", err)
	}
	full, err := r.bookChunks()
	if err != nil {
		return cannot("re-chunk the source", err)
	}
	hashes := r.cachedRenderedContentHashes(full, precomputeSticky(full, r.baseMemory, r.Pipeline.Context.GlossaryTokenBudget))
	finalRow := r.finalStageRows(statuses)
	stale = map[UnitRef]bool{}
	for _, ce := range exp.Chunks {
		if ce.Disposition == exportPending {
			continue
		}
		key := chunkKey{ce.Chapter, ce.ChunkIdx}
		cs, ok := finalRow[key]
		h, hok := hashes[key][cs.Stage]
		if !ok || cs.ContentHash == "" || !hok {
			unknown = true
			continue
		}
		if h != cs.ContentHash {
			stale[UnitRef{Chapter: ce.Chapter, ChunkIdx: ce.ChunkIdx}] = true
		}
	}
	if unknown {
		r.Log.Warn("build: the stale check could not be made for every shipped unit (a row without a content hash, or a position the run cannot re-render); those units are reported as unknown", "book", r.Book.BookID)
	}
	return stale, unknown
}

// buildFromExport is BuildBook after the projection: the decision (refuse or write), the assembly and
// the writes. Split from the export read so the states the harness cannot reach through a run — a book
// of no units — are testable on a synthetic projection.
func (r *Runner) buildFromExport(exp *BookExport, stale map[UnitRef]bool, staleUnknown bool, formats []string, opts BuildOptions) (*BuildReport, error) {
	// The two metadata the file cannot do without and the config does not require. Refused with the
	// config class rather than failed: book.yaml is the thing to fix, and a plain error would put the
	// operator's typo on exit 1 among the crashes.
	if strings.TrimSpace(r.Book.Title) == "" {
		return nil, refuse(RefusalBadConfig, fmt.Errorf("pipeline: build %s: book.yaml has no `title` — the reader's copy carries the title as dc:title and the writer invents none", r.Book.BookID))
	}
	if !looksLikeLanguageTag(r.Book.TargetLang) {
		return nil, refuse(RefusalBadConfig, fmt.Errorf("pipeline: build %s: book.yaml target_lang %q is not a language tag (dc:language wants BCP 47: `ru`, `zh`, `pt-BR`)", r.Book.BookID, r.Book.TargetLang))
	}
	if exp.TotalUnits == 0 {
		// A spine with no document is not an EPUB, and a text file of nothing is not a book. The class is
		// the one the ingest path raises for the same fact: the source was read and cut and there is no
		// book in it (refusal.go sourceHasNoContent).
		return nil, sourceHasNoContent(fmt.Errorf("pipeline: build %s: the book has no output units — nothing to write", r.Book.BookID))
	}
	words, present, err := lang.LoadReaderWords(r.Book.LangpackRoot, r.Book.TargetLang)
	if err != nil {
		// A data file the operator ships and this engine cannot read: the operator's file is the thing to
		// fix, nothing about the book is wrong — the config class.
		return nil, refuse(RefusalBadConfig, fmt.Errorf("pipeline: build %s: %w", r.Book.BookID, err))
	}
	book, holes, err := assembleBook(exp, stale, r.Book.Title, r.Book.TargetLang, words)
	if err != nil {
		return nil, err
	}
	complete := len(holes) == 0 && exp.GhostRows == 0
	if !complete && !opts.Partial {
		return nil, refuse(RefusalBookIncomplete, describeHoles(r.Book.BookID, exp, holes, staleUnknown))
	}
	if !complete && !present {
		r.Log.Warn("build: the target language ships no reader words; holes and the notice are marked in the NON-VERBAL form (a sign and numbers)",
			"book", r.Book.BookID, "target_lang", r.Book.TargetLang, "langpack_root", r.Book.LangpackRoot)
	}
	rep := &BuildReport{
		Version: buildVersion, BookID: exp.BookID, Files: map[string]string{},
		TotalUnits: exp.TotalUnits, GhostRows: exp.GhostRows, ConfigDrift: exp.ConfigDrift, StaleUnknown: staleUnknown,
		Complete: complete, TextModified: bookfile.Modified(book.Modified),
	}
	for _, h := range holes {
		switch h.Kind {
		case HolePending:
			rep.PendingUnits++
		case HoleWithheld:
			rep.WithheldUnits++
		case HoleIncomplete:
			rep.IncompleteUnits++
		case HoleStale:
			rep.StaleUnits++
		}
	}

	// Stage every format, then commit every format (the writeDecisionFiles discipline, artifact.go): an
	// environment failure while PREPARING the second file must not leave a new epub beside the previous txt.
	var staged []*stagedFile
	abort := func() {
		for _, s := range staged {
			s.abort()
		}
	}
	for _, f := range formats {
		var buf bytes.Buffer
		switch f {
		case "epub":
			err = bookfile.WriteEPUB(&buf, book)
		case "txt":
			err = bookfile.WriteTXT(&buf, book)
		}
		if err != nil {
			abort()
			return nil, err
		}
		path := r.bookFilePath(f)
		if opts.Out != "" {
			path = opts.Out
		}
		s, err := stageFileAtomic(path, buf.Bytes())
		if err != nil {
			abort()
			if opts.Out != "" {
				// The caller named the place and the place cannot take a file: their argument, the
				// config class — not an engine failure on exit 1.
				return nil, refuse(RefusalBadConfig, fmt.Errorf("pipeline: build %s: --out %s cannot be written: %w", r.Book.BookID, opts.Out, err))
			}
			return nil, err
		}
		staged = append(staged, s)
		rep.Files[f] = absPath(path)
	}
	for i, s := range staged {
		if err := s.commit(); err != nil {
			for _, rest := range staged[i+1:] {
				rest.abort()
			}
			landed := make([]string, 0, i)
			for _, done := range staged[:i] {
				landed = append(landed, done.path)
			}
			return nil, refuse(RefusalWriteIncomplete, fmt.Errorf("pipeline: build %s: the copies were prepared and the write did not complete — %w; landed: [%s]; not landed: %s and the rest — rebuild to converge",
				r.Book.BookID, err, strings.Join(landed, " "), s.path))
		}
	}
	if opts.Out == "" {
		// The set beside the database is from THIS build: a format not asked for is removed rather than
		// left as an older copy the envelope would present as current (see the file comment).
		//
		// ⛔ TWO DISCLOSURE DEFECTS LIVED IN THIS LOOP, and both were found on the cold run of 31.08.
		//
		// (a) IT DESTROYED A FILE IN SILENCE. `os.Remove` returning nil means a file WAS there and is now
		// gone; returning fs.ErrNotExist means there was nothing. The loop branched on the error class and
		// threw the distinction away, so `build --format epub` deleted the neighbouring .book.txt with
		// exit 0 and not one line about it. The engine knew; nobody was told (§2.1).
		//
		// (b) A FAILURE HERE RETURNED A BARE ERROR — with the new files ALREADY COMMITTED to disk and the
		// BuildReport, complete and correct, thrown away with it. The exit mapper turns an unclassified
		// error into 1, and by the band's contract a reader concludes «infra failure, nothing written» and
		// writes the run off. That is §2.5: the irreversible act is done, so its report must survive any
		// later error on the same path. Cleaning up an old copy is HOUSEKEEPING — it cannot un-write the
		// book — so its failure is a WARN carried in the report, never a verdict about the build.
		for _, f := range bookfile.Formats {
			if _, wanted := rep.Files[f]; wanted {
				continue
			}
			path := r.bookFilePath(f)
			switch err := os.Remove(path); {
			case err == nil:
				rep.RemovedFiles = append(rep.RemovedFiles, absPath(path))
				r.Log.Warn("build: an EXISTING copy of a format this build did not produce was deleted — the set beside the database is always from ONE build",
					"book", r.Book.BookID, "format", f, "path", absPath(path))
			case errors.Is(err, fs.ErrNotExist):
				// Nothing was there. Not a fact anybody needs.
			default:
				rep.StaleCopies = append(rep.StaleCopies, absPath(path))
				r.Log.Warn("build: a previous copy of a format this build did not produce could NOT be removed; the book IS written and this report stands, but that file is STALE and does not belong to this build",
					"book", r.Book.BookID, "format", f, "path", absPath(path), "err", err)
			}
		}
	}
	if exp.ConfigDrift {
		r.Log.Warn("build: CONFIG-DRIFT — the file carries the text the run shipped; the current config would render a different snapshot (not written into the file)",
			"book", r.Book.BookID, "current", exp.CurrentSnapshot)
	}
	r.Log.Info("build: book written", "book", r.Book.BookID, "files", rep.Files, "complete", complete,
		"pending", rep.PendingUnits, "withheld", rep.WithheldUnits, "incomplete", rep.IncompleteUnits, "stale", rep.StaleUnits,
		"stale_unknown", staleUnknown, "ghost_rows", exp.GhostRows)
	return rep, nil
}

// describeListMax bounds the per-hole lines of a refusal: a never-run 2000-chapter book has thousands of
// pending units, and the refusal is a diagnostic, not the manifest.
const describeListMax = 50

// describeHoles is the refusal's text: the counts, then every hole by place and reason.
func describeHoles(bookID string, exp *BookExport, holes []hole, staleUnknown bool) error {
	counts := map[HoleKind]int{}
	for _, h := range holes {
		counts[h.Kind]++
	}
	var sb strings.Builder
	fmt.Fprintf(&sb, "pipeline: build %s: the book is not whole and --partial was not given — nothing written: pending=%d withheld=%d incomplete=%d stale=%d ghost_rows=%d of total_units=%d",
		bookID, counts[HolePending], counts[HoleWithheld], counts[HoleIncomplete], counts[HoleStale], exp.GhostRows, exp.TotalUnits)
	if staleUnknown {
		sb.WriteString(" (stale: UNKNOWN for some or all units — the config drifted, or a row carries no content hash)")
	}
	for i, h := range holes {
		if i == describeListMax {
			fmt.Fprintf(&sb, "\n  … and %d more", len(holes)-describeListMax)
			break
		}
		switch h.Kind {
		case HolePending:
			fmt.Fprintf(&sb, "\n  chapter %d unit %d: pending (not yet translated)", h.Chapter, h.Unit)
		case HoleWithheld:
			// The operator gets the same THREE-way distinction the reader's file and the export banner get,
			// and the third part is the one that costs him a second run: a stop mark is work the next run
			// buys — unless the unit is ALSO short a member for good, which no purchase brings back. Told
			// only the first half, he tops up, resumes, and the build refuses again with a different hole.
			switch {
			case FlagReason(h.Reason).AnswersForResume():
				fmt.Fprintf(&sb, "\n  chapter %d unit %d: withheld%s", h.Chapter, h.Unit, parenReason(h.Reason))
			case h.Dropped > 0:
				fmt.Fprintf(&sb, "\n  chapter %d unit %d: withheld — paid for and NOT done, AND %d member(s) of it are missing for good; the next run finishes the unit but cannot bring those back%s",
					h.Chapter, h.Unit, h.Dropped, parenReason(h.Reason))
			default:
				fmt.Fprintf(&sb, "\n  chapter %d unit %d: withheld — paid for and NOT done; the next run re-does it%s", h.Chapter, h.Unit, parenReason(h.Reason))
			}
		case HoleIncomplete:
			fmt.Fprintf(&sb, "\n  chapter %d unit %d: incomplete — %d member(s) of the unit missing from its text%s", h.Chapter, h.Unit, h.Dropped, parenReason(h.Reason))
		case HoleStale:
			fmt.Fprintf(&sb, "\n  chapter %d unit %d: stale (the source changed under this unit since the run; `translate` re-buys it)", h.Chapter, h.Unit)
		}
	}
	if len(exp.GhostUnits) > 0 {
		sb.WriteString("\n  ghost rows (translated text the current cut of the book cannot place):")
		for i, g := range exp.GhostUnits {
			if i == describeListMax {
				fmt.Fprintf(&sb, " … and %d more", len(exp.GhostUnits)-describeListMax)
				break
			}
			fmt.Fprintf(&sb, " %d/%d", g.Chapter, g.ChunkIdx)
		}
	}
	return fmt.Errorf("%s", sb.String())
}

func parenReason(reason string) string {
	if reason == "" {
		return ""
	}
	return " (" + reason + ")"
}

// assembleBook turns the export into the document model: chapters from the units in manifest order,
// the known heading prefix stripped and re-used as the title, a mark at every hole, the notice when
// there is one — and returns the holes it marked, in reading order, which is what the refusal and the
// report count (the writer's own reading of the units, not the counters'). Pure: the same export, stale
// set and words yield the same Book.
func assembleBook(exp *BookExport, stale map[UnitRef]bool, title, language string, words lang.ReaderWords) (*bookfile.Book, []hole, error) {
	modified, err := bookModified(exp.TextModified)
	if err != nil {
		return nil, nil, err
	}
	book := &bookfile.Book{
		Identifier: bookIdentifierPrefix + exp.BookID,
		Title:      strings.TrimSpace(title),
		Language:   language,
		Modified:   modified,
	}
	// Ghost rows by the chapter number they were STORED under — the old cut's numbering. Chapter numbers
	// are dense, so a source edit that empties a chapter shifts every later number, and a ghost's mark can
	// then sit in a neighbouring chapter's text; the book is still marked not-whole, only the mark's
	// coordinate is the old cut's. A ghost whose chapter the current cut no longer has is in the notice's
	// count and nowhere else.
	ghostsByChapter := map[int]int{}
	for _, g := range exp.GhostUnits {
		ghostsByChapter[g.Chapter]++
	}
	var holes []hole
	chapterNo := 0 // dense chapter number of the chapter being assembled; 0 = none yet (numbers are 1-based)
	closeChapter := func() {
		if chapterNo == 0 {
			return
		}
		if n := ghostsByChapter[chapterNo]; n > 0 {
			i := len(book.Chapters) - 1
			book.Chapters[i].Paragraphs = append(book.Chapters[i].Paragraphs,
				lang.FillReaderTemplate(words.HoleGhost, "chapter", strconv.Itoa(chapterNo), "ghost", strconv.Itoa(n)))
			delete(ghostsByChapter, chapterNo)
		}
	}
	for _, ce := range exp.Chunks {
		if ce.Chapter != chapterNo {
			closeChapter()
			chapterNo = ce.Chapter
			t := ce.Heading
			if t == "" {
				t = strconv.Itoa(ce.Chapter)
			}
			book.Chapters = append(book.Chapters, bookfile.Chapter{Title: t})
		}
		i := len(book.Chapters) - 1
		chapter, unit := strconv.Itoa(ce.Chapter), strconv.Itoa(ce.ChunkIdx)
		text := ce.FinalText
		if ce.Heading != "" {
			// The literal chunk.ApplyHeading glued on: `heading + "\n\n" + text`. A KNOWN prefix, stripped
			// only when it is there; never a guess at the text.
			text = strings.TrimPrefix(text, ce.Heading+"\n\n")
		}
		kind := UnitHole(ce)
		var paragraphs []string
		switch kind {
		case HoleNone, HoleIncomplete:
			if stale[UnitRef{Chapter: ce.Chapter, ChunkIdx: ce.ChunkIdx}] {
				// The text is the translation of a source that is no longer the book's: not shown.
				kind = HoleStale
				break
			}
			paragraphs = textParagraphs(text)
			if len(paragraphs) == 0 {
				// A row that says «text» over characters no reader's file can carry shipped nothing a
				// reader can see; the file says so rather than carrying an empty chapter.
				kind = HoleWithheld
			}
		}
		mark := func(tmpl string, extra ...string) {
			args := append([]string{"chapter", chapter, "unit", unit}, extra...)
			book.Chapters[i].Paragraphs = append(book.Chapters[i].Paragraphs, lang.FillReaderTemplate(tmpl, args...))
		}
		switch kind {
		case HolePending:
			mark(words.HolePending)
		case HoleWithheld:
			// ⛔ WHICH SENTENCE DEPENDS ON WHETHER THE ROW IS A VERDICT, and it used to depend on nothing.
			// The withheld phrase tells the reader the fragment «requires a human check» (langpacks/<target>/
			// reader.txt, `hole.withheld`), which is true of a refusal or a contaminated output — and FALSE
			// of a stop mark: a call a person cut, or a re-attack a ceiling refused, is a position that was
			// paid for and not finished, and no human has anything to check. The next run does it. Of the
			// phrases that exist, `hole.pending` («not translated yet») is the true one for those two.
			//
			// ⚠ THE REASON STAYS ON THE HOLE EITHER WAY, and that is deliberate rather than incidental: a
			// third phrase of its own («paid for, not finished: top up and resume») is then a ROW IN
			// reader.txt plus a branch here, not a re-design of this predicate. The question is asked of the
			// REASON (AnswersForResume) so this site and every other asker cannot drift; the askers are
			// counted in one place only — beside AnswersForResume in disposition.go — so the number has a
			// single carrier to keep honest.
			//
			// ⚠ AND WHERE THE REASON DOES *NOT* REACH, so nobody builds on a promise this does not make:
			// BuildReport carries COUNTS and no reasons, so `withheld_units` holds both kinds and the
			// platform — which reads exactly five counters of it — cannot tell «needs money» from «needs a
			// human». The refusal text below (describeHoles) does carry it, and so does the export record.
			// Splitting the counter is a contract change across two zones and is named in the pack's report
			// rather than smuggled in here.
			// ⚠ AND «DROPPED MEMBERS» KEEPS THE WITHHELD PHRASE WHATEVER THE REASON: a c-lite unit whose
			// member is permanently flagged has lost that text for good, so «not translated yet» would
			// promise a next run that cannot bring it back. The reader is told a human is needed, which is
			// true of the member even when the stage that would have assembled it merely ran out of money.
			if FlagReason(ce.FlagReason).AnswersForResume() || ce.DroppedMembers > 0 {
				mark(words.HoleWithheld)
			} else {
				mark(words.HolePending)
			}
			// ⛔ AND THE DROP COUNT TRAVELS WITH IT, because the operator's refusal text is the THIRD surface
			// of this distinction and it could not ask without the field: a unit can be both «paid for and
			// not done» and short a member FOR GOOD, and that is the one person who can act on the second
			// half. Before this the reader and the export had the caveat and the operator did not — the
			// «took the form, not the guarantee» class, one surface further along.
			holes = append(holes, hole{Kind: HoleWithheld, Chapter: ce.Chapter, Unit: ce.ChunkIdx, Reason: ce.FlagReason, Dropped: ce.DroppedMembers})
			continue
		case HoleStale:
			mark(words.HoleStale)
		case HoleIncomplete:
			mark(words.HoleIncomplete, "dropped", strconv.Itoa(ce.DroppedMembers))
			holes = append(holes, hole{Kind: HoleIncomplete, Chapter: ce.Chapter, Unit: ce.ChunkIdx, Reason: ce.DroppedReason, Dropped: ce.DroppedMembers})
		}
		if kind == HolePending || kind == HoleStale {
			holes = append(holes, hole{Kind: kind, Chapter: ce.Chapter, Unit: ce.ChunkIdx})
			continue
		}
		book.Chapters[i].Paragraphs = append(book.Chapters[i].Paragraphs, paragraphs...)
	}
	closeChapter()
	if len(holes) > 0 {
		book.Notice = append(book.Notice,
			lang.FillReaderTemplate(words.NoticeHoles, "holes", strconv.Itoa(len(holes)), "total", strconv.Itoa(exp.TotalUnits)))
	}
	if exp.GhostRows > 0 {
		book.Notice = append(book.Notice, lang.FillReaderTemplate(words.NoticeGhost, "ghost", strconv.Itoa(exp.GhostRows)))
	}
	book.Description = strings.Join(book.Notice, " ")
	return book, holes, nil
}

// textParagraphs splits export text into paragraphs: one per non-blank line (CR, LF and CRLF all end a
// line — what text.NormalizeSource does on ingest, so a bare CR cannot make the two formats disagree),
// trimmed and cleaned of the control characters no reader's format admits (bookfile.CleanText — applied
// HERE, once, so the EPUB and the text file carry the same paragraphs; see the file comment on why the
// line is the unit).
func textParagraphs(text string) []string {
	text = strings.ReplaceAll(text, "\r\n", "\n")
	text = strings.ReplaceAll(text, "\r", "\n")
	var out []string
	for _, line := range strings.Split(text, "\n") {
		if p := strings.TrimSpace(bookfile.CleanText(line)); p != "" {
			out = append(out, p)
		}
	}
	return out
}

// looksLikeLanguageTag is the syntactic shape of a BCP 47 tag (a 2–3 letter primary subtag, optional
// alphanumeric subtags): the one check that keeps an unusable dc:language a loud refusal instead of a
// file every validator rejects. It knows no language; it knows what a tag looks like.
func looksLikeLanguageTag(s string) bool {
	subtags := strings.Split(s, "-")
	if len(subtags[0]) < 2 || len(subtags[0]) > 3 {
		return false
	}
	for i, sub := range subtags {
		if sub == "" || len(sub) > 8 {
			return false
		}
		for _, r := range sub {
			isLetter := (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')
			isDigit := r >= '0' && r <= '9'
			if !isLetter && !(isDigit && i > 0) {
				return false
			}
		}
	}
	return true
}

// bookModified turns BookExport.TextModified into the file's modification time. A book that shipped
// nothing ("" — no final row) is stamped with the epoch: nothing has ever modified its text, and a
// constant is the one value that is neither a clock reading nor a claim.
func bookModified(textModified string) (time.Time, error) {
	if textModified == "" {
		return time.Unix(0, 0).UTC(), nil
	}
	t, err := time.Parse("2006-01-02T15:04:05Z", textModified)
	if err != nil {
		return time.Time{}, fmt.Errorf("pipeline: build: export text_modified %q is not CCYY-MM-DDThh:mm:ssZ: %w", textModified, err)
	}
	return t.UTC(), nil
}