textmachine/backend/internal/chunk/ingest.go

package chunk

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/text"

	"golang.org/x/net/html"
	"golang.org/x/text/encoding/simplifiedchinese"
	xunicode "golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
)

// ingest.go: the step-3a import layer. Ingest turns a source file (txt or epub)
// into an ordered list of per-chapter NORMALIZED text plus the ruby/furigana
// readings captured on the way (04-unhappy §4 / D9). It is the ONLY place that
// reads the source; the runner then feeds doc.Chapters to SplitChunks and persists
// doc.Ruby (seeding.go). It is offline and deterministic — no LLM, no time/rand — so
// the whole path is $0 and reproducible.
//
// epub is a text extraction (02-mvp:25): read the chapters the book's own table of contents declares —
// nav (EPUB 3) or NCX (EPUB 2), falling back to the spine only when neither resolves — strip tags, drop
// inline markup/styling. The ONE thing we do NOT drop is ruby:
// <ruby>base<rt>reading</rt></ruby> carries an author reading of a name/term that a
// plain text export would silently lose (the documented hole, 04-unhappy §4). We
// keep the BASE in the body and hand the READING to the caller for a glossary lock
// (memory v2, step 4) — never injecting it into a prompt here (§7d).

// chapterSep is the ASCII form feed (U+000C), the "page/section break". It is a SEPARATOR the file
// carries, NOT a statement that a chapter begins — see the Structure constants. It is invisible in prose
// and untouched by text.NormalizeSource, and it draws the boundaries only when no header line does
// (cutTXT). A txt with neither is a single chapter. epub has its own table of contents and never uses this.
const chapterSep = "\f"

// RubyReading is one captured (base, reading) pair and the chapter it appeared in.
// The pipeline aggregates these (min chapter, count) before persisting (seeding.go).
type RubyReading struct {
	Base    string // the ruby body: the kanji/base surface form
	Reading string // the <rt> reading (furigana)
	Chapter int    // 1-based chapter where this occurrence was found
}

// Document is the ingested source: per-chapter normalized text (in reading order)
// plus every ruby occurrence found. Chapters is what SplitChunks consumes.
type Document struct {
	Chapters []string
	Ruby     []RubyReading
	// FormFeedsIgnored counts page breaks the winning cut turned into paragraph breaks. Reported in a WARN,
	// deliberately NOT on the wire: there is no consumer for it, and a field without one is a mechanism kept
	// for its own sake.
	FormFeedsIgnored int
	// Titles are the chapter titles AS THE SOURCE SPELLS THEM, index-aligned to Chapters: the header line
	// for a txt, the navLabel for an EPUB. Additive — the engine's own rendered heading is a different
	// string in a different language and neither replaces the other. "" where the source names nothing.
	Titles []string
	// Structure says WHERE the chapter boundaries came from, which is a different question from how many
	// there are. A caller offering an order «up to chapter N» has to know whether N names something the
	// source itself declared or something this package inferred from prose, because the two fail
	// differently: a declared cut is wrong only if the file is wrong, an inferred one is wrong whenever a
	// sentence happens to open like a header. See the four constants.
	Structure string
	// The counts below are the ALARM this work ships INSTEAD of a validator: a degenerate table of contents
	// shows up as a large DocumentsAttached, not as a wrong label. They are deliberately NOT on the wire —
	// no consumer exists for them — and they reach an operator through exactly one place, IngestNotes, so
	// that adding a count and forgetting to report it is not possible one field at a time.
	DocumentsAttached int // spine documents folded into a chapter rather than opening one
	// Excluded NAMES every service page that was not read, with its role and its size.
	//
	// ⚠ A NUMBER HERE WOULD HIDE THE ONE THING WORTH SEEING. The conservation invariant cannot redden on a
	// WRONG exclusion — the document is accounted for, so the sum closes — which means a mistaken exclusion
	// is exactly the defect no gate can catch. What is left is a human reading the warning, and "excluded: 1"
	// gives them nothing to doubt. «excluded OEBPS/index_split_124.html as `toc`, 2019 runes» does.
	Excluded []ExcludedDocument
	// TOCUnreadable counts tables of contents the book DECLARED and this reader could not read at all —
	// the manifest names a nav document, or the spine names an NCX, and the bytes are missing or will not
	// parse.
	//
	// ⛔ IT IS NOT TOCUnresolved, and merging the two would hide the worse case inside the milder one.
	// "Read it, and some of its targets named nothing" is a book with a partly broken table; "declared one
	// and could not read it at all" is a book whose whole statement about its chapters was lost, after
	// which the cut silently falls back to the spine and the book is sold by characters. A reader must be
	// able to tell those apart, because only the second explains why an EPUB with a real nav came back
	// `delimited`.
	//
	// ⚠ A book that declares NO table of contents is not counted here: going to the spine is its normal,
	// correct path, not a degradation.
	TOCUnreadable               int
	TOCUnresolved               int // table-of-contents targets that named nothing this book reads
	TargetsCollapsed            int // targets that shared a document with an earlier one, so the cut is coarser
	ServiceRolesInsideDocuments int // toc/cover roles declared at a POINT inside a document, so NOT excluded
}

// ExcludedDocument is one spine document this reader declined to read as book text, and why.
type ExcludedDocument struct {
	Entry string // the document, as the zip spells it
	Role  string // what made it service: "nav", or the declared role ("toc" / "cover")
	Runes int    // how much text was NOT read — the size of the doubt, if the exclusion was wrong
}

// IngestNotes returns the alarm counts as log key/value pairs, or nil when the reading was unremarkable.
//
// ONE reporter for every count is the point. The first version logged two of five, and the three silent ones
// included the number that makes a degenerate cut visible — 126 documents folded into 6 chapters printed
// nothing at all. A single accessor cannot be half-wired.
func (d *Document) IngestNotes() []any {
	var kv []any
	add := func(k string, v int) {
		if v > 0 {
			kv = append(kv, k, v)
		}
	}

	add("form_feeds_ignored", d.FormFeedsIgnored)
	add("documents_attached", d.DocumentsAttached)
	if n := len(d.Excluded); n > 0 {
		kv = append(kv, "documents_excluded", n, "excluded", d.Excluded)
	}
	add("toc_unreadable", d.TOCUnreadable)
	add("toc_unresolved", d.TOCUnresolved)
	add("targets_collapsed", d.TargetsCollapsed)
	add("service_roles_inside_documents", d.ServiceRolesInsideDocuments)
	if kv == nil {
		return nil
	}
	return append(kv, "chapters", len(d.Chapters), "structure", d.Structure)
}

// The four values of Document.Structure — WHICH PATH drew the chapter boundaries. Not a confidence score:
// the question is provenance, with one exception that is also provenance (a single-chapter document is
// `none` whichever path produced it, because a boundary nothing crossed was never tested).
//
//   - DECLARED — the format states the chapter STRUCTURE: an EPUB nav or NCX, a table of contents the file
//     itself carries. Getting these wrong requires the file to be wrong.
//   - DELIMITED — the file carried a SEPARATOR, but "this separator is a chapter" is the engine's
//     assumption. An EPUB spine declares reading ORDER; a form feed is the ASCII PAGE break, and in practice
//     comes from pdftotext rather than from an author. Both are real marks in the file and neither is a
//     statement about chapters.
//   - DETECTED — the engine inferred the boundary from CONTENT, by matching header-shaped lines. Right on
//     the corpus it was built for, and a guess: a paragraph opening «第三节…» within the length bound is
//     indistinguishable from a header.
//   - NONE — one chapter, or no grammar was declared for the source at all.
//
// ⚠ The header match runs on the SOURCE language's structure data, so a source with no such data finds no
// headers and reports `none` however clearly its chapters are marked. That asymmetry is not introduced here
// — the cut has been CJK-only since it was written (row 283) — but it now has a NAME on the wire instead of
// hiding inside a chapter count, and a consumer can decline to offer an order in chapters rather than offer
// one against a cut nobody made.
const (
	StructureDeclared  = "declared"
	StructureDelimited = "delimited"
	StructureDetected  = "detected"
	StructureNone      = "none"
)

// structureOf reports the provenance of a cut: `none` below two chapters, otherwise the witness of the path
// that WON. There is nothing to aggregate — the paths compete, so exactly one of them drew every boundary.
func structureOf(chapters []string, witness string) string {
	if len(chapters) < 2 {
		return StructureNone
	}
	return witness
}

// ingest reads a source file with automatic encoding detection (the txt path), source language
// unspecified. Dispatch is by extension: .epub → the epub reader; anything else → plain text. Kept
// as the terse form used by tests and the epub path.
func ingest(p string) (*Document, error) {
	return IngestEncoded(p, "", "", lang.DefaultCJKStructure())
}

// IngestEncoded reads a source file and returns its Document, decoding a txt source per the book's
// declared encoding (auto|utf8|gb18030; "" == auto) and source language (zh|ja|en; "" == unknown).
// sourceLang scopes the GB18030 auto-probe to Chinese sources — a Shift-JIS/EUC-JP ja file would
// otherwise decode to Han-shaped mojibake that passes the plausibility guard (self-review critical).
// Dispatch is by extension: .epub → the epub reader (its own per-document charset handling);
// anything else → plain text.
func IngestEncoded(p, encoding, sourceLang string, st *lang.SourceStructure) (*Document, error) {
	if strings.EqualFold(extOf(p), ".epub") {
		return ingestEPUB(p)
	}
	return ingestTXT(p, encoding, sourceLang, st)
}

// extOf returns the lower-cased file extension incl. the dot ("" if none).
func extOf(p string) string {
	i := strings.LastIndexByte(p, '.')
	if i < 0 || strings.ContainsAny(p[i:], "/\\") {
		return ""
	}
	return strings.ToLower(p[i:])
}

// ingestTXT reads a plain-text source: decode the raw bytes to UTF-8 per `encoding`, split on \f
// into chapters, text.NormalizeSource each (BOM/CRLF/NFC/outer-trim). No ruby in txt. Encoding decode
// runs BEFORE text.NormalizeSource so the whole downstream path (chunker fuzz-invariant, memory keys)
// sees valid UTF-8; a decode failure is a LOUD error, never silent U+FFFD corruption.
func ingestTXT(p, encoding, sourceLang string, st *lang.SourceStructure) (*Document, error) {
	raw, err := os.ReadFile(p)
	if err != nil {
		return nil, fmt.Errorf("chunk: ingest txt: %w", err)
	}
	decoded, err := decodeSourceBytes(raw, encoding, sourceLang)
	if err != nil {
		return nil, fmt.Errorf("chunk: ingest txt %s: %w", p, err)
	}
	chapters, witness, formFeeds := cutTXT(decoded, st)
	titles := make([]string, len(chapters))
	// WHERE the header may be found depends on which path drew the boundaries, and the difference is not
	// cosmetic.
	//
	// On a cut the HEADERS won, a header may sit deep inside chapter one: the frozen preamble rule folds
	// everything before the first header into it, so the whole chapter is scanned.
	//
	// On any other cut the boundaries came from page breaks, and a header-shaped line buried in the prose is
	// a single occurrence that never met the ≥2 floor — calling it this chapter's title would be a confident
	// wrong answer. Only the OPENING line counts there, which still gives a one-chapter document its own real
	// title without inventing one for the rest.
	scanAll := witness == StructureDetected
	for i, c := range chapters {
		if scanAll {
			titles[i] = chapterTitleRaw(c, st)
			continue
		}
		titles[i] = chapterTitleRaw(firstLine(c), st)
	}
	return &Document{
		Chapters:         chapters,
		Titles:           titles,
		Structure:        structureOf(chapters, witness),
		FormFeedsIgnored: formFeeds,
	}, nil
}

// chapterTitleRaw returns the chapter's header line verbatim, or "" when it has none.
//
// It scans the WHOLE chapter rather than only its first line: the preamble rule folds everything before the
// first header into chapter one, so chapter one's header is not at its start. Scanning finds it there and
// nowhere else, which is why chapter one gets a title at all.
func chapterTitleRaw(chapter string, st *lang.SourceStructure) string {
	if st == nil {
		return ""
	}
	for _, ln := range strings.Split(chapter, "\n") {
		for _, u := range st.UnitOrdered {
			if isChapterHeader(ln, u, st) {
				return strings.TrimSpace(ln)
			}
		}
	}
	return ""
}

// firstLine returns everything before the first newline.
func firstLine(s string) string {
	if i := strings.IndexByte(s, '\n'); i >= 0 {
		return s[:i]
	}
	return s
}

// cutTXT runs the two candidate cuts against each other and returns the winner's chapters, its provenance
// witness, and how many form feeds the winner ignored.
//
// ⛔ THE PATHS COMPETE, THEY DO NOT ADD UP. They used to: \f split first, then each part was split on header
// lines, and a single stray \f among fifty matched headers made the whole cut `declared` — forty-nine
// guesses reported as the file's own word. Competing, there is no mixed case to aggregate: headers found ⇒
// headers drew every boundary; none found ⇒ the form feeds did.
//
// ⛔ THE ORDER BELOW IS LOAD-BEARING, and the naive one loses a chapter SILENTLY. There is no \n before a
// \f, so a header sitting right after one shares a LINE with the previous chapter's prose, where both the
// length bound and the ^ anchor reject it — detect first and that chapter is never seen, while the label
// stays correct and the loss goes unreported. Hence: neutralise on a COPY, detect on the copy, and fall back
// to the ORIGINAL if headers lose, because neutralising in place would destroy the separator the fallback
// needs.
func cutTXT(decoded string, st *lang.SourceStructure) (chapters []string, witness string, formFeedsIgnored int) {
	working := strings.ReplaceAll(decoded, chapterSep, "\n\n")
	if headerCut := splitTextChapters(working, st); len(headerCut) > 1 {
		for _, chap := range headerCut {
			chapters = append(chapters, text.NormalizeSource(chap))
		}
		return chapters, StructureDetected, strings.Count(decoded, chapterSep)
	}
	for _, part := range strings.Split(decoded, chapterSep) {
		chapters = append(chapters, text.NormalizeSource(part))
	}
	return chapters, StructureDelimited, 0
}

// --- CJK chapter-header splitting (D18: real zh/ja txt mark chapters as «第N章/节/回») ---------

// chapterHeaderMaxRunes bounds a header line so a prose sentence that merely opens with «第三节…»
// (a longer line) is not mistaken for a header. The 蛊真人 headers are ≤23 runes; 60 leaves room for
// a long subtitle while still excluding a full prose sentence (self-review: 40 could miss a long
// legitimate subtitle → merged chapters).
const chapterHeaderMaxRunes = 60

// isChapterHeader reports whether a single line is a chapter header for the given unit rune, under the
// book's resolved source structure. It
// requires: a short trimmed line (≤60 runes); a leading 第<numerals>; the unit rune immediately
// after the numerals; and — critically — the char AFTER the unit is a SEPARATOR (whitespace / ：:、,.
// / dash) or end-of-line, NOT a content glyph. The separator guard is the precision fix (self-
// review): 回 is a common measure word, so prose like «第一回见面…» glues a content char (见) right
// after the unit and must NOT split; a real header writes «第一节：…» or «第1章 …» with a separator.
// (A glued-title header «第一章天空…» without a separator is not detected — rare; a recall trade for
// no false splits. Deterministic and pure.)
func isChapterHeader(line string, unit rune, st *lang.SourceStructure) bool {
	if st == nil {
		return false
	}
	t := strings.TrimSpace(line)
	if utf8.RuneCountInString(t) == 0 || utf8.RuneCountInString(t) > chapterHeaderMaxRunes {
		return false
	}
	loc := st.HeaderNumeralRE().FindStringIndex(t)
	if loc == nil {
		return false
	}
	rest := t[loc[1]:] // the runes right after the numerals
	r, sz := utf8.DecodeRuneInString(rest)
	if r != unit {
		return false
	}
	after := rest[sz:]
	if after == "" {
		return true // the header is exactly 第N章
	}
	nr, _ := utf8.DecodeRuneInString(after)
	return isHeaderSeparator(nr)
}

// isHeaderSeparator reports whether a rune separates a chapter number from its title (so the line is
// a header, not a prose sentence that continues with a content glyph after 第N章). Anything that is
// NOT content (letter / ideograph / kana / digit) counts as a separator. It is the exact complement of
// chunker.go's isHeaderContentRune — one shared rune-class list, so the two cannot byte-drift apart (row
// 89: previously the same classification was spelled out in both files, synced only by hand). NOT to be
// confused with chunker.go's isHeadingSeparator, a narrower punctuation whitelist for trimming subtitles.
func isHeaderSeparator(r rune) bool {
	return !isHeaderContentRune(r)
}

// detectChapterUnit picks the section marker that appears most as a header line (≥2 to avoid a
// single stray match). Returns 0 when no unit qualifies (→ the part stays a single chapter).
func detectChapterUnit(lines []string, st *lang.SourceStructure) rune {
	if st == nil {
		return 0
	}
	best, bestN := rune(0), 0
	// The section units come from the book's RESOLVED source structure (lang.SourceStructure — its own
	// <lang>/structure.txt, else the embedded CJK default); iterated in AUTHORED
	// order so a tie between two units breaks deterministically (first-seen), as the fixed slice once did.
	for _, unit := range st.UnitOrdered {
		n := 0
		for _, ln := range lines {
			if isChapterHeader(ln, unit, st) {
				n++
			}
		}
		if n >= 2 && n > bestN {
			best, bestN = unit, n
		}
	}
	return best
}

// splitTextChapters splits one text block on its dominant CJK chapter-header lines. The header line
// STARTS its chapter (the title is kept). Any preamble before the FIRST header is merged into
// chapter 1 so «第一节» == chapter 1 (dense numbering aligns with the section numbers). A block with
// no detectable headers is returned unchanged as a single chapter.
func splitTextChapters(text string, st *lang.SourceStructure) []string {
	norm := strings.ReplaceAll(text, "\r\n", "\n")
	norm = strings.ReplaceAll(norm, "\r", "\n") // bare-CR (old Mac) lines too, so a CR-only header is seen
	lines := strings.Split(norm, "\n")
	unit := detectChapterUnit(lines, st)
	if unit == 0 {
		return []string{text}
	}
	var chapters []string
	var cur []string
	seenHeader := false
	for _, ln := range lines {
		if isChapterHeader(ln, unit, st) {
			if seenHeader && hasNonBlank(cur) {
				chapters = append(chapters, strings.Join(cur, "\n"))
				cur = nil
			}
			seenHeader = true
		}
		cur = append(cur, ln)
	}
	if hasNonBlank(cur) {
		chapters = append(chapters, strings.Join(cur, "\n"))
	}
	if len(chapters) == 0 {
		return []string{text}
	}
	return chapters
}

// hasNonBlank reports whether any line in the slice has non-whitespace content.
func hasNonBlank(lines []string) bool {
	for _, ln := range lines {
		if strings.TrimSpace(ln) != "" {
			return true
		}
	}
	return false
}

// --- source encoding detection (Task 2 / D18) ----------------------------------
//
// Real zh .txt are frequently GB18030 (a superset of GBK/GB2312), not UTF-8 — the 蛊真人 acceptance
// book is exactly this case. decodeSourceBytes converts raw source bytes to a UTF-8 string per the
// book's declared encoding, defaulting to a conservative auto-detect. The invariant is fail-LOUD:
// any doubt is an error, never silent mojibake — a corrupt/foreign byte must never reach a U+FFFD
// replacement in ch.Text (which would defeat the lossless-chunker fuzz invariant and mis-key the
// glossary). Dependency: golang.org/x/text/encoding (already in go.mod for NFC) — first use of its
// simplifiedchinese + unicode subpackages (noted in the session journal).
func decodeSourceBytes(raw []byte, encoding, sourceLang string) (string, error) {
	switch enc := strings.ToLower(strings.TrimSpace(encoding)); enc {
	case "", "auto":
		return autoDecodeSource(raw, sourceLang)
	case "utf8", "utf-8":
		b := bytes.TrimPrefix(raw, utf8BOM)
		if !utf8.Valid(b) {
			return "", fmt.Errorf("declared encoding utf8 but the bytes are not valid UTF-8 (declare gb18030, or convert the file)")
		}
		return checkNoNUL(string(b))
	case "gb18030", "gbk", "gb2312":
		// Operator asserted the encoding: decode strictly (fail on undecodable bytes / U+FFFD) but
		// skip the Chinese-plausibility heuristic — they told us it is GB18030.
		return decodeGB18030(raw, false)
	default:
		return "", fmt.Errorf("unknown encoding %q in book.yaml (use auto|utf8|gb18030)", encoding)
	}
}

// checkNoNUL rejects a decoded string that contains a NUL (U+0000). Real prose never contains NUL;
// its presence means a mis-detected / mis-declared encoding: a UTF-16 stream WITHOUT a BOM whose
// ASCII half is valid UTF-8 with interleaved NULs, OR a UTF-16/binary file wrongly fed to the
// GB18030/UTF-16 decoder (0x00 decodes to a VALID U+0000, not U+FFFD, so the replacement-char check
// cannot catch it — self-review major, D20.4). Applied on EVERY decode path (utf8/gb18030/utf16),
// not just UTF-8, so the guard is symmetric across encodings.
func checkNoNUL(s string) (string, error) {
	if strings.IndexByte(s, 0) >= 0 {
		return "", fmt.Errorf("decoded text contains NUL (U+0000) — the source is likely UTF-16 without a BOM, or a binary/mis-declared file; convert it to UTF-8 or declare its encoding correctly")
	}
	return s, nil
}

var (
	utf8BOM    = []byte{0xEF, 0xBB, 0xBF}
	utf16LEBOM = []byte{0xFF, 0xFE}
	utf16BEBOM = []byte{0xFE, 0xFF}
)

// autoDecodeSource is the conservative detection ladder: BOM → valid UTF-8 (+NUL guard) → GB18030
// probe (ZH sources only) → fail-loud. Order matters: a BOM is definitive; valid UTF-8 is
// unambiguous (but rejected if it carries interleaved NUL — UTF-16 without a BOM); GB18030 is tried
// only when the bytes are NOT valid UTF-8 AND the source is Chinese (or unspecified). GB18030 is a
// Chinese encoding that maps almost every byte sequence to Han-shaped codepoints, so a ja Shift-JIS
// / EUC-JP or a European Latin-1 file would decode to CJK-shaped MOJIBAKE that the plausibility
// guard cannot distinguish from real Chinese (self-review CRITICAL). So we auto-probe GB18030 only
// for zh; for ja/en a non-UTF-8 file fails loud and the operator must convert or declare it.
func autoDecodeSource(raw []byte, sourceLang string) (string, error) {
	switch {
	case bytes.HasPrefix(raw, utf8BOM):
		b := raw[len(utf8BOM):]
		if !utf8.Valid(b) {
			return "", fmt.Errorf("file has a UTF-8 BOM but the body is not valid UTF-8 (corrupt file)")
		}
		return checkNoNUL(string(b))
	case bytes.HasPrefix(raw, utf16LEBOM):
		return decodeUTF16(raw, xunicode.LittleEndian)
	case bytes.HasPrefix(raw, utf16BEBOM):
		return decodeUTF16(raw, xunicode.BigEndian)
	}
	if utf8.Valid(raw) {
		return checkNoNUL(string(raw)) // no BOM, already UTF-8 (covers en/ja/UTF-8 zh); NUL ⇒ UTF-16-no-BOM
	}
	// Not valid UTF-8. GB18030 auto-probe is scoped to Chinese sources (see doc above).
	zhScoped := sourceLang == "" || strings.EqualFold(sourceLang, "zh")
	if zhScoped {
		if s, err := decodeGB18030(raw, true); err == nil {
			return s, nil
		}
	}
	probe := "was tried and failed (truncated/corrupt or not GB18030)"
	if !zhScoped {
		probe = fmt.Sprintf("is restricted to zh sources (this source is %q) — a Shift-JIS/EUC-JP/Latin-1 file would decode to Han-shaped mojibake and is NOT auto-detected", sourceLang)
	}
	return "", fmt.Errorf("could not auto-detect the source encoding: the bytes are not valid UTF-8, and the GB18030 probe %s; convert the file to UTF-8 or declare `encoding:` explicitly in book.yaml", probe)
}

// decodeGB18030 decodes raw as GB18030 (superset of GBK/GB2312). It is strict: a decode error, or a
// U+FFFD replacement char in the output (undecodable/truncated bytes), is a failure — broken bytes
// must never reach ch.Text. When probe is true (auto-detect) it additionally requires the decoded
// text to look like Chinese (plausibleCJK), because GB18030 maps almost every byte sequence to
// SOMETHING, so a Latin-1/other file decodes without error into mojibake unless we sanity-check it.
func decodeGB18030(raw []byte, probe bool) (string, error) {
	out, _, err := transform.Bytes(simplifiedchinese.GB18030.NewDecoder(), raw)
	if err != nil {
		return "", fmt.Errorf("GB18030 decode error (not valid GB18030): %w", err)
	}
	s := string(out)
	if strings.ContainsRune(s, utf8.RuneError) {
		return "", fmt.Errorf("GB18030 decode produced replacement chars (U+FFFD) — bytes are truncated/corrupt or not GB18030")
	}
	if probe {
		if err := plausibleCJK(s); err != nil {
			return "", err
		}
	}
	return checkNoNUL(s) // 0x00 bytes decode to a valid U+0000, not U+FFFD — a mis-declared UTF-16/binary file slips the replacement-char check (D20.4)
}

// decodeUTF16 decodes a BOM-prefixed UTF-16 stream (ExpectBOM consumes the BOM). Strict: a decode
// error or a U+FFFD in the output is a failure.
func decodeUTF16(raw []byte, endian xunicode.Endianness) (string, error) {
	dec := xunicode.UTF16(endian, xunicode.ExpectBOM).NewDecoder()
	out, _, err := transform.Bytes(dec, raw)
	if err != nil {
		return "", fmt.Errorf("UTF-16 decode error: %w", err)
	}
	s := string(out)
	if strings.ContainsRune(s, utf8.RuneError) {
		return "", fmt.Errorf("UTF-16 decode produced replacement chars (U+FFFD) — corrupt")
	}
	return checkNoNUL(s) // a genuine UTF-16 stream carrying embedded NULs (or a mis-detected binary) is caught here (D20.4)
}

// plausibleCJK is the GB18030 auto-accept guard: a real GB18030 source is Chinese, so the decoded
// text must be (a) almost entirely CJK-plausible runes (Han/kana/CJK-punct/ASCII/whitespace) and
// (b) genuinely Han-dense. Latin-1/other mojibake fails (b) — it decodes to sparse Han among ASCII
// (café → "caf" + 轳), so every rune is "plausible" in isolation yet Han density is ~0. The Han
// floor is intentionally low (5%) so a heavily Latin-mixed but real Chinese source still passes,
// while mojibake (Han≈0) is rejected loudly. Thresholds bias toward fail-loud (D18 invariant); a
// false reject is recoverable (declare `encoding: gb18030`), a false accept is silent corruption.
func plausibleCJK(s string) error {
	var total, han, plausible int
	for _, r := range s {
		total++
		switch {
		case unicode.Is(unicode.Han, r):
			han++
			plausible++
		case isPlausibleSourceRune(r):
			plausible++
		}
	}
	if total == 0 {
		return fmt.Errorf("GB18030 probe: empty decode")
	}
	if pf := float64(plausible) / float64(total); pf < 0.90 {
		return fmt.Errorf("GB18030 probe: only %.1f%% of decoded runes are CJK-plausible — likely mojibake, not Chinese", 100*pf)
	}
	if hf := float64(han) / float64(total); hf < 0.05 {
		return fmt.Errorf("GB18030 probe: Han density %.2f%% is too low for a Chinese source — likely a mis-detected encoding; declare `encoding` explicitly if this really is GB18030", 100*hf)
	}
	return nil
}

// isPlausibleSourceRune reports whether a rune belongs to the expected repertoire of a zh/ja source
// (excluding Han, which the caller counts separately): ASCII text, kana, CJK/general punctuation,
// and fullwidth/halfwidth forms. Deliberately tight — it must NOT whitelist the Latin-1 supplement
// or arbitrary symbols, or Latin-1 mojibake would pass the plausibility fraction.
func isPlausibleSourceRune(r rune) bool {
	switch {
	case r == '\n', r == '\r', r == '\t', r == ' ':
		return true
	case r >= 0x20 && r <= 0x7E: // printable ASCII
		return true
	case unicode.Is(unicode.Hiragana, r), unicode.Is(unicode.Katakana, r):
		return true
	case r >= 0x3000 && r <= 0x303F: // CJK symbols & punctuation (。、「」〜…)
		return true
	case r >= 0x2010 && r <= 0x206F: // general punctuation (— – ‘’ “” … ‰ etc.)
		return true
	case r >= 0xFF00 && r <= 0xFFEF: // halfwidth & fullwidth forms (！？（）ｱ fullwidth digits/letters)
		return true
	case r == 0x30FC || r == 0x30FB: // prolonged-sound mark ー, middle dot ・ (also in Katakana block)
		return true
	}
	return false
}

// --- epub ----------------------------------------------------------------------

// containerXML is META-INF/container.xml: it points at the OPF package file. Tags
// are matched by LOCAL name (no namespace in the struct tags), so a namespaced or
// prefixed container still binds.
type containerXML struct {
	Rootfiles []struct {
		FullPath  string `xml:"full-path,attr"`
		MediaType string `xml:"media-type,attr"`
	} `xml:"rootfiles>rootfile"`
}

// opfPackage is the OPF: the manifest (id→href→media-type→properties), the spine (reading order, with each
// itemref's `linear`), and the EPUB 2 `<guide>`.
//
// `properties` and `linear` are read because the pack that introduced nav/NCX needs them and nothing else
// supplies them: `properties="nav"` is the ONLY way to find the navigation document, and `linear="no"` marks
// a document that is text but is not part of the reading sequence.
//
// `<guide>` is not legacy trivia. Landmarks — the EPUB 3 way a book declares which document is its toc or
// cover — live INSIDE the navigation document, and an EPUB 2 has no navigation document at all. Reading only
// landmarks would leave every EPUB 2 with its toc page glued to chapter one, which is precisely the mass
// case the exclusion rule exists for.
type opfPackage struct {
	Manifest []struct {
		ID         string `xml:"id,attr"`
		Href       string `xml:"href,attr"`
		MediaType  string `xml:"media-type,attr"`
		Properties string `xml:"properties,attr"`
	} `xml:"manifest>item"`
	Spine struct {
		TOC      string `xml:"toc,attr"` // EPUB 2: the manifest id of the NCX
		ItemRefs []struct {
			IDRef  string `xml:"idref,attr"`
			Linear string `xml:"linear,attr"`
		} `xml:"itemref"`
	} `xml:"spine"`
	Guide []struct {
		Type string `xml:"type,attr"`
		Href string `xml:"href,attr"`
	} `xml:"guide>reference"`
}

// ingestEPUB reads chapters in spine order and captures ruby. Every structural
// problem (bad zip, missing container/opf, empty spine, no readable chapter) is a
// loud error, never a panic (a truncated/foreign epub must not crash a run).
func ingestEPUB(p string) (*Document, error) {
	zr, err := zip.OpenReader(p)
	if err != nil {
		return nil, fmt.Errorf("chunk: ingest epub: open zip %s: %w", p, err)
	}
	defer zr.Close()

	files := map[string]*zip.File{}
	for _, f := range zr.File {
		files[path.Clean(f.Name)] = f
	}

	// 1) container.xml → OPF path.
	cdata, err := readZipEntry(files, "META-INF/container.xml")
	if err != nil {
		return nil, fmt.Errorf("chunk: ingest epub: %w", err)
	}
	var container containerXML
	if err := xml.Unmarshal(cdata, &container); err != nil {
		return nil, fmt.Errorf("chunk: ingest epub: parse container.xml: %w", err)
	}
	opfPath := ""
	for _, rf := range container.Rootfiles {
		if strings.TrimSpace(rf.FullPath) != "" {
			opfPath = path.Clean(rf.FullPath)
			break
		}
	}
	if opfPath == "" {
		return nil, fmt.Errorf("chunk: ingest epub: container.xml has no rootfile full-path")
	}

	// 2) OPF → manifest + spine.
	odata, err := readZipEntry(files, opfPath)
	if err != nil {
		return nil, fmt.Errorf("chunk: ingest epub: %w", err)
	}
	var opf opfPackage
	if err := xml.Unmarshal(odata, &opf); err != nil {
		return nil, fmt.Errorf("chunk: ingest epub: parse opf %s: %w", opfPath, err)
	}
	if len(opf.Spine.ItemRefs) == 0 {
		return nil, fmt.Errorf("chunk: ingest epub: opf %s has an empty spine", opfPath)
	}
	type manifestItem struct{ href, mediaType, properties string }
	byID := map[string]manifestItem{}
	for _, it := range opf.Manifest {
		byID[it.ID] = manifestItem{it.Href, it.MediaType, it.Properties}
	}
	opfDir := path.Dir(opfPath)

	// 3) Spine → documents. A dangling idref is a LOST chapter, not a skip: continuing silently would drop
	// its content AND shift every later chapter's dense number. Collected and failed loud after the loop.
	docs := make([]epubSpineDoc, 0, len(opf.Spine.ItemRefs))
	var dangling []string
	for _, ref := range opf.Spine.ItemRefs {
		item, ok := byID[ref.IDRef]
		if !ok {
			dangling = append(dangling, ref.IDRef)
			continue
		}
		docs = append(docs, epubSpineDoc{
			idref:    ref.IDRef,
			entry:    resolveHref(opfDir, item.href),
			linear:   !strings.EqualFold(strings.TrimSpace(ref.Linear), "no"),
			readable: isXHTML(item.mediaType, item.href),
			service:  hasProperty(item.properties, "nav"),
		})
	}
	if len(dangling) > 0 {
		return nil, fmt.Errorf("chunk: ingest epub: spine references %d manifest id(s) with no manifest item %v — a chapter would be silently lost", len(dangling), dangling)
	}

	// 4) Which spine documents are SERVICE pages rather than book text. The list is exact and short: the
	// navigation document itself, and whatever the book declares as its toc or cover. Nothing else is
	// excluded — everything else in the spine is text, whatever it looks like.
	service := map[string]string{} // entry → the role that made it service
	// serviceInsideDoc counts role declarations this package REFUSED to act on: a guide reference or landmark
	// naming a POINT inside a document ("toc.xhtml#pos") says the table of contents begins there, not that
	// the whole file is one — and since chapters are grouped by document, excluding it would delete whatever
	// prose shares the file. Refusing is the safe half; the count is what keeps the refusal visible.
	serviceInsideDoc := 0
	for _, g := range opf.Guide { // EPUB 2, which has no navigation document to hold landmarks
		if !isServiceRole(g.Type) {
			continue
		}
		h, insideDoc := hrefTarget(opfDir, g.Href)
		switch {
		case h == "":
		case insideDoc:
			serviceInsideDoc++
		default:
			service[h] = strings.ToLower(strings.TrimSpace(g.Type))
		}
	}
	navEntry := ""
	for _, it := range opf.Manifest {
		if hasProperty(it.Properties, "nav") {
			navEntry = resolveHref(opfDir, it.Href)
			break
		}
	}
	var navTargets []string
	labels := map[string]string{}
	tocUnreadable := 0
	if navEntry != "" {
		data, rerr := readZipEntry(files, navEntry)
		if rerr != nil {
			// The manifest SAYS this book has a navigation document. Falling through to the spine without
			// saying so is the silence this counter exists to end.
			tocUnreadable++
		} else {
			targets, navLabels, landmarks, insideDoc := parseNavDoc(data, path.Dir(navEntry))
			navTargets, labels = targets, navLabels
			serviceInsideDoc += insideDoc
			for h, role := range landmarks { // EPUB 3
				if isServiceRole(role) {
					service[h] = strings.ToLower(strings.TrimSpace(role))
				}
			}
		}
	}
	var excluded []ExcludedDocument
	for i := range docs {
		role := service[docs[i].entry]
		if role != "" {
			docs[i].service = true
		} else if docs[i].service {
			role = "nav" // set from properties="nav" when the spine was read
		}
		if !docs[i].service || !docs[i].readable {
			continue
		}
		// Read it — only to SIZE it. An exclusion no gate can second-guess should at least say how much text
		// it took with it.
		runes := 0
		if data, rerr := readZipEntry(files, docs[i].entry); rerr == nil {
			if body, _, xerr := extractXHTML(data); xerr == nil {
				runes = utf8.RuneCountInString(text.NormalizeSource(body))
			}
		}
		excluded = append(excluded, ExcludedDocument{Entry: docs[i].entry, Role: role, Runes: runes})
	}

	// 5) The table of contents, nav → NCX → spine. Each witness is tried only if the previous resolved to
	// nothing: a nav whose targets all dangle is a broken nav, not a book with no chapters.
	toc := resolveTOC(tocNav, navTargets, docs)
	if toc.kind == tocNone {
		// A nav that resolved to nothing still SAID something, and how much of it dangled is a fact about the
		// book. Falling through to the NCX must not erase that count along with the failed table.
		navUnresolved := toc.unresolved
		if ncxEntry := ncxPath(opf, opfDir); ncxEntry != "" {
			data, rerr := readZipEntry(files, ncxEntry)
			switch {
			case rerr != nil:
				tocUnreadable++ // the spine names an NCX and its bytes are not there
			default:
				targets, ncxLabels, perr := parseNCX(data, path.Dir(ncxEntry))
				if perr != nil {
					tocUnreadable++ // the bytes are there and are not an NCX
				}
				if ncx := resolveTOC(tocNCX, targets, docs); ncx.kind == tocNCX {
					// Switching tables: carry the nav's dangling count across, or it vanishes with the table.
					toc, labels = ncx, ncxLabels
					toc.unresolved += navUnresolved
				} else {
					toc.unresolved += ncx.unresolved
				}
			}
		}
	}
	starts := toc.starts
	witness := StructureDeclared
	if toc.collapsed > 0 {
		// The format named more chapters than this package can hand back, so the cut is no longer the
		// format's statement — see epubTOC.collapsed.
		witness = StructureDelimited
	}
	if toc.kind == tocNone {
		// No table of contents resolved: fall back to the spine, where every readable document opens its own
		// chapter. That is the assumption this package has always made, and it is now NAMED as one.
		starts = starts[:0]
		for i, d := range docs {
			if d.readable && !d.service && d.linear {
				starts = append(starts, i)
			}
		}
		witness = StructureDelimited
	}

	// 6) Group, then read. Reading happens per GROUP so a chapter split across documents arrives whole.
	groups, attached := groupChapters(docs, starts)
	// ⛔ CONSERVATION, checked before a single paid call can be reached. The counts this reader keeps are
	// OBSERVABILITY — they say what happened once you go looking. This is the GUARANTEE: every readable spine
	// document is either excluded ON PURPOSE or lands in exactly one chapter, and nothing else is possible.
	//
	// It exists because the two defects that mattered most in this reader were both invisible to counts: a
	// service role declared at a point inside a document deleted the whole file, and the chapter total stayed
	// plausible while the provenance stayed a confident `declared`. An invariant catches that class by ANY
	// route — the fragment of yesterday and whatever tomorrow's is — and it costs nothing to check.
	placed := map[int]bool{}
	for _, g := range groups {
		for _, i := range g.docs {
			if placed[i] {
				return nil, fmt.Errorf("chunk: ingest epub: spine document %q landed in two chapters — the cut is not a partition", docs[i].entry)
			}
			placed[i] = true
		}
	}
	for i, d := range docs {
		if !d.readable {
			continue
		}
		if d.service == placed[i] {
			// Either a readable document is BOTH excluded and placed, or it is neither — text left the reading
			// without anybody deciding to drop it.
			return nil, fmt.Errorf("chunk: ingest epub: spine document %q is neither excluded nor part of a chapter (excluded=%v placed=%v) — book text would be lost silently",
				docs[i].entry, d.service, placed[i])
		}
	}
	doc := &Document{
		DocumentsAttached: attached, Excluded: excluded,
		TOCUnreadable: tocUnreadable, TOCUnresolved: toc.unresolved, TargetsCollapsed: toc.collapsed,
		ServiceRolesInsideDocuments: serviceInsideDoc,
	}
	denseNo := 0 // matches SplitChunks: only a NON-empty chapter takes a number
	read := 0
	for _, g := range groups {
		// The chapter's source title is the label of the document that OPENED it.
		title := ""
		if g.start >= 0 {
			title = labels[docs[g.start].entry]
		}
		doc.Titles = append(doc.Titles, title)
		var parts []string
		var ruby []RubyReading
		for _, i := range g.docs {
			data, rerr := readZipEntry(files, docs[i].entry)
			if rerr != nil {
				return nil, fmt.Errorf("chunk: ingest epub: spine item %q → %s: %w", docs[i].idref, docs[i].entry, rerr)
			}
			read++
			body, rb, xerr := extractXHTML(data)
			if xerr != nil {
				return nil, fmt.Errorf("chunk: ingest epub: extract %s: %w", docs[i].entry, xerr)
			}
			if b := text.NormalizeSource(body); b != "" {
				parts = append(parts, b)
			}
			ruby = append(ruby, rb...)
		}
		norm := strings.Join(parts, "\n\n")
		// The rune half of the same invariant: a chapter is EXACTLY its documents plus the separators this
		// function inserted between them. A body silently dropped inside the join would keep every count
		// right and every document "placed".
		want := 0
		for _, b := range parts {
			want += utf8.RuneCountInString(b)
		}
		if sep := 2 * (len(parts) - 1); len(parts) > 0 && utf8.RuneCountInString(norm) != want+sep {
			return nil, fmt.Errorf("chunk: ingest epub: chapter %d holds %d runes but its %d document(s) carry %d + %d separator(s) — text was lost in the join",
				len(doc.Chapters)+1, utf8.RuneCountInString(norm), len(parts), want, sep)
		}
		doc.Chapters = append(doc.Chapters, norm)
		// The chapter number ruby carries MUST equal the number SplitChunks assigns (dense — empty chapters
		// are skipped), otherwise memory v2's since_ch is off by the count of empty chapters before it.
		if len(splitParagraphs(norm)) == 0 {
			continue
		}
		denseNo++
		for i := range ruby {
			ruby[i].Chapter = denseNo
		}
		doc.Ruby = append(doc.Ruby, ruby...)
	}
	if read == 0 {
		return nil, fmt.Errorf("chunk: ingest epub: spine resolved to no readable (x)html chapters")
	}
	doc.Structure = structureOf(doc.Chapters, witness)
	return doc, nil
}

// isXHTML reports whether a manifest item is an (x)html content document. It
// checks the media-type (parameters like "; charset=utf-8" stripped), accepting
// the (x)html types AND the generic XML types real epubs mislabel xhtml chapters
// with (application/xml, text/xml), then falls back to a content-document href
// extension for ANY unrecognized/absent type. A genuine non-content asset keeps its
// own extension (.jpg/.css/.ncx/.svg), so the fallback does not misclassify it.
// (External-review #2: a chapter typed application/xml with a .xml href used to slip
// past BOTH the type switch and a .xhtml-only extension fallback and vanish silently.)
func isXHTML(mediaType, href string) bool {
	mt := strings.ToLower(strings.TrimSpace(mediaType))
	if i := strings.IndexByte(mt, ';'); i >= 0 {
		mt = strings.TrimSpace(mt[:i])
	}
	switch mt {
	case "application/xhtml+xml", "text/html", "application/html",
		"application/xml", "text/xml":
		return true
	}
	switch extOf(href) {
	case ".xhtml", ".html", ".htm", ".xml":
		return true
	}
	return false
}

// resolveHref joins an OPF-relative href to the OPF directory (percent-decoded,
// slash-cleaned) into a zip entry path.
func resolveHref(opfDir, href string) string {
	doc, _ := hrefTarget(opfDir, href)
	return doc
}

// readZipEntry reads a zip entry by cleaned path.
func readZipEntry(files map[string]*zip.File, name string) ([]byte, error) {
	f, ok := files[path.Clean(name)]
	if !ok {
		return nil, fmt.Errorf("missing zip entry %q", name)
	}
	rc, err := f.Open()
	if err != nil {
		return nil, fmt.Errorf("open zip entry %q: %w", name, err)
	}
	defer rc.Close()
	return io.ReadAll(rc)
}

// --- xhtml text + ruby extraction ----------------------------------------------

// blockTags emit a paragraph break (blank line) so the chunker's paragraph packing
// survives extraction; without them a whole chapter collapses into one paragraph.
var blockTags = map[string]bool{
	"p": true, "div": true, "h1": true, "h2": true, "h3": true, "h4": true,
	"h5": true, "h6": true, "li": true, "blockquote": true, "section": true,
	"article": true, "tr": true, "table": true, "ul": true, "ol": true, "dl": true,
	"dd": true, "dt": true, "pre": true, "figure": true, "figcaption": true,
	"header": true, "footer": true, "aside": true, "nav": true, "hr": true,
	"main": true, "td": true, "th": true, "caption": true,
}

// skipRoots are subtrees whose text is not prose (CSS/JS/metadata).
var skipRoots = map[string]bool{"script": true, "style": true, "head": true}

// rawTextTags are the elements whose content the tokenizer hands back verbatim instead of lexing
// it. That is exactly what we want for a <script>/<style> subtree we drop anyway — it is why
// markup-shaped JS no longer breaks the parse. Everywhere else raw mode is wrong: it would put
// literal tags into the prose, and after a SELF-CLOSED raw-text tag (<script src=".."/>, a common
// xhtml spelling) no closing tag ever arrives, so the whole rest of the chapter would be swallowed
// onto the wire as markup. Those cases are switched back to normal lexing.
var rawTextTags = map[string]bool{
	"script": true, "style": true, "title": true, "textarea": true, "iframe": true,
	"noscript": true, "noembed": true, "noframes": true, "xmp": true, "plaintext": true,
}

// voidTags never have content, so the tokenizer emits ONE token for them and no end tag. The
// extractor closes them itself, which is also what encoding/xml did via HTMLAutoClose — that
// parity is what keeps <hr/> and <br/> extracting byte-for-byte as before.
var voidTags = map[string]bool{
	"area": true, "base": true, "br": true, "col": true, "embed": true, "hr": true,
	"img": true, "input": true, "link": true, "meta": true, "param": true,
	"source": true, "track": true, "wbr": true,
}

// xmlDeclCharset picks the encoding out of a leading <?xml … encoding="…"?> declaration.
var xmlDeclCharset = regexp.MustCompile(`(?is)^\s*<\?xml\s[^>]*?encoding\s*=\s*["']([^"']+)["']`)

// checkXHTMLCharset keeps the epub-v1 = UTF-8 contract that the xml decoder's CharsetReader used
// to enforce: a document DECLARING another encoding is a loud error, never silently mis-decoded
// prose. Only the declaration is inspected, exactly as before — a <meta charset> is not consulted.
func checkXHTMLCharset(data []byte) error {
	head := bytes.TrimPrefix(data, utf8BOM)
	if len(head) > 512 {
		head = head[:512]
	}
	m := xmlDeclCharset.FindSubmatch(head)
	if m == nil {
		return nil
	}
	switch cs := strings.ToLower(strings.TrimSpace(string(m[1]))); cs {
	case "", "utf-8", "utf8", "us-ascii", "ascii":
		return nil
	default:
		return fmt.Errorf("unsupported xhtml charset %q (epub v1 expects UTF-8)", cs)
	}
}

var cdataOpen, cdataClose = []byte("<![CDATA["), []byte("]]>")

// unwrapCDATA turns XML CDATA sections into their escaped text before tokenizing. The HTML
// tokenizer has no CDATA: it lexes «<![CDATA[…]]>» as a bogus comment, dropping part of the
// content and leaking «]]>» into the prose. Unwrapping keeps what the xml decoder gave — CDATA
// content is literal text — so a chapter that used to import cleanly still extracts byte-for-byte.
func unwrapCDATA(data []byte) []byte {
	if !bytes.Contains(data, cdataOpen) {
		return data
	}
	var out bytes.Buffer
	rest := data
	for {
		i := bytes.Index(rest, cdataOpen)
		if i < 0 {
			out.Write(rest)
			return out.Bytes()
		}
		out.Write(rest[:i])
		rest = rest[i+len(cdataOpen):]
		j := bytes.Index(rest, cdataClose)
		if j < 0 { // unterminated section: the remainder is literal text, as XML would have it
			out.WriteString(html.EscapeString(string(rest)))
			return out.Bytes()
		}
		out.WriteString(html.EscapeString(string(rest[:j])))
		rest = rest[j+len(cdataClose):]
	}
}

// tokenTagName is the token's element name: already lower-cased by the tokenizer, with any
// namespace prefix dropped (<epub:switch> → "switch") so matching stays on the LOCAL name the
// xml decoder used to report.
func tokenTagName(z *html.Tokenizer) string {
	raw, _ := z.TagName()
	name := string(raw)
	if i := strings.IndexByte(name, ':'); i >= 0 {
		name = name[i+1:]
	}
	return name
}

// extractXHTML strips tags to text and captures ruby. Rules:
//   - block tag → blank line; <br> → newline (paragraph structure for the chunker);
//   - <ruby>base<rt>reading</rt></ruby>: BASE goes to the body, READING is captured
//     as a (base,reading) pair, <rp> parenthesis fallbacks are dropped;
//   - <script>/<style>/<head> subtrees are dropped.
//
// It runs on the x/net/html TOKENIZER (a lexer, not the tree parser): real epubs ship xhtml that
// is not well-formed XML, and encoding/xml aborted the whole chapter on any of it — a bare «<» in
// prose, mis-nested tags, markup-looking <script> content, or «--» inside a comment. The tokenizer
// never fails on those, so a dirty chapter imports instead of killing the run. A non-UTF-8 declared
// charset stays a loud error (epub v1 = UTF-8).
// Ruby is returned with Chapter unset (0); the caller stamps the dense chapter no.
func extractXHTML(data []byte) (string, []RubyReading, error) {
	if err := checkXHTMLCharset(data); err != nil {
		return "", nil, err
	}
	z := html.NewTokenizer(bytes.NewReader(unwrapCDATA(data)))

	var body strings.Builder
	var ruby []RubyReading
	var base, reading strings.Builder
	// Ruby sub-state is a MODE, not a nesting depth. HTML5 makes </rt>, </rp> and </rb> OPTIONAL —
	// a sibling or the closing </ruby> ends them — and the tokenizer does not synthesise those
	// implied end tags the way non-strict encoding/xml did. With counters, one omitted </rt> leaks
	// forever and every LATER base in the chapter is routed into the reading buffer, i.e. deleted
	// from the prose. A mode cannot leak: every sibling and every </ruby> resets it.
	const (
		rubyBase = iota
		rubyRT
		rubyRP
	)
	rubyDepth, rubyPart := 0, rubyBase
	// A skipped subtree is tracked BY NAME, not by raw nesting depth: the tokenizer emits no end
	// tag for void/self-closing children (<meta/>, <link/> in <head>), so a blind depth counter
	// would never return to zero and would swallow the rest of the chapter.
	skip, skipDepth := "", 0

	// flushRuby closes the current ruby, emitting its (base, reading) if both are non-empty, and
	// clears ALL ruby state. Called on </ruby> and at a block boundary — ruby is inline, so an
	// unclosed <ruby> must not outlive its paragraph (left open it suppresses every later
	// paragraph break and swallows the rest of the chapter into one run-on block).
	flushRuby := func() {
		b := strings.TrimSpace(base.String())
		r := strings.TrimSpace(reading.String())
		if b != "" && r != "" {
			ruby = append(ruby, RubyReading{Base: b, Reading: r})
		}
		base.Reset()
		reading.Reset()
		rubyDepth, rubyPart = 0, rubyBase
	}

	start := func(name string) {
		if skip != "" {
			switch {
			case skip == "head" && name == "body":
				skip, skipDepth = "", 0 // an unclosed <head>: <body> ends it rather than eating the chapter
			case name == skip:
				skipDepth++
			}
			return
		}
		switch {
		case skipRoots[name]:
			skip, skipDepth = name, 1
		case name == "ruby":
			rubyDepth++
			if rubyDepth == 1 {
				base.Reset()
				reading.Reset()
			}
			rubyPart = rubyBase
		case name == "rt":
			rubyPart = rubyRT
		case name == "rp":
			rubyPart = rubyRP
		case name == "rb":
			rubyPart = rubyBase
		case name == "br" && rubyDepth == 0:
			body.WriteByte('\n')
		case blockTags[name]:
			if rubyDepth > 0 {
				flushRuby()
			}
			body.WriteString("\n\n")
		}
	}

	end := func(name string) {
		if skip != "" {
			if name == skip {
				if skipDepth--; skipDepth <= 0 {
					skip, skipDepth = "", 0
				}
			}
			return
		}
		switch {
		case name == "ruby":
			if rubyDepth > 0 {
				rubyDepth--
				rubyPart = rubyBase
				if rubyDepth == 0 {
					flushRuby()
				}
			}
		case name == "rt", name == "rp", name == "rb":
			rubyPart = rubyBase
		case blockTags[name]:
			if rubyDepth > 0 {
				flushRuby()
			}
			body.WriteString("\n\n")
		}
	}

	for {
		tt := z.Next()
		if tt == html.ErrorToken {
			if err := z.Err(); !errors.Is(err, io.EOF) {
				return "", nil, fmt.Errorf("tokenize xhtml: %w", err)
			}
			break
		}
		switch tt {
		case html.StartTagToken, html.SelfClosingTagToken:
			name := tokenTagName(z)
			// Keep raw text ONLY for a script/style subtree we are about to drop whole.
			if rawTextTags[name] && (tt == html.SelfClosingTagToken || !skipRoots[name]) {
				z.NextIsNotRawText()
			}
			start(name)
			// A void or self-closing element has no content and no end tag of its own.
			if tt == html.SelfClosingTagToken || voidTags[name] {
				end(name)
			}
		case html.EndTagToken:
			end(tokenTagName(z))
		case html.TextToken:
			if skip != "" {
				continue // <script>/<style>/<head> subtree — not prose
			}
			text := string(z.Text())
			if rubyDepth > 0 {
				switch {
				case rubyPart == rubyRP:
					// ruby parenthesis fallback — drop.
				case rubyPart == rubyRT:
					reading.WriteString(text)
				case strings.TrimSpace(text) == "":
					// Formatting whitespace BETWEEN ruby base segments (pretty-printed
					// jukugo <rb>旧</rb> <rb>字</rb> …) must not enter the captured base
					// (it would mis-key the glossary reading) nor the body (it would
					// inject stray whitespace between CJK glyphs into ch.Text) —
					// self-review finding. Drop it; real base glyphs are non-whitespace.
				default:
					base.WriteString(text)
					body.WriteString(text)
				}
				continue
			}
			body.WriteString(text)
		}
		// CommentToken / DoctypeToken carry no prose — dropped.
	}
	return body.String(), ruby, nil
}