textmachine/backend/internal/chunk/epubtoc.go

package chunk

import (
	"bytes"
	"encoding/xml"
	"net/url"
	"path"
	"sort"
	"strings"

	"golang.org/x/net/html"
)

// The EPUB table of contents, and why the spine is not one.
//
// A spine states READING ORDER: the sequence a reader is walked through. A nav (EPUB 3) or an NCX (EPUB 2)
// states the CHAPTER STRUCTURE the book itself claims. Reading the spine as a chapter list is an assumption
// — right for the common one-document-per-chapter build, wrong for the many that split a chapter across
// documents or pack several into one, and wrong for every service page that rides the spine.
//
// Precedence is nav → NCX → spine, and a nav whose targets all fail to resolve counts as ABSENT rather than
// as an empty table of contents: a broken file must fall through to the next witness, not produce zero
// chapters.

// tocKind names which witness drew the boundaries, so the caller can report provenance without re-deriving
// it.
type tocKind int

const (
	tocNone tocKind = iota
	tocNav
	tocNCX
)

// epubSpineDoc is one entry of the spine, after the manifest has been resolved.
type epubSpineDoc struct {
	idref  string
	entry  string // resolved zip path
	linear bool
	// service marks a document that is not book text: the nav document itself, or one the book declares as
	// its toc or cover. Excluded from the reading WITH a count.
	service bool
	// readable is false for a spine item that is not an (x)html content document (an image, a stylesheet).
	readable bool
}

// epubTOC is a resolved table of contents: the spine indices that START a chapter, already filtered,
// de-duplicated and ordered.
type epubTOC struct {
	kind tocKind
	// starts are spine indices, unique, kept ascending. Read as a SET — see resolveTOC.
	starts []int
	// unresolved counts targets that named NOTHING IN THE SPINE — a broken table of contents, which is a real
	// property of scraped EPUBs and worth an alarm.
	//
	// ⚠ It deliberately does NOT count a target that resolved fine but landed on a service page or a
	// non-linear one. Those are ordinary and expected — an EPUB 2 whose toc lists its own cover would report
	// a "broken" table of contents on every normal book, and an alarm that fires on the normal case is one
	// nobody reads. Their exclusion is already counted, once, as DocumentsExcluded.
	unresolved int
	// collapsed counts targets that landed in a document another target had already claimed.
	//
	// ⛔ IT DOWNGRADES THE PROVENANCE, not just the report. A book packing three chapters into one document
	// with anchors declares THREE; this package hands back ONE, because it groups whole documents. Calling
	// that `declared` would put the engine's own coarser answer out under the format's name — the exact lie
	// the spine used to tell. The file did delimit its documents, so `delimited` is the true word for it.
	collapsed int
}

// hasProperty reports whether an OPF `properties` attribute carries a token. Token-wise, not substring-wise:
// the attribute is a space-separated list, and `cover-image` must not read as `cover`.
func hasProperty(properties, want string) bool {
	for _, tok := range strings.Fields(properties) {
		if tok == want {
			return true
		}
	}
	return false
}

// hrefTarget resolves an OPF or TOC href against the directory of the file that carried it, returning the
// DOCUMENT it names and whether it pointed at a place INSIDE that document rather than at the whole of it.
//
// ⛔ THE ORDER IS NOT INTERCHANGEABLE, and the two halves of this package used to disagree about it. The '#'
// that opens a fragment exists in the ENCODED form; a literal '#' in a file name arrives as %23. Unescape
// first and that literal becomes a delimiter, truncating the name — which is what the spine resolver did
// while the TOC resolver did the reverse. They are compared against each other, so they parted company on
// exactly the files nobody writes a test for, and the disagreement is silent.
//
// insideDoc is not a detail: a `<reference type="toc" href="c2.xhtml#pos"/>` says the table of contents
// BEGINS there, inside a document that may hold prose before and after it. Reading that as "the whole
// document is the table of contents" deletes book text.
func hrefTarget(baseDir, href string) (doc string, insideDoc bool) {
	h := strings.TrimSpace(href)
	if i := strings.IndexByte(h, '#'); i >= 0 {
		h, insideDoc = h[:i], true
	}
	if h == "" {
		return "", insideDoc
	}
	if dec, err := url.PathUnescape(h); err == nil {
		h = dec
	}
	return path.Clean(path.Join(baseDir, h)), insideDoc
}

// parseNavDoc reads an EPUB 3 navigation document, returning the toc targets in document order and the
// landmark roles it declares (resolved zip path → epub:type).
//
// Nested <ol> are FLATTENED by construction: every <a href> inside the toc nav is a boundary, whatever its
// depth. A book's volume/part hierarchy is a rendering question, and treating a nested entry as "not really a
// chapter" would drop text into whichever ancestor happened to be shallower.
func parseNavDoc(data []byte, baseDir string) (toc []string, labels map[string]string, landmarks map[string]string, rolesInsideDoc int) {
	labels = map[string]string{}
	landmarks = map[string]string{}
	z := html.NewTokenizer(bytes.NewReader(data))
	navType := "" // epub:type of the <nav> currently open
	depth := 0
	pending := "" // href of the <a> whose text is being collected
	for {
		switch z.Next() {
		case html.ErrorToken:
			return toc, labels, landmarks, rolesInsideDoc
		case html.TextToken:
			if pending != "" {
				labels[pending] += string(z.Text())
			}
		case html.StartTagToken, html.SelfClosingTagToken:
			name, hasAttr := z.TagName()
			n := string(name)
			attrs := map[string]string{}
			for hasAttr {
				k, v, more := z.TagAttr()
				attrs[string(k)] = string(v)
				hasAttr = more
			}
			switch {
			case n == "nav":
				navType = attrs["epub:type"]
				if navType == "" {
					navType = attrs["type"]
				}
				depth = 1
			case n == "a" && depth > 0:
				href, insideDoc := hrefTarget(baseDir, attrs["href"])
				if href == "" {
					continue
				}
				switch navType {
				case "toc":
					toc = append(toc, href)
					if _, seen := labels[href]; !seen {
						pending = href // the FIRST entry naming a document supplies its title
					}
				case "landmarks":
					role := attrs["epub:type"]
					if role == "" {
						role = attrs["type"]
					}
					// A landmark pointing INSIDE a document marks a place, not the document. Excluding the
					// whole of it on that basis deletes whatever prose shares the file. Counted, not merely
					// skipped: the guide half counts its refusals, and a refusal visible on one path and
					// silent on the other is the same asymmetry this pack exists to remove.
					switch {
					case insideDoc && isServiceRole(role):
						rolesInsideDoc++
					case !insideDoc:
						landmarks[href] = role
					}
				}
			}
		case html.EndTagToken:
			switch name, _ := z.TagName(); string(name) {
			case "a":
				pending = ""
			case "nav":
				navType, depth, pending = "", 0, ""
			}
		}
	}
}

// ncxDoc is the EPUB 2 navigation map. navPoints nest, and the nesting is flattened for the same reason the
// nav's <ol> is.
type ncxDoc struct {
	Points []ncxPoint `xml:"navMap>navPoint"`
}

type ncxPoint struct {
	Label struct {
		Text string `xml:"text"`
	} `xml:"navLabel"`
	Content struct {
		Src string `xml:"src,attr"`
	} `xml:"content"`
	Points []ncxPoint `xml:"navPoint"`
}

func (p ncxPoint) flatten(baseDir string, out *[]string, labels map[string]string) {
	if h, _ := hrefTarget(baseDir, p.Content.Src); h != "" {
		*out = append(*out, h)
		if _, seen := labels[h]; !seen {
			labels[h] = strings.TrimSpace(p.Label.Text)
		}
	}
	for _, c := range p.Points {
		c.flatten(baseDir, out, labels)
	}
}

// parseNCX reads an EPUB 2 NCX, returning its navPoint targets in document order.
func parseNCX(data []byte, baseDir string) (targets []string, labels map[string]string) {
	labels = map[string]string{}
	var d ncxDoc
	if err := xml.Unmarshal(data, &d); err != nil {
		return nil, labels
	}
	for _, p := range d.Points {
		p.flatten(baseDir, &targets, labels)
	}
	return targets, labels
}

// resolveTOC maps TOC hrefs onto spine positions.
//
// A table of contents may list its entries in any order, and reading order is the SPINE's to state. That is
// why grouping walks the spine and treats these positions as a SET: a mis-ordered toc cannot interleave
// chapters, because nothing ever iterates the toc's own sequence. ⚠ The sort below is therefore NOT
// load-bearing — reversing it changes no output, verified by mutation — it keeps the field's declared
// ascending order true for a reader, and that is all it does.
//
// Targets naming a document outside the spine, or one that is not readable, or a non-linear one, are dropped
// and counted — and a non-linear document is dropped as a BOUNDARY specifically: a footnote appendix the toc
// points at is still text, and it attaches to what precedes it rather than opening a chapter.
func resolveTOC(kind tocKind, targets []string, docs []epubSpineDoc) epubTOC {
	pos := map[string]int{}
	for i, d := range docs {
		if _, seen := pos[d.entry]; !seen {
			pos[d.entry] = i
		}
	}
	seen := map[int]bool{}
	out := epubTOC{kind: kind}
	for _, t := range targets {
		i, ok := pos[t]
		if !ok || !docs[i].readable {
			out.unresolved++ // the toc names something this book does not read
			continue
		}
		if docs[i].service || !docs[i].linear {
			continue // resolved, and deliberately not a boundary — see the field
		}
		if seen[i] {
			out.collapsed++ // two toc entries in ONE document: one boundary here, and the cut is coarser
			continue
		}
		seen[i] = true
		out.starts = append(out.starts, i)
	}
	sort.Ints(out.starts)
	if len(out.starts) == 0 {
		out.kind = tocNone
	}
	return out
}

// chapterGroup is one chapter: the spine documents it is made of, and the index of the document that OPENED
// it. The opener is recorded rather than derived — it is the FIRST document only when nothing was folded in
// ahead of it, and a caller reading its title off the wrong end gets a title that is subtly, silently wrong.
type chapterGroup struct {
	docs  []int
	start int // -1 when no boundary opened this chapter
}

// groupChapters assigns every readable non-service spine document to a chapter, returning the groups in
// reading order and how many documents were ATTACHED rather than opening a chapter of their own.
//
// ⛔ NO DOCUMENT IS DROPPED HERE. Uncovered documents before the first boundary (a title page, a foreword the
// TOC skips) join the FIRST chapter — the mirror of the txt preamble rule; every other uncovered document
// joins the PREVIOUS one. Service documents were already removed by the caller, with their own count, and
// they are the only text this package ever declines to read.
func groupChapters(docs []epubSpineDoc, starts []int) (groups []chapterGroup, attached int) {
	isStart := map[int]bool{}
	for _, s := range starts {
		isStart[s] = true
	}
	var pre []int // documents seen before the first boundary
	for i, d := range docs {
		if !d.readable || d.service {
			continue
		}
		switch {
		case isStart[i]:
			g := chapterGroup{docs: append(append([]int{}, pre...), i), start: i}
			attached += len(pre)
			pre = nil
			groups = append(groups, g)
		case len(groups) == 0:
			pre = append(pre, i) // held back: it belongs to chapter ONE, not to a chapter of its own
		default:
			last := &groups[len(groups)-1]
			last.docs = append(last.docs, i)
			attached++
		}
	}
	if len(pre) > 0 { // no boundary ever fired: the whole book is one chapter
		groups = append(groups, chapterGroup{docs: pre, start: -1})
		attached += len(pre) - 1
	}
	return groups, attached
}

// isServiceRole reports whether a declared role marks a document as a service page rather than book text.
// EXACTLY two roles qualify, and the shortness is the point: everything else in a spine is text, and a
// broader rule would silently delete chapters that merely look structural.
func isServiceRole(role string) bool {
	switch strings.ToLower(strings.TrimSpace(role)) {
	case "toc", "cover":
		return true
	}
	return false
}

// ncxPath finds the EPUB 2 navigation document: the manifest item the spine's `toc` attribute names, or —
// when that attribute is missing, as it is in books built by tools that half-migrated to EPUB 3 — the first
// item carrying the NCX media type.
func ncxPath(opf opfPackage, opfDir string) string {
	if id := strings.TrimSpace(opf.Spine.TOC); id != "" {
		for _, it := range opf.Manifest {
			if it.ID == id {
				return path.Clean(path.Join(opfDir, it.Href))
			}
		}
	}
	for _, it := range opf.Manifest {
		if strings.Contains(strings.ToLower(it.MediaType), "dtbncx") {
			return path.Clean(path.Join(opfDir, it.Href))
		}
	}
	return ""
}