textmachine/backend/internal/pipeline/ingest.go

package pipeline

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"net/url"
	"os"
	"path"
	"strings"
)

// ingest.go: the шаг-3a import layer. Ingest turns a source file (txt or epub)
// into an ordered list of per-chapter NORMALIZED text plus the ruby/furigana
// readings captured on the way (04-unhappy §4 / D9). It is the ONLY place that
// reads the source; the runner then feeds doc.Chapters to SplitChunks and persists
// doc.Ruby (runner.go). It is offline and deterministic — no LLM, no time/rand — so
// the whole path is $0 and reproducible.
//
// epub v1 is a text extraction (02-mvp:25): read the chapters in spine order,
// strip tags, drop inline markup/styling. The ONE thing we do NOT drop is ruby:
// <ruby>base<rt>reading</rt></ruby> carries an author reading of a name/term that a
// plain text export would silently lose (the documented hole, 04-unhappy §4). We
// keep the BASE in the body and hand the READING to the caller for a glossary lock
// (memory v2, шаг 4) — never injecting it into a prompt here (§7d).

// chapterSep splits a TXT source into chapters. Form feed (U+000C) is the ASCII
// "page/section break": semantically a chapter boundary, invisible in prose, and
// untouched by NormalizeSource. A txt with no \f is a single chapter (Фаза-0 backward
// compat: the one-file example stays one chapter). epub carries real chapter
// structure in its spine, so it does not use this.
const chapterSep = "\f"

// RubyReading is one captured (base, reading) pair and the chapter it appeared in.
// The pipeline aggregates these (min chapter, count) before persisting (runner.go).
type RubyReading struct {
	Base    string // the ruby body: the kanji/base surface form
	Reading string // the <rt> reading (furigana)
	Chapter int    // 1-based chapter where this occurrence was found
}

// Document is the ingested source: per-chapter normalized text (in reading order)
// plus every ruby occurrence found. Chapters is what SplitChunks consumes.
type Document struct {
	Chapters []string
	Ruby     []RubyReading
}

// Ingest reads a source file and returns its Document. Dispatch is by extension:
// .epub → the epub reader; anything else → plain text (the example uses .txt; an
// unknown extension is treated as text rather than rejected).
func Ingest(p string) (*Document, error) {
	if strings.EqualFold(extOf(p), ".epub") {
		return ingestEPUB(p)
	}
	return ingestTXT(p)
}

// extOf returns the lower-cased file extension incl. the dot ("" if none).
func extOf(p string) string {
	i := strings.LastIndexByte(p, '.')
	if i < 0 || strings.ContainsAny(p[i:], "/\\") {
		return ""
	}
	return strings.ToLower(p[i:])
}

// ingestTXT reads a plain-text source: split on \f into chapters, NormalizeSource
// each (BOM/CRLF/NFC/outer-trim). No ruby in txt.
func ingestTXT(p string) (*Document, error) {
	raw, err := os.ReadFile(p)
	if err != nil {
		return nil, fmt.Errorf("pipeline: ingest txt: %w", err)
	}
	var chapters []string
	for _, part := range strings.Split(string(raw), chapterSep) {
		chapters = append(chapters, NormalizeSource(part))
	}
	return &Document{Chapters: chapters}, nil
}

// --- epub ----------------------------------------------------------------------

// containerXML is META-INF/container.xml: it points at the OPF package file. Tags
// are matched by LOCAL name (no namespace in the struct tags), so a namespaced or
// prefixed container still binds.
type containerXML struct {
	Rootfiles []struct {
		FullPath  string `xml:"full-path,attr"`
		MediaType string `xml:"media-type,attr"`
	} `xml:"rootfiles>rootfile"`
}

// opfPackage is the OPF: the manifest (id→href→media-type) and the spine (the
// reading order of itemref idrefs).
type opfPackage struct {
	Manifest []struct {
		ID        string `xml:"id,attr"`
		Href      string `xml:"href,attr"`
		MediaType string `xml:"media-type,attr"`
	} `xml:"manifest>item"`
	Spine []struct {
		IDRef string `xml:"idref,attr"`
	} `xml:"spine>itemref"`
}

// ingestEPUB reads chapters in spine order and captures ruby. Every structural
// problem (bad zip, missing container/opf, empty spine, no readable chapter) is a
// loud error, never a panic (a truncated/foreign epub must not crash a run).
func ingestEPUB(p string) (*Document, error) {
	zr, err := zip.OpenReader(p)
	if err != nil {
		return nil, fmt.Errorf("pipeline: ingest epub: open zip %s: %w", p, err)
	}
	defer zr.Close()

	files := map[string]*zip.File{}
	for _, f := range zr.File {
		files[path.Clean(f.Name)] = f
	}

	// 1) container.xml → OPF path.
	cdata, err := readZipEntry(files, "META-INF/container.xml")
	if err != nil {
		return nil, fmt.Errorf("pipeline: ingest epub: %w", err)
	}
	var container containerXML
	if err := xml.Unmarshal(cdata, &container); err != nil {
		return nil, fmt.Errorf("pipeline: ingest epub: parse container.xml: %w", err)
	}
	opfPath := ""
	for _, rf := range container.Rootfiles {
		if strings.TrimSpace(rf.FullPath) != "" {
			opfPath = path.Clean(rf.FullPath)
			break
		}
	}
	if opfPath == "" {
		return nil, fmt.Errorf("pipeline: ingest epub: container.xml has no rootfile full-path")
	}

	// 2) OPF → manifest + spine.
	odata, err := readZipEntry(files, opfPath)
	if err != nil {
		return nil, fmt.Errorf("pipeline: ingest epub: %w", err)
	}
	var opf opfPackage
	if err := xml.Unmarshal(odata, &opf); err != nil {
		return nil, fmt.Errorf("pipeline: ingest epub: parse opf %s: %w", opfPath, err)
	}
	if len(opf.Spine) == 0 {
		return nil, fmt.Errorf("pipeline: ingest epub: opf %s has an empty spine", opfPath)
	}
	byID := map[string]struct{ href, mediaType string }{}
	for _, it := range opf.Manifest {
		byID[it.ID] = struct{ href, mediaType string }{it.Href, it.MediaType}
	}
	opfDir := path.Dir(opfPath)

	// 3) Read each spine document in order (= one chapter). A dangling idref or a
	// non-(x)html item is skipped, not fatal; a spine that yields zero readable
	// content documents is a loud error.
	doc := &Document{}
	denseNo := 0 // matches SplitChunks: only a NON-empty chapter takes a number
	read := 0
	var dangling []string
	for _, ref := range opf.Spine {
		item, ok := byID[ref.IDRef]
		if !ok {
			// A spine itemref is a DECLARED reading-order document; a dangling idref
			// (no manifest item) is a LOST chapter, not a skip. Silently continuing
			// would drop its content AND shift every later chapter's dense number
			// (since_ch for memory v2). Collect all, fail loud after the loop — the
			// same fail-loud a missing zip entry already gets (external-review #1).
			dangling = append(dangling, ref.IDRef)
			continue
		}
		if !isXHTML(item.mediaType, item.href) {
			continue // a genuine non-content asset in the spine (image/css) — no text
		}
		entry := resolveHref(opfDir, item.href)
		data, err := readZipEntry(files, entry)
		if err != nil {
			return nil, fmt.Errorf("pipeline: ingest epub: spine item %q → %s: %w", ref.IDRef, entry, err)
		}
		read++
		text, ruby, err := extractXHTML(data)
		if err != nil {
			return nil, fmt.Errorf("pipeline: ingest epub: extract %s: %w", entry, err)
		}
		norm := NormalizeSource(text)
		doc.Chapters = append(doc.Chapters, norm)
		// The chapter number ruby carries MUST equal the number SplitChunks assigns
		// (dense — empty chapters are skipped), otherwise memory v2's since_ch is off
		// by the count of empty spine items (cover/nav) before the reading. A
		// ruby-bearing chapter is never empty (its base sits in the body), so denseNo
		// is well-defined here; empty chapters carry no ruby.
		if len(splitParagraphs(norm)) == 0 {
			continue
		}
		denseNo++
		for i := range ruby {
			ruby[i].Chapter = denseNo
		}
		doc.Ruby = append(doc.Ruby, ruby...)
	}
	if len(dangling) > 0 {
		return nil, fmt.Errorf("pipeline: ingest epub: spine references %d manifest id(s) with no manifest item %v — a chapter would be silently lost", len(dangling), dangling)
	}
	if read == 0 {
		return nil, fmt.Errorf("pipeline: ingest epub: spine resolved to no readable (x)html chapters")
	}
	return doc, nil
}

// isXHTML reports whether a manifest item is an (x)html content document. It
// checks the media-type (parameters like "; charset=utf-8" stripped), accepting
// the (x)html types AND the generic XML types real epubs mislabel xhtml chapters
// with (application/xml, text/xml), then falls back to a content-document href
// extension for ANY unrecognized/absent type. A genuine non-content asset keeps its
// own extension (.jpg/.css/.ncx/.svg), so the fallback does not misclassify it.
// (External-review #2: a chapter typed application/xml with a .xml href used to slip
// past BOTH the type switch and a .xhtml-only extension fallback and vanish silently.)
func isXHTML(mediaType, href string) bool {
	mt := strings.ToLower(strings.TrimSpace(mediaType))
	if i := strings.IndexByte(mt, ';'); i >= 0 {
		mt = strings.TrimSpace(mt[:i])
	}
	switch mt {
	case "application/xhtml+xml", "text/html", "application/html",
		"application/xml", "text/xml":
		return true
	}
	switch extOf(href) {
	case ".xhtml", ".html", ".htm", ".xml":
		return true
	}
	return false
}

// resolveHref joins an OPF-relative href to the OPF directory (percent-decoded,
// slash-cleaned) into a zip entry path.
func resolveHref(opfDir, href string) string {
	if h, err := url.PathUnescape(href); err == nil {
		href = h
	}
	// Drop any fragment (chapter.xhtml#frag).
	if i := strings.IndexByte(href, '#'); i >= 0 {
		href = href[:i]
	}
	return path.Clean(path.Join(opfDir, href))
}

// readZipEntry reads a zip entry by cleaned path.
func readZipEntry(files map[string]*zip.File, name string) ([]byte, error) {
	f, ok := files[path.Clean(name)]
	if !ok {
		return nil, fmt.Errorf("missing zip entry %q", name)
	}
	rc, err := f.Open()
	if err != nil {
		return nil, fmt.Errorf("open zip entry %q: %w", name, err)
	}
	defer rc.Close()
	return io.ReadAll(rc)
}

// --- xhtml text + ruby extraction ----------------------------------------------

// blockTags emit a paragraph break (blank line) so the chunker's paragraph packing
// survives extraction; without them a whole chapter collapses into one paragraph.
var blockTags = map[string]bool{
	"p": true, "div": true, "h1": true, "h2": true, "h3": true, "h4": true,
	"h5": true, "h6": true, "li": true, "blockquote": true, "section": true,
	"article": true, "tr": true, "table": true, "ul": true, "ol": true, "dl": true,
	"dd": true, "dt": true, "pre": true, "figure": true, "figcaption": true,
	"header": true, "footer": true, "aside": true, "nav": true, "hr": true,
	"main": true, "td": true, "th": true, "caption": true,
}

// skipRoots are subtrees whose text is not prose (CSS/JS/metadata).
var skipRoots = map[string]bool{"script": true, "style": true, "head": true}

// extractXHTML strips tags to text and captures ruby. Rules:
//   - block tag → blank line; <br> → newline (paragraph structure for the chunker);
//   - <ruby>base<rt>reading</rt></ruby>: BASE goes to the body, READING is captured
//     as a (base,reading) pair, <rp> parenthesis fallbacks are dropped;
//   - <script>/<style>/<head> subtrees are dropped.
//
// Non-strict HTML decoding (HTMLEntity/HTMLAutoClose) tolerates the loose xhtml
// real epubs ship. A non-UTF-8 declared charset is a loud error (epub v1 = UTF-8).
// Ruby is returned with Chapter unset (0); the caller stamps the dense chapter no.
func extractXHTML(data []byte) (string, []RubyReading, error) {
	dec := xml.NewDecoder(bytes.NewReader(data))
	dec.Strict = false
	dec.Entity = xml.HTMLEntity
	dec.AutoClose = xml.HTMLAutoClose
	dec.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) {
		switch strings.ToLower(strings.TrimSpace(charset)) {
		case "", "utf-8", "utf8", "us-ascii", "ascii":
			return input, nil
		}
		return nil, fmt.Errorf("unsupported xhtml charset %q (epub v1 expects UTF-8)", charset)
	}

	var body strings.Builder
	var ruby []RubyReading
	var base, reading strings.Builder
	var rubyDepth, rtDepth, rpDepth, skipDepth int

	for {
		tok, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", nil, err
		}
		switch t := tok.(type) {
		case xml.StartElement:
			if skipDepth > 0 {
				skipDepth++
				continue
			}
			name := strings.ToLower(t.Name.Local)
			switch {
			case skipRoots[name]:
				skipDepth++
			case name == "ruby":
				rubyDepth++
				if rubyDepth == 1 {
					base.Reset()
					reading.Reset()
				}
			case name == "rt":
				rtDepth++
			case name == "rp":
				rpDepth++
			case name == "br" && rubyDepth == 0:
				body.WriteByte('\n')
			case blockTags[name] && rubyDepth == 0:
				body.WriteString("\n\n")
			}
		case xml.EndElement:
			if skipDepth > 0 {
				skipDepth--
				continue
			}
			name := strings.ToLower(t.Name.Local)
			switch {
			case name == "ruby":
				if rubyDepth > 0 {
					rubyDepth--
					if rubyDepth == 0 {
						b := strings.TrimSpace(base.String())
						r := strings.TrimSpace(reading.String())
						if b != "" && r != "" {
							ruby = append(ruby, RubyReading{Base: b, Reading: r})
						}
						base.Reset()
						reading.Reset()
					}
				}
			case name == "rt":
				if rtDepth > 0 {
					rtDepth--
				}
			case name == "rp":
				if rpDepth > 0 {
					rpDepth--
				}
			case blockTags[name] && rubyDepth == 0:
				body.WriteString("\n\n")
			}
		case xml.CharData:
			if skipDepth > 0 {
				continue
			}
			text := string(t)
			if rubyDepth > 0 {
				switch {
				case rpDepth > 0:
					// ruby parenthesis fallback — drop.
				case rtDepth > 0:
					reading.WriteString(text)
				case strings.TrimSpace(text) == "":
					// Formatting whitespace BETWEEN ruby base segments (pretty-printed
					// jukugo <rb>旧</rb> <rb>字</rb> …) must not enter the captured base
					// (it would mis-key the glossary reading) nor the body (it would
					// inject stray whitespace between CJK glyphs into ch.Text) —
					// self-review finding. Drop it; real base glyphs are non-whitespace.
				default:
					base.WriteString(text)
					body.WriteString(text)
				}
				continue
			}
			body.WriteString(text)
		}
	}
	return body.String(), ruby, nil
}