728 lines
29 KiB
Go
728 lines
29 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"archive/zip"
|
||
"bytes"
|
||
"encoding/xml"
|
||
"fmt"
|
||
"io"
|
||
"net/url"
|
||
"os"
|
||
"path"
|
||
"regexp"
|
||
"strings"
|
||
"unicode"
|
||
"unicode/utf8"
|
||
|
||
"golang.org/x/text/encoding/simplifiedchinese"
|
||
xunicode "golang.org/x/text/encoding/unicode"
|
||
"golang.org/x/text/transform"
|
||
)
|
||
|
||
// ingest.go: the шаг-3a import layer. Ingest turns a source file (txt or epub)
|
||
// into an ordered list of per-chapter NORMALIZED text plus the ruby/furigana
|
||
// readings captured on the way (04-unhappy §4 / D9). It is the ONLY place that
|
||
// reads the source; the runner then feeds doc.Chapters to SplitChunks and persists
|
||
// doc.Ruby (runner.go). It is offline and deterministic — no LLM, no time/rand — so
|
||
// the whole path is $0 and reproducible.
|
||
//
|
||
// epub v1 is a text extraction (02-mvp:25): read the chapters in spine order,
|
||
// strip tags, drop inline markup/styling. The ONE thing we do NOT drop is ruby:
|
||
// <ruby>base<rt>reading</rt></ruby> carries an author reading of a name/term that a
|
||
// plain text export would silently lose (the documented hole, 04-unhappy §4). We
|
||
// keep the BASE in the body and hand the READING to the caller for a glossary lock
|
||
// (memory v2, шаг 4) — never injecting it into a prompt here (§7d).
|
||
|
||
// chapterSep splits a TXT source into chapters. Form feed (U+000C) is the ASCII
|
||
// "page/section break": semantically a chapter boundary, invisible in prose, and
|
||
// untouched by NormalizeSource. A txt with no \f is a single chapter (Фаза-0 backward
|
||
// compat: the one-file example stays one chapter). epub carries real chapter
|
||
// structure in its spine, so it does not use this.
|
||
const chapterSep = "\f"
|
||
|
||
// RubyReading is one captured (base, reading) pair and the chapter it appeared in.
|
||
// The pipeline aggregates these (min chapter, count) before persisting (runner.go).
|
||
type RubyReading struct {
|
||
Base string // the ruby body: the kanji/base surface form
|
||
Reading string // the <rt> reading (furigana)
|
||
Chapter int // 1-based chapter where this occurrence was found
|
||
}
|
||
|
||
// Document is the ingested source: per-chapter normalized text (in reading order)
|
||
// plus every ruby occurrence found. Chapters is what SplitChunks consumes.
|
||
type Document struct {
|
||
Chapters []string
|
||
Ruby []RubyReading
|
||
}
|
||
|
||
// Ingest reads a source file with automatic encoding detection (the txt path), source language
|
||
// unspecified. Dispatch is by extension: .epub → the epub reader; anything else → plain text. Kept
|
||
// as the terse form used by tests and the epub path.
|
||
func Ingest(p string) (*Document, error) {
|
||
return IngestEncoded(p, "", "")
|
||
}
|
||
|
||
// IngestEncoded reads a source file and returns its Document, decoding a txt source per the book's
|
||
// declared encoding (auto|utf8|gb18030; "" == auto) and source language (zh|ja|en; "" == unknown).
|
||
// sourceLang scopes the GB18030 auto-probe to Chinese sources — a Shift-JIS/EUC-JP ja file would
|
||
// otherwise decode to Han-shaped mojibake that passes the plausibility guard (self-review critical).
|
||
// Dispatch is by extension: .epub → the epub reader (its own per-document charset handling);
|
||
// anything else → plain text.
|
||
func IngestEncoded(p, encoding, sourceLang string) (*Document, error) {
|
||
if strings.EqualFold(extOf(p), ".epub") {
|
||
return ingestEPUB(p)
|
||
}
|
||
return ingestTXT(p, encoding, sourceLang)
|
||
}
|
||
|
||
// extOf returns the lower-cased file extension incl. the dot ("" if none).
|
||
func extOf(p string) string {
|
||
i := strings.LastIndexByte(p, '.')
|
||
if i < 0 || strings.ContainsAny(p[i:], "/\\") {
|
||
return ""
|
||
}
|
||
return strings.ToLower(p[i:])
|
||
}
|
||
|
||
// ingestTXT reads a plain-text source: decode the raw bytes to UTF-8 per `encoding`, split on \f
|
||
// into chapters, NormalizeSource each (BOM/CRLF/NFC/outer-trim). No ruby in txt. Encoding decode
|
||
// runs BEFORE NormalizeSource so the whole downstream path (chunker fuzz-invariant, memory keys)
|
||
// sees valid UTF-8; a decode failure is a LOUD error, never silent U+FFFD corruption.
|
||
func ingestTXT(p, encoding, sourceLang string) (*Document, error) {
|
||
raw, err := os.ReadFile(p)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest txt: %w", err)
|
||
}
|
||
text, err := decodeSourceBytes(raw, encoding, sourceLang)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest txt %s: %w", p, err)
|
||
}
|
||
var chapters []string
|
||
// Honor the ASCII form feed as a hard chapter boundary (Фаза-0 example), THEN split each part
|
||
// on CJK chapter-header lines (第N章/节/回) — real zh/ja web novels mark chapters that way, not
|
||
// with \f (the 蛊真人 acceptance book uses «第N节», ~2283 of them). A part with neither stays one
|
||
// chapter (backward-compat).
|
||
for _, part := range strings.Split(text, chapterSep) {
|
||
for _, chap := range splitTextChapters(part) {
|
||
chapters = append(chapters, NormalizeSource(chap))
|
||
}
|
||
}
|
||
return &Document{Chapters: chapters}, nil
|
||
}
|
||
|
||
// --- CJK chapter-header splitting (D18: real zh/ja txt mark chapters as «第N章/节/回») ---------
|
||
|
||
// chapterUnitRunes are the section-level chapter markers auto-detected in a txt. 卷 (volume) is
|
||
// deliberately EXCLUDED — it is coarser than a chapter and would carve a tiny title-only "chapter".
|
||
var chapterUnitRunes = []rune{'章', '节', '節', '回'}
|
||
|
||
// chapterNumeralRE matches the numeral run of a chapter header: Arabic (half/fullwidth) or CJK.
|
||
var chapterNumeralRE = regexp.MustCompile(`^\s*第[0-9０-９〇零一二三四五六七八九十百千两兩]+`)
|
||
|
||
// chapterHeaderMaxRunes bounds a header line so a prose sentence that merely opens with «第三节…»
|
||
// (a longer line) is not mistaken for a header. The 蛊真人 headers are ≤23 runes; 60 leaves room for
|
||
// a long subtitle while still excluding a full prose sentence (self-review: 40 could miss a long
|
||
// legitimate subtitle → merged chapters).
|
||
const chapterHeaderMaxRunes = 60
|
||
|
||
// isCJKChapterHeader reports whether a single line is a chapter header for the given unit rune. It
|
||
// requires: a short trimmed line (≤60 runes); a leading 第<numerals>; the unit rune immediately
|
||
// after the numerals; and — critically — the char AFTER the unit is a SEPARATOR (whitespace / ：:、,.
|
||
// / dash) or end-of-line, NOT a content glyph. The separator guard is the precision fix (self-
|
||
// review): 回 is a common measure word, so prose like «第一回见面…» glues a content char (见) right
|
||
// after the unit and must NOT split; a real header writes «第一节：…» or «第1章 …» with a separator.
|
||
// (A glued-title header «第一章天空…» without a separator is not detected — rare; a recall trade for
|
||
// no false splits. Deterministic and pure.)
|
||
func isCJKChapterHeader(line string, unit rune) bool {
|
||
t := strings.TrimSpace(line)
|
||
if utf8.RuneCountInString(t) == 0 || utf8.RuneCountInString(t) > chapterHeaderMaxRunes {
|
||
return false
|
||
}
|
||
loc := chapterNumeralRE.FindStringIndex(t)
|
||
if loc == nil {
|
||
return false
|
||
}
|
||
rest := t[loc[1]:] // the runes right after the numerals
|
||
r, sz := utf8.DecodeRuneInString(rest)
|
||
if r != unit {
|
||
return false
|
||
}
|
||
after := rest[sz:]
|
||
if after == "" {
|
||
return true // the header is exactly 第N章
|
||
}
|
||
nr, _ := utf8.DecodeRuneInString(after)
|
||
return isHeaderSeparator(nr)
|
||
}
|
||
|
||
// isHeaderSeparator reports whether a rune separates a chapter number from its title (so the line is
|
||
// a header, not a prose sentence that continues with a content glyph after 第N章). Anything that is
|
||
// NOT a letter / ideograph / kana / digit counts as a separator (space, ：:、，,.。-—— etc.).
|
||
func isHeaderSeparator(r rune) bool {
|
||
if unicode.IsLetter(r) || unicode.IsDigit(r) ||
|
||
unicode.Is(unicode.Han, r) || unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r) {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// detectChapterUnit picks the section marker that appears most as a header line (≥2 to avoid a
|
||
// single stray match). Returns 0 when no unit qualifies (→ the part stays a single chapter).
|
||
func detectChapterUnit(lines []string) rune {
|
||
best, bestN := rune(0), 0
|
||
for _, unit := range chapterUnitRunes {
|
||
n := 0
|
||
for _, ln := range lines {
|
||
if isCJKChapterHeader(ln, unit) {
|
||
n++
|
||
}
|
||
}
|
||
if n >= 2 && n > bestN {
|
||
best, bestN = unit, n
|
||
}
|
||
}
|
||
return best
|
||
}
|
||
|
||
// splitTextChapters splits one text block on its dominant CJK chapter-header lines. The header line
|
||
// STARTS its chapter (the title is kept). Any preamble before the FIRST header is merged into
|
||
// chapter 1 so «第一节» == chapter 1 (dense numbering aligns with the section numbers). A block with
|
||
// no detectable headers is returned unchanged as a single chapter.
|
||
func splitTextChapters(text string) []string {
|
||
norm := strings.ReplaceAll(text, "\r\n", "\n")
|
||
norm = strings.ReplaceAll(norm, "\r", "\n") // bare-CR (old Mac) lines too, so a CR-only header is seen
|
||
lines := strings.Split(norm, "\n")
|
||
unit := detectChapterUnit(lines)
|
||
if unit == 0 {
|
||
return []string{text}
|
||
}
|
||
var chapters []string
|
||
var cur []string
|
||
seenHeader := false
|
||
for _, ln := range lines {
|
||
if isCJKChapterHeader(ln, unit) {
|
||
if seenHeader && hasNonBlank(cur) {
|
||
chapters = append(chapters, strings.Join(cur, "\n"))
|
||
cur = nil
|
||
}
|
||
seenHeader = true
|
||
}
|
||
cur = append(cur, ln)
|
||
}
|
||
if hasNonBlank(cur) {
|
||
chapters = append(chapters, strings.Join(cur, "\n"))
|
||
}
|
||
if len(chapters) == 0 {
|
||
return []string{text}
|
||
}
|
||
return chapters
|
||
}
|
||
|
||
// hasNonBlank reports whether any line in the slice has non-whitespace content.
|
||
func hasNonBlank(lines []string) bool {
|
||
for _, ln := range lines {
|
||
if strings.TrimSpace(ln) != "" {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// --- source encoding detection (Task 2 / D18) ----------------------------------
|
||
//
|
||
// Real zh .txt are frequently GB18030 (a superset of GBK/GB2312), not UTF-8 — the 蛊真人 acceptance
|
||
// book is exactly this case. decodeSourceBytes converts raw source bytes to a UTF-8 string per the
|
||
// book's declared encoding, defaulting to a conservative auto-detect. The invariant is fail-LOUD:
|
||
// any doubt is an error, never silent mojibake — a corrupt/foreign byte must never reach a U+FFFD
|
||
// replacement in ch.Text (which would defeat the lossless-chunker fuzz invariant and mis-key the
|
||
// glossary). Dependency: golang.org/x/text/encoding (already in go.mod for NFC) — first use of its
|
||
// simplifiedchinese + unicode subpackages (noted in the session journal).
|
||
func decodeSourceBytes(raw []byte, encoding, sourceLang string) (string, error) {
|
||
switch enc := strings.ToLower(strings.TrimSpace(encoding)); enc {
|
||
case "", "auto":
|
||
return autoDecodeSource(raw, sourceLang)
|
||
case "utf8", "utf-8":
|
||
b := bytes.TrimPrefix(raw, utf8BOM)
|
||
if !utf8.Valid(b) {
|
||
return "", fmt.Errorf("declared encoding utf8 but the bytes are not valid UTF-8 (declare gb18030, or convert the file)")
|
||
}
|
||
return checkNoNUL(string(b))
|
||
case "gb18030", "gbk", "gb2312":
|
||
// Operator asserted the encoding: decode strictly (fail on undecodable bytes / U+FFFD) but
|
||
// skip the Chinese-plausibility heuristic — they told us it is GB18030.
|
||
return decodeGB18030(raw, false)
|
||
default:
|
||
return "", fmt.Errorf("unknown encoding %q in book.yaml (use auto|utf8|gb18030)", encoding)
|
||
}
|
||
}
|
||
|
||
// checkNoNUL rejects a decoded string that contains a NUL (U+0000). Real prose never contains NUL;
|
||
// its presence means a mis-detected / mis-declared encoding: a UTF-16 stream WITHOUT a BOM whose
|
||
// ASCII half is valid UTF-8 with interleaved NULs, OR a UTF-16/binary file wrongly fed to the
|
||
// GB18030/UTF-16 decoder (0x00 decodes to a VALID U+0000, not U+FFFD, so the replacement-char check
|
||
// cannot catch it — self-review major, D20.4). Applied on EVERY decode path (utf8/gb18030/utf16),
|
||
// not just UTF-8, so the guard is symmetric across encodings.
|
||
func checkNoNUL(s string) (string, error) {
|
||
if strings.IndexByte(s, 0) >= 0 {
|
||
return "", fmt.Errorf("decoded text contains NUL (U+0000) — the source is likely UTF-16 without a BOM, or a binary/mis-declared file; convert it to UTF-8 or declare its encoding correctly")
|
||
}
|
||
return s, nil
|
||
}
|
||
|
||
var (
|
||
utf8BOM = []byte{0xEF, 0xBB, 0xBF}
|
||
utf16LEBOM = []byte{0xFF, 0xFE}
|
||
utf16BEBOM = []byte{0xFE, 0xFF}
|
||
)
|
||
|
||
// autoDecodeSource is the conservative detection ladder: BOM → valid UTF-8 (+NUL guard) → GB18030
|
||
// probe (ZH sources only) → fail-loud. Order matters: a BOM is definitive; valid UTF-8 is
|
||
// unambiguous (but rejected if it carries interleaved NUL — UTF-16 without a BOM); GB18030 is tried
|
||
// only when the bytes are NOT valid UTF-8 AND the source is Chinese (or unspecified). GB18030 is a
|
||
// Chinese encoding that maps almost every byte sequence to Han-shaped codepoints, so a ja Shift-JIS
|
||
// / EUC-JP or a European Latin-1 file would decode to CJK-shaped MOJIBAKE that the plausibility
|
||
// guard cannot distinguish from real Chinese (self-review CRITICAL). So we auto-probe GB18030 only
|
||
// for zh; for ja/en a non-UTF-8 file fails loud and the operator must convert or declare it.
|
||
func autoDecodeSource(raw []byte, sourceLang string) (string, error) {
|
||
switch {
|
||
case bytes.HasPrefix(raw, utf8BOM):
|
||
b := raw[len(utf8BOM):]
|
||
if !utf8.Valid(b) {
|
||
return "", fmt.Errorf("file has a UTF-8 BOM but the body is not valid UTF-8 (corrupt file)")
|
||
}
|
||
return checkNoNUL(string(b))
|
||
case bytes.HasPrefix(raw, utf16LEBOM):
|
||
return decodeUTF16(raw, xunicode.LittleEndian)
|
||
case bytes.HasPrefix(raw, utf16BEBOM):
|
||
return decodeUTF16(raw, xunicode.BigEndian)
|
||
}
|
||
if utf8.Valid(raw) {
|
||
return checkNoNUL(string(raw)) // no BOM, already UTF-8 (covers en/ja/UTF-8 zh); NUL ⇒ UTF-16-no-BOM
|
||
}
|
||
// Not valid UTF-8. GB18030 auto-probe is scoped to Chinese sources (see doc above).
|
||
zhScoped := sourceLang == "" || strings.EqualFold(sourceLang, "zh")
|
||
if zhScoped {
|
||
if s, err := decodeGB18030(raw, true); err == nil {
|
||
return s, nil
|
||
}
|
||
}
|
||
probe := "was tried and failed (truncated/corrupt or not GB18030)"
|
||
if !zhScoped {
|
||
probe = fmt.Sprintf("is restricted to zh sources (this source is %q) — a Shift-JIS/EUC-JP/Latin-1 file would decode to Han-shaped mojibake and is NOT auto-detected", sourceLang)
|
||
}
|
||
return "", fmt.Errorf("could not auto-detect the source encoding: the bytes are not valid UTF-8, and the GB18030 probe %s; convert the file to UTF-8 or declare `encoding:` explicitly in book.yaml", probe)
|
||
}
|
||
|
||
// decodeGB18030 decodes raw as GB18030 (superset of GBK/GB2312). It is strict: a decode error, or a
|
||
// U+FFFD replacement char in the output (undecodable/truncated bytes), is a failure — broken bytes
|
||
// must never reach ch.Text. When probe is true (auto-detect) it additionally requires the decoded
|
||
// text to look like Chinese (plausibleCJK), because GB18030 maps almost every byte sequence to
|
||
// SOMETHING, so a Latin-1/other file decodes without error into mojibake unless we sanity-check it.
|
||
func decodeGB18030(raw []byte, probe bool) (string, error) {
|
||
out, _, err := transform.Bytes(simplifiedchinese.GB18030.NewDecoder(), raw)
|
||
if err != nil {
|
||
return "", fmt.Errorf("GB18030 decode error (not valid GB18030): %w", err)
|
||
}
|
||
s := string(out)
|
||
if strings.ContainsRune(s, utf8.RuneError) {
|
||
return "", fmt.Errorf("GB18030 decode produced replacement chars (U+FFFD) — bytes are truncated/corrupt or not GB18030")
|
||
}
|
||
if probe {
|
||
if err := plausibleCJK(s); err != nil {
|
||
return "", err
|
||
}
|
||
}
|
||
return checkNoNUL(s) // 0x00 bytes decode to a valid U+0000, not U+FFFD — a mis-declared UTF-16/binary file slips the replacement-char check (D20.4)
|
||
}
|
||
|
||
// decodeUTF16 decodes a BOM-prefixed UTF-16 stream (ExpectBOM consumes the BOM). Strict: a decode
|
||
// error or a U+FFFD in the output is a failure.
|
||
func decodeUTF16(raw []byte, endian xunicode.Endianness) (string, error) {
|
||
dec := xunicode.UTF16(endian, xunicode.ExpectBOM).NewDecoder()
|
||
out, _, err := transform.Bytes(dec, raw)
|
||
if err != nil {
|
||
return "", fmt.Errorf("UTF-16 decode error: %w", err)
|
||
}
|
||
s := string(out)
|
||
if strings.ContainsRune(s, utf8.RuneError) {
|
||
return "", fmt.Errorf("UTF-16 decode produced replacement chars (U+FFFD) — corrupt")
|
||
}
|
||
return checkNoNUL(s) // a genuine UTF-16 stream carrying embedded NULs (or a mis-detected binary) is caught here (D20.4)
|
||
}
|
||
|
||
// plausibleCJK is the GB18030 auto-accept guard: a real GB18030 source is Chinese, so the decoded
|
||
// text must be (a) almost entirely CJK-plausible runes (Han/kana/CJK-punct/ASCII/whitespace) and
|
||
// (b) genuinely Han-dense. Latin-1/other mojibake fails (b) — it decodes to sparse Han among ASCII
|
||
// (café → "caf" + 轳), so every rune is "plausible" in isolation yet Han density is ~0. The Han
|
||
// floor is intentionally low (5%) so a heavily Latin-mixed but real Chinese source still passes,
|
||
// while mojibake (Han≈0) is rejected loudly. Thresholds bias toward fail-loud (D18 invariant); a
|
||
// false reject is recoverable (declare `encoding: gb18030`), a false accept is silent corruption.
|
||
func plausibleCJK(s string) error {
|
||
var total, han, plausible int
|
||
for _, r := range s {
|
||
total++
|
||
switch {
|
||
case unicode.Is(unicode.Han, r):
|
||
han++
|
||
plausible++
|
||
case isPlausibleSourceRune(r):
|
||
plausible++
|
||
}
|
||
}
|
||
if total == 0 {
|
||
return fmt.Errorf("GB18030 probe: empty decode")
|
||
}
|
||
if pf := float64(plausible) / float64(total); pf < 0.90 {
|
||
return fmt.Errorf("GB18030 probe: only %.1f%% of decoded runes are CJK-plausible — likely mojibake, not Chinese", 100*pf)
|
||
}
|
||
if hf := float64(han) / float64(total); hf < 0.05 {
|
||
return fmt.Errorf("GB18030 probe: Han density %.2f%% is too low for a Chinese source — likely a mis-detected encoding; declare `encoding` explicitly if this really is GB18030", 100*hf)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// isPlausibleSourceRune reports whether a rune belongs to the expected repertoire of a zh/ja source
|
||
// (excluding Han, which the caller counts separately): ASCII text, kana, CJK/general punctuation,
|
||
// and fullwidth/halfwidth forms. Deliberately tight — it must NOT whitelist the Latin-1 supplement
|
||
// or arbitrary symbols, or Latin-1 mojibake would pass the plausibility fraction.
|
||
func isPlausibleSourceRune(r rune) bool {
|
||
switch {
|
||
case r == '\n', r == '\r', r == '\t', r == ' ':
|
||
return true
|
||
case r >= 0x20 && r <= 0x7E: // printable ASCII
|
||
return true
|
||
case unicode.Is(unicode.Hiragana, r), unicode.Is(unicode.Katakana, r):
|
||
return true
|
||
case r >= 0x3000 && r <= 0x303F: // CJK symbols & punctuation (。、「」〜…)
|
||
return true
|
||
case r >= 0x2010 && r <= 0x206F: // general punctuation (— – ‘’ “” … ‰ etc.)
|
||
return true
|
||
case r >= 0xFF00 && r <= 0xFFEF: // halfwidth & fullwidth forms (！？（）ｱ fullwidth digits/letters)
|
||
return true
|
||
case r == 0x30FC || r == 0x30FB: // prolonged-sound mark ー, middle dot ・ (also in Katakana block)
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// --- epub ----------------------------------------------------------------------
|
||
|
||
// containerXML is META-INF/container.xml: it points at the OPF package file. Tags
|
||
// are matched by LOCAL name (no namespace in the struct tags), so a namespaced or
|
||
// prefixed container still binds.
|
||
type containerXML struct {
|
||
Rootfiles []struct {
|
||
FullPath string `xml:"full-path,attr"`
|
||
MediaType string `xml:"media-type,attr"`
|
||
} `xml:"rootfiles>rootfile"`
|
||
}
|
||
|
||
// opfPackage is the OPF: the manifest (id→href→media-type) and the spine (the
|
||
// reading order of itemref idrefs).
|
||
type opfPackage struct {
|
||
Manifest []struct {
|
||
ID string `xml:"id,attr"`
|
||
Href string `xml:"href,attr"`
|
||
MediaType string `xml:"media-type,attr"`
|
||
} `xml:"manifest>item"`
|
||
Spine []struct {
|
||
IDRef string `xml:"idref,attr"`
|
||
} `xml:"spine>itemref"`
|
||
}
|
||
|
||
// ingestEPUB reads chapters in spine order and captures ruby. Every structural
|
||
// problem (bad zip, missing container/opf, empty spine, no readable chapter) is a
|
||
// loud error, never a panic (a truncated/foreign epub must not crash a run).
|
||
func ingestEPUB(p string) (*Document, error) {
|
||
zr, err := zip.OpenReader(p)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: open zip %s: %w", p, err)
|
||
}
|
||
defer zr.Close()
|
||
|
||
files := map[string]*zip.File{}
|
||
for _, f := range zr.File {
|
||
files[path.Clean(f.Name)] = f
|
||
}
|
||
|
||
// 1) container.xml → OPF path.
|
||
cdata, err := readZipEntry(files, "META-INF/container.xml")
|
||
if err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: %w", err)
|
||
}
|
||
var container containerXML
|
||
if err := xml.Unmarshal(cdata, &container); err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: parse container.xml: %w", err)
|
||
}
|
||
opfPath := ""
|
||
for _, rf := range container.Rootfiles {
|
||
if strings.TrimSpace(rf.FullPath) != "" {
|
||
opfPath = path.Clean(rf.FullPath)
|
||
break
|
||
}
|
||
}
|
||
if opfPath == "" {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: container.xml has no rootfile full-path")
|
||
}
|
||
|
||
// 2) OPF → manifest + spine.
|
||
odata, err := readZipEntry(files, opfPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: %w", err)
|
||
}
|
||
var opf opfPackage
|
||
if err := xml.Unmarshal(odata, &opf); err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: parse opf %s: %w", opfPath, err)
|
||
}
|
||
if len(opf.Spine) == 0 {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: opf %s has an empty spine", opfPath)
|
||
}
|
||
byID := map[string]struct{ href, mediaType string }{}
|
||
for _, it := range opf.Manifest {
|
||
byID[it.ID] = struct{ href, mediaType string }{it.Href, it.MediaType}
|
||
}
|
||
opfDir := path.Dir(opfPath)
|
||
|
||
// 3) Read each spine document in order (= one chapter). A dangling idref or a
|
||
// non-(x)html item is skipped, not fatal; a spine that yields zero readable
|
||
// content documents is a loud error.
|
||
doc := &Document{}
|
||
denseNo := 0 // matches SplitChunks: only a NON-empty chapter takes a number
|
||
read := 0
|
||
var dangling []string
|
||
for _, ref := range opf.Spine {
|
||
item, ok := byID[ref.IDRef]
|
||
if !ok {
|
||
// A spine itemref is a DECLARED reading-order document; a dangling idref
|
||
// (no manifest item) is a LOST chapter, not a skip. Silently continuing
|
||
// would drop its content AND shift every later chapter's dense number
|
||
// (since_ch for memory v2). Collect all, fail loud after the loop — the
|
||
// same fail-loud a missing zip entry already gets (external-review #1).
|
||
dangling = append(dangling, ref.IDRef)
|
||
continue
|
||
}
|
||
if !isXHTML(item.mediaType, item.href) {
|
||
continue // a genuine non-content asset in the spine (image/css) — no text
|
||
}
|
||
entry := resolveHref(opfDir, item.href)
|
||
data, err := readZipEntry(files, entry)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: spine item %q → %s: %w", ref.IDRef, entry, err)
|
||
}
|
||
read++
|
||
text, ruby, err := extractXHTML(data)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: extract %s: %w", entry, err)
|
||
}
|
||
norm := NormalizeSource(text)
|
||
doc.Chapters = append(doc.Chapters, norm)
|
||
// The chapter number ruby carries MUST equal the number SplitChunks assigns
|
||
// (dense — empty chapters are skipped), otherwise memory v2's since_ch is off
|
||
// by the count of empty spine items (cover/nav) before the reading. A
|
||
// ruby-bearing chapter is never empty (its base sits in the body), so denseNo
|
||
// is well-defined here; empty chapters carry no ruby.
|
||
if len(splitParagraphs(norm)) == 0 {
|
||
continue
|
||
}
|
||
denseNo++
|
||
for i := range ruby {
|
||
ruby[i].Chapter = denseNo
|
||
}
|
||
doc.Ruby = append(doc.Ruby, ruby...)
|
||
}
|
||
if len(dangling) > 0 {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: spine references %d manifest id(s) with no manifest item %v — a chapter would be silently lost", len(dangling), dangling)
|
||
}
|
||
if read == 0 {
|
||
return nil, fmt.Errorf("pipeline: ingest epub: spine resolved to no readable (x)html chapters")
|
||
}
|
||
return doc, nil
|
||
}
|
||
|
||
// isXHTML reports whether a manifest item is an (x)html content document. It
|
||
// checks the media-type (parameters like "; charset=utf-8" stripped), accepting
|
||
// the (x)html types AND the generic XML types real epubs mislabel xhtml chapters
|
||
// with (application/xml, text/xml), then falls back to a content-document href
|
||
// extension for ANY unrecognized/absent type. A genuine non-content asset keeps its
|
||
// own extension (.jpg/.css/.ncx/.svg), so the fallback does not misclassify it.
|
||
// (External-review #2: a chapter typed application/xml with a .xml href used to slip
|
||
// past BOTH the type switch and a .xhtml-only extension fallback and vanish silently.)
|
||
func isXHTML(mediaType, href string) bool {
|
||
mt := strings.ToLower(strings.TrimSpace(mediaType))
|
||
if i := strings.IndexByte(mt, ';'); i >= 0 {
|
||
mt = strings.TrimSpace(mt[:i])
|
||
}
|
||
switch mt {
|
||
case "application/xhtml+xml", "text/html", "application/html",
|
||
"application/xml", "text/xml":
|
||
return true
|
||
}
|
||
switch extOf(href) {
|
||
case ".xhtml", ".html", ".htm", ".xml":
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// resolveHref joins an OPF-relative href to the OPF directory (percent-decoded,
|
||
// slash-cleaned) into a zip entry path.
|
||
func resolveHref(opfDir, href string) string {
|
||
if h, err := url.PathUnescape(href); err == nil {
|
||
href = h
|
||
}
|
||
// Drop any fragment (chapter.xhtml#frag).
|
||
if i := strings.IndexByte(href, '#'); i >= 0 {
|
||
href = href[:i]
|
||
}
|
||
return path.Clean(path.Join(opfDir, href))
|
||
}
|
||
|
||
// readZipEntry reads a zip entry by cleaned path.
|
||
func readZipEntry(files map[string]*zip.File, name string) ([]byte, error) {
|
||
f, ok := files[path.Clean(name)]
|
||
if !ok {
|
||
return nil, fmt.Errorf("missing zip entry %q", name)
|
||
}
|
||
rc, err := f.Open()
|
||
if err != nil {
|
||
return nil, fmt.Errorf("open zip entry %q: %w", name, err)
|
||
}
|
||
defer rc.Close()
|
||
return io.ReadAll(rc)
|
||
}
|
||
|
||
// --- xhtml text + ruby extraction ----------------------------------------------
|
||
|
||
// blockTags emit a paragraph break (blank line) so the chunker's paragraph packing
|
||
// survives extraction; without them a whole chapter collapses into one paragraph.
|
||
var blockTags = map[string]bool{
|
||
"p": true, "div": true, "h1": true, "h2": true, "h3": true, "h4": true,
|
||
"h5": true, "h6": true, "li": true, "blockquote": true, "section": true,
|
||
"article": true, "tr": true, "table": true, "ul": true, "ol": true, "dl": true,
|
||
"dd": true, "dt": true, "pre": true, "figure": true, "figcaption": true,
|
||
"header": true, "footer": true, "aside": true, "nav": true, "hr": true,
|
||
"main": true, "td": true, "th": true, "caption": true,
|
||
}
|
||
|
||
// skipRoots are subtrees whose text is not prose (CSS/JS/metadata).
|
||
var skipRoots = map[string]bool{"script": true, "style": true, "head": true}
|
||
|
||
// extractXHTML strips tags to text and captures ruby. Rules:
|
||
// - block tag → blank line; <br> → newline (paragraph structure for the chunker);
|
||
// - <ruby>base<rt>reading</rt></ruby>: BASE goes to the body, READING is captured
|
||
// as a (base,reading) pair, <rp> parenthesis fallbacks are dropped;
|
||
// - <script>/<style>/<head> subtrees are dropped.
|
||
//
|
||
// Non-strict HTML decoding (HTMLEntity/HTMLAutoClose) tolerates the loose xhtml
|
||
// real epubs ship. A non-UTF-8 declared charset is a loud error (epub v1 = UTF-8).
|
||
// Ruby is returned with Chapter unset (0); the caller stamps the dense chapter no.
|
||
func extractXHTML(data []byte) (string, []RubyReading, error) {
|
||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||
dec.Strict = false
|
||
dec.Entity = xml.HTMLEntity
|
||
dec.AutoClose = xml.HTMLAutoClose
|
||
dec.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) {
|
||
switch strings.ToLower(strings.TrimSpace(charset)) {
|
||
case "", "utf-8", "utf8", "us-ascii", "ascii":
|
||
return input, nil
|
||
}
|
||
return nil, fmt.Errorf("unsupported xhtml charset %q (epub v1 expects UTF-8)", charset)
|
||
}
|
||
|
||
var body strings.Builder
|
||
var ruby []RubyReading
|
||
var base, reading strings.Builder
|
||
var rubyDepth, rtDepth, rpDepth, skipDepth int
|
||
|
||
for {
|
||
tok, err := dec.Token()
|
||
if err == io.EOF {
|
||
break
|
||
}
|
||
if err != nil {
|
||
return "", nil, err
|
||
}
|
||
switch t := tok.(type) {
|
||
case xml.StartElement:
|
||
if skipDepth > 0 {
|
||
skipDepth++
|
||
continue
|
||
}
|
||
name := strings.ToLower(t.Name.Local)
|
||
switch {
|
||
case skipRoots[name]:
|
||
skipDepth++
|
||
case name == "ruby":
|
||
rubyDepth++
|
||
if rubyDepth == 1 {
|
||
base.Reset()
|
||
reading.Reset()
|
||
}
|
||
case name == "rt":
|
||
rtDepth++
|
||
case name == "rp":
|
||
rpDepth++
|
||
case name == "br" && rubyDepth == 0:
|
||
body.WriteByte('\n')
|
||
case blockTags[name] && rubyDepth == 0:
|
||
body.WriteString("\n\n")
|
||
}
|
||
case xml.EndElement:
|
||
if skipDepth > 0 {
|
||
skipDepth--
|
||
continue
|
||
}
|
||
name := strings.ToLower(t.Name.Local)
|
||
switch {
|
||
case name == "ruby":
|
||
if rubyDepth > 0 {
|
||
rubyDepth--
|
||
if rubyDepth == 0 {
|
||
b := strings.TrimSpace(base.String())
|
||
r := strings.TrimSpace(reading.String())
|
||
if b != "" && r != "" {
|
||
ruby = append(ruby, RubyReading{Base: b, Reading: r})
|
||
}
|
||
base.Reset()
|
||
reading.Reset()
|
||
}
|
||
}
|
||
case name == "rt":
|
||
if rtDepth > 0 {
|
||
rtDepth--
|
||
}
|
||
case name == "rp":
|
||
if rpDepth > 0 {
|
||
rpDepth--
|
||
}
|
||
case blockTags[name] && rubyDepth == 0:
|
||
body.WriteString("\n\n")
|
||
}
|
||
case xml.CharData:
|
||
if skipDepth > 0 {
|
||
continue
|
||
}
|
||
text := string(t)
|
||
if rubyDepth > 0 {
|
||
switch {
|
||
case rpDepth > 0:
|
||
// ruby parenthesis fallback — drop.
|
||
case rtDepth > 0:
|
||
reading.WriteString(text)
|
||
case strings.TrimSpace(text) == "":
|
||
// Formatting whitespace BETWEEN ruby base segments (pretty-printed
|
||
// jukugo <rb>旧</rb> <rb>字</rb> …) must not enter the captured base
|
||
// (it would mis-key the glossary reading) nor the body (it would
|
||
// inject stray whitespace between CJK glyphs into ch.Text) —
|
||
// self-review finding. Drop it; real base glyphs are non-whitespace.
|
||
default:
|
||
base.WriteString(text)
|
||
body.WriteString(text)
|
||
}
|
||
continue
|
||
}
|
||
body.WriteString(text)
|
||
}
|
||
}
|
||
return body.String(), ruby, nil
|
||
}
|