package pipeline
import (
"archive/zip"
"os"
"path/filepath"
"strings"
"testing"
)
// --- txt ------------------------------------------------------------------------
func TestIngestTXTSingleChapter(t *testing.T) {
path := filepath.Join(t.TempDir(), "src.txt")
if err := os.WriteFile(path, []byte("静かな図書館の朝。"), 0o644); err != nil {
t.Fatal(err)
}
doc, err := Ingest(path)
if err != nil {
t.Fatal(err)
}
if len(doc.Chapters) != 1 || doc.Chapters[0] != "静かな図書館の朝。" {
t.Fatalf("chapters = %#v", doc.Chapters)
}
if len(doc.Ruby) != 0 {
t.Fatalf("txt has no ruby, got %#v", doc.Ruby)
}
}
func TestIngestTXTFormFeedChapters(t *testing.T) {
path := filepath.Join(t.TempDir(), "src.txt")
if err := os.WriteFile(path, []byte("Глава A"+chapterSep+"Глава B"+chapterSep+"Глава C"), 0o644); err != nil {
t.Fatal(err)
}
doc, err := Ingest(path)
if err != nil {
t.Fatal(err)
}
want := []string{"Глава A", "Глава B", "Глава C"}
if strings.Join(doc.Chapters, "|") != strings.Join(want, "|") {
t.Fatalf("chapters = %#v, want %#v", doc.Chapters, want)
}
}
func TestIngestTXTNormalizes(t *testing.T) {
path := filepath.Join(t.TempDir(), "src.txt")
// UTF-8 BOM + CRLF + surrounding whitespace must be normalized away.
if err := os.WriteFile(path, []byte("\uFEFF строка один\r\nстрока два "), 0o644); err != nil {
t.Fatal(err)
}
doc, err := Ingest(path)
if err != nil {
t.Fatal(err)
}
if doc.Chapters[0] != "строка один\nстрока два" {
t.Fatalf("normalized = %q", doc.Chapters[0])
}
}
// --- epub builder ---------------------------------------------------------------
type epubChapter struct {
id string
href string // relative to the OPF dir (OEBPS/)
mtype string // "" → application/xhtml+xml
body string // inner xhtml of
}
// buildEPUB writes a minimal but real epub (container.xml → OPF → spine → xhtml)
// to a temp .epub file and returns its path. spineIDs gives the reading order
// (may differ from manifest order to prove the spine drives it).
func buildEPUB(t *testing.T, chapters []epubChapter, spineIDs []string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "book.epub")
buildEPUBAt(t, path, chapters, spineIDs)
return path
}
// buildEPUBAt writes the epub to a caller-chosen path (used by the runner e2e).
func buildEPUBAt(t *testing.T, path string, chapters []epubChapter, spineIDs []string) {
t.Helper()
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
zw := zip.NewWriter(f)
add := func(name, content string) {
w, err := zw.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write([]byte(content)); err != nil {
t.Fatal(err)
}
}
add("mimetype", "application/epub+zip")
add("META-INF/container.xml", `
`)
var manifest, spine strings.Builder
for _, c := range chapters {
mt := c.mtype
if mt == "" {
mt = "application/xhtml+xml"
}
manifest.WriteString(`第三章。
`},
{id: "c1", href: "ch1.xhtml", body: `第一章。
`},
{id: "c2", href: "ch2.xhtml", body: `第二章。
`},
}
doc, err := Ingest(buildEPUB(t, chapters, []string{"c1", "c2", "c3"}))
if err != nil {
t.Fatal(err)
}
if len(doc.Chapters) != 3 {
t.Fatalf("want 3 chapters, got %d: %#v", len(doc.Chapters), doc.Chapters)
}
for i, want := range []string{"第一章。", "第二章。", "第三章。"} {
if strings.TrimSpace(doc.Chapters[i]) != want {
t.Fatalf("chapter %d = %q, want %q (spine order broken)", i+1, doc.Chapters[i], want)
}
}
}
func TestIngestEPUBStripsTagsAndEntities(t *testing.T) {
body := `Hello & bold world.
Second paragraph.
`
doc, err := Ingest(buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: body}}, []string{"c1"}))
if err != nil {
t.Fatal(err)
}
txt := doc.Chapters[0]
if !strings.Contains(txt, "Hello & bold world.") {
t.Fatalf("entities/tags not handled: %q", txt)
}
if strings.Contains(txt, "") || strings.Contains(txt, "color:red") || strings.Contains(txt, "") {
t.Fatalf("markup/style/head leaked into text: %q", txt)
}
// Two <p> blocks must be separated by a blank line so the chunker sees paragraphs.
if paras := splitParagraphs(NormalizeSource(txt)); len(paras) != 2 {
t.Fatalf("want 2 paragraphs from 2 <p>, got %d: %#v", len(paras), paras)
}
}
func TestIngestEPUBRubyCaptureAndBaseInBody(t *testing.T) {
body := `<p><ruby>漢字<rt>かんじ</rt></ruby>を読む。</p>` +
`<p><ruby>東<rp>(</rp><rt>とう</rt><rp>)</rp>京<rp>(</rp><rt>きょう</rt><rp>)</rp></ruby>タワー。</p>` +
`<p><ruby>字<rt></rt></ruby>だけ。</p>` // empty reading → not captured, base kept
doc, err := Ingest(buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: body}}, []string{"c1"}))
if err != nil {
t.Fatal(err)
}
txt := doc.Chapters[0]
// Base stays in the body; readings and rp parens do NOT.
for _, want := range []string{"漢字を読む", "東京タワー", "字だけ"} {
if !strings.Contains(txt, want) {
t.Fatalf("base missing from body %q (want %q)", txt, want)
}
}
for _, bad := range []string{"かんじ", "とう", "きょう", "(", ")"} {
if strings.Contains(txt, bad) {
t.Fatalf("reading/paren %q leaked into body %q", bad, txt)
}
}
// Two readings captured (mono-ruby merged to one base+reading); empty rt skipped.
got := map[string]string{}
for _, r := range doc.Ruby {
got[r.Base] = r.Reading
if r.Chapter != 1 {
t.Fatalf("ruby chapter = %d, want 1", r.Chapter)
}
}
if len(doc.Ruby) != 2 || got["漢字"] != "かんじ" || got["東京"] != "とうきょう" {
t.Fatalf("ruby capture = %#v", doc.Ruby)
}
}
func TestIngestEPUBRubyChapterMatchesDenseNumbering(t *testing.T) {
// An empty cover page in the spine before a ruby-bearing chapter must NOT shift
// ruby's first_chapter off the number SplitChunks assigns (dense — cover skipped).
chapters := []epubChapter{
{id: "cover", href: "cover.xhtml", body: `<div> </div>`}, // whitespace only → no chapter number
{id: "c1", href: "ch1.xhtml", body: `<p>ふつうの文。</p>`},
{id: "c2", href: "ch2.xhtml", body: `<p><ruby>朱雀<rt>すざく</rt></ruby>が舞う。</p>`},
}
doc, err := Ingest(buildEPUB(t, chapters, []string{"cover", "c1", "c2"}))
if err != nil {
t.Fatal(err)
}
// The ruby is in the 3rd spine item but the 2nd NON-empty chapter → chapter 2,
// exactly what SplitChunks(doc.Chapters) labels it.
if len(doc.Ruby) != 1 || doc.Ruby[0].Chapter != 2 {
t.Fatalf("ruby dense chapter = %#v, want chapter 2", doc.Ruby)
}
chunks := SplitChunks(doc.Chapters)
var rubyChunkChapter int
for _, c := range chunks {
if strings.Contains(c.Text, "朱雀") {
rubyChunkChapter = c.Chapter
}
}
if rubyChunkChapter != doc.Ruby[0].Chapter {
t.Fatalf("ruby first_chapter %d != chunk chapter %d — memory-v2 since_ch would be off",
doc.Ruby[0].Chapter, rubyChunkChapter)
}
}
func TestIngestEPUBAcceptsGenericAndParameterizedMediaTypes(t *testing.T) {
// Real epubs mislabel xhtml chapters as application/xml or text/xml (incl. a .xml
// href — external-review #2), or add a "; charset=utf-8" parameter. All must be
// read, not silently dropped (a dropped chapter also shifts every since_ch after).
chapters := []epubChapter{
{id: "c1", href: "ch1.xml", mtype: "application/xml", body: `<p>第一章。</p>`},
{id: "c2", href: "ch2.xhtml", mtype: "text/xml", body: `<p>第二章。</p>`},
{id: "c3", href: "ch3.xhtml", mtype: "application/xhtml+xml; charset=utf-8", body: `<p>第三章。</p>`},
}
doc, err := Ingest(buildEPUB(t, chapters, []string{"c1", "c2", "c3"}))
if err != nil {
t.Fatal(err)
}
if len(doc.Chapters) != 3 {
t.Fatalf("mislabeled/parameterized media-types must all be read, got %d: %#v", len(doc.Chapters), doc.Chapters)
}
for i, want := range []string{"第一章。", "第二章。", "第三章。"} {
if strings.TrimSpace(doc.Chapters[i]) != want {
t.Fatalf("chapter %d = %q, want %q", i+1, doc.Chapters[i], want)
}
}
}
func TestIngestEPUBDanglingIdrefFailsLoud(t *testing.T) {
// A spine idref with no manifest item, AMONG valid chapters, must fail loud — not
// silently drop that chapter (which would also shift every later since_ch).
chapters := []epubChapter{
{id: "c1", href: "ch1.xhtml", body: `<p>第一章。</p>`},
{id: "c2", href: "ch2.xhtml", body: `<p>第二章。</p>`},
{id: "c3", href: "ch3.xhtml", body: `<p>第三章。</p>`},
}
_, err := Ingest(buildEPUB(t, chapters, []string{"c1", "ghost", "c3"}))
if err == nil {
t.Fatal("a dangling spine idref among valid chapters must fail loud, not silently lose a chapter")
}
if !strings.Contains(err.Error(), "ghost") {
t.Fatalf("error should name the dangling idref, got: %v", err)
}
}
func TestIngestEPUBBrInsideRubyDoesNotLeak(t *testing.T) {
// A <br> inside <ruby> must not inject a newline between base glyphs (it would
// desync base from body and pollute ch.Text — external-review #5).
body := `<p><ruby>漢<br/>字<rt>かんじ</rt></ruby>を読む。</p>`
doc, err := Ingest(buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: body}}, []string{"c1"}))
if err != nil {
t.Fatal(err)
}
if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "漢字" || doc.Ruby[0].Reading != "かんじ" {
t.Fatalf("ruby base/reading = %#v", doc.Ruby)
}
if !strings.Contains(doc.Chapters[0], "漢字を読む") || strings.Contains(doc.Chapters[0], "漢\n字") {
t.Fatalf("<br> leaked a newline between base glyphs: %q", doc.Chapters[0])
}
}
func TestIngestEPUBRubyBaseWhitespaceCollapsed(t *testing.T) {
// Pretty-printed jukugo ruby: whitespace BETWEEN <rb> base segments must not
// enter the captured base (mis-keys the glossary) nor the body (pollutes ch.Text).
body := "<p><ruby>\n <rb>旧</rb>\n <rb>字</rb>\n <rb>体</rb>\n <rt>きゅう</rt><rt>じ</rt><rt>たい</rt>\n</ruby>の話。</p>"
doc, err := Ingest(buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: body}}, []string{"c1"}))
if err != nil {
t.Fatal(err)
}
if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "旧字体" || doc.Ruby[0].Reading != "きゅうじたい" {
t.Fatalf("ruby base/reading not clean: %#v", doc.Ruby)
}
if !strings.Contains(doc.Chapters[0], "旧字体の話") {
t.Fatalf("clean base missing from body: %q", doc.Chapters[0])
}
for _, bad := range []string{"旧 字", "旧\n"} {
if strings.Contains(doc.Chapters[0], bad) {
t.Fatalf("ruby-internal whitespace leaked into body: %q", doc.Chapters[0])
}
}
}
func TestIngestEPUBSkipsNonXHTML(t *testing.T) {
chapters := []epubChapter{
{id: "img", href: "cover.jpg", mtype: "image/jpeg", body: "\xff\xd8not-really-jpeg"},
{id: "c1", href: "ch1.xhtml", body: `<p>本文。</p>`},
}
doc, err := Ingest(buildEPUB(t, chapters, []string{"img", "c1"}))
if err != nil {
t.Fatal(err)
}
if len(doc.Chapters) != 1 || strings.TrimSpace(doc.Chapters[0]) != "本文。" {
t.Fatalf("non-xhtml spine item must be skipped: %#v", doc.Chapters)
}
}
func TestIngestEPUBBrokenFailsLoud(t *testing.T) {
t.Run("not a zip", func(t *testing.T) {
p := filepath.Join(t.TempDir(), "bad.epub")
os.WriteFile(p, []byte("this is not a zip"), 0o644)
if _, err := Ingest(p); err == nil {
t.Fatal("a non-zip .epub must error, not panic")
}
})
t.Run("missing container", func(t *testing.T) {
p := filepath.Join(t.TempDir(), "noc.epub")
f, _ := os.Create(p)
zw := zip.NewWriter(f)
w, _ := zw.Create("OEBPS/content.opf")
w.Write([]byte("<package/>"))
zw.Close()
f.Close()
if _, err := Ingest(p); err == nil {
t.Fatal("missing container.xml must error")
}
})
t.Run("empty spine", func(t *testing.T) {
p := buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: `<p>x</p>`}}, nil)
if _, err := Ingest(p); err == nil {
t.Fatal("an empty spine must error")
}
})
t.Run("dangling spine idref", func(t *testing.T) {
// spine references a missing manifest id → resolves to zero chapters → error.
p := buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: `<p>x</p>`}}, []string{"ghost"})
if _, err := Ingest(p); err == nil {
t.Fatal("a spine resolving to no readable chapters must error")
}
})
}