366 lines
14 KiB
Go
366 lines
14 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"archive/zip"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// --- txt ------------------------------------------------------------------------
|
||
|
||
func TestIngestTXTSingleChapter(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "src.txt")
|
||
if err := os.WriteFile(path, []byte("静かな図書館の朝。"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
doc, err := Ingest(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Chapters) != 1 || doc.Chapters[0] != "静かな図書館の朝。" {
|
||
t.Fatalf("chapters = %#v", doc.Chapters)
|
||
}
|
||
if len(doc.Ruby) != 0 {
|
||
t.Fatalf("txt has no ruby, got %#v", doc.Ruby)
|
||
}
|
||
}
|
||
|
||
func TestIngestTXTFormFeedChapters(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "src.txt")
|
||
if err := os.WriteFile(path, []byte("Глава A"+chapterSep+"Глава B"+chapterSep+"Глава C"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
doc, err := Ingest(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
want := []string{"Глава A", "Глава B", "Глава C"}
|
||
if strings.Join(doc.Chapters, "|") != strings.Join(want, "|") {
|
||
t.Fatalf("chapters = %#v, want %#v", doc.Chapters, want)
|
||
}
|
||
}
|
||
|
||
func TestIngestTXTNormalizes(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "src.txt")
|
||
// UTF-8 BOM + CRLF + surrounding whitespace must be normalized away.
|
||
if err := os.WriteFile(path, []byte("\uFEFF строка один\r\nстрока два "), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
doc, err := Ingest(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if doc.Chapters[0] != "строка один\nстрока два" {
|
||
t.Fatalf("normalized = %q", doc.Chapters[0])
|
||
}
|
||
}
|
||
|
||
// --- epub builder ---------------------------------------------------------------
|
||
|
||
type epubChapter struct {
|
||
id string
|
||
href string // relative to the OPF dir (OEBPS/)
|
||
mtype string // "" → application/xhtml+xml
|
||
body string // inner xhtml of <body>
|
||
}
|
||
|
||
// buildEPUB writes a minimal but real epub (container.xml → OPF → spine → xhtml)
|
||
// to a temp .epub file and returns its path. spineIDs gives the reading order
|
||
// (may differ from manifest order to prove the spine drives it).
|
||
func buildEPUB(t *testing.T, chapters []epubChapter, spineIDs []string) string {
|
||
t.Helper()
|
||
path := filepath.Join(t.TempDir(), "book.epub")
|
||
buildEPUBAt(t, path, chapters, spineIDs)
|
||
return path
|
||
}
|
||
|
||
// buildEPUBAt writes the epub to a caller-chosen path (used by the runner e2e).
|
||
func buildEPUBAt(t *testing.T, path string, chapters []epubChapter, spineIDs []string) {
|
||
t.Helper()
|
||
f, err := os.Create(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
defer f.Close()
|
||
zw := zip.NewWriter(f)
|
||
add := func(name, content string) {
|
||
w, err := zw.Create(name)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := w.Write([]byte(content)); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
add("mimetype", "application/epub+zip")
|
||
add("META-INF/container.xml", `<?xml version="1.0"?>
|
||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||
<rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles>
|
||
</container>`)
|
||
|
||
var manifest, spine strings.Builder
|
||
for _, c := range chapters {
|
||
mt := c.mtype
|
||
if mt == "" {
|
||
mt = "application/xhtml+xml"
|
||
}
|
||
manifest.WriteString(`<item id="` + c.id + `" href="` + c.href + `" media-type="` + mt + `"/>` + "\n")
|
||
}
|
||
for _, id := range spineIDs {
|
||
spine.WriteString(`<itemref idref="` + id + `"/>` + "\n")
|
||
}
|
||
add("OEBPS/content.opf", `<?xml version="1.0" encoding="utf-8"?>
|
||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid">
|
||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>Test</dc:title></metadata>
|
||
<manifest>`+manifest.String()+`</manifest>
|
||
<spine>`+spine.String()+`</spine>
|
||
</package>`)
|
||
|
||
for _, c := range chapters {
|
||
// The FILE is (x)html when its href says so, regardless of how the manifest
|
||
// media-type labels it — real epubs mislabel xhtml as application/xml, etc.
|
||
switch extOf(c.href) {
|
||
case ".xhtml", ".html", ".htm", ".xml":
|
||
add("OEBPS/"+c.href, `<?xml version="1.0" encoding="utf-8"?>
|
||
<html xmlns="http://www.w3.org/1999/xhtml"><head><title>c</title><style>.x{color:red}</style></head>
|
||
<body>`+c.body+`</body></html>`)
|
||
default:
|
||
add("OEBPS/"+c.href, c.body) // non-xhtml asset (e.g. image bytes stand-in)
|
||
}
|
||
}
|
||
if err := zw.Close(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBSpineOrder(t *testing.T) {
|
||
// Manifest order (c3,c1,c2) differs from spine order (c1,c2,c3): the spine wins.
|
||
chapters := []epubChapter{
|
||
{id: "c3", href: "ch3.xhtml", body: `<p>第三章。</p>`},
|
||
{id: "c1", href: "ch1.xhtml", body: `<p>第一章。</p>`},
|
||
{id: "c2", href: "ch2.xhtml", body: `<p>第二章。</p>`},
|
||
}
|
||
doc, err := Ingest(buildEPUB(t, chapters, []string{"c1", "c2", "c3"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Chapters) != 3 {
|
||
t.Fatalf("want 3 chapters, got %d: %#v", len(doc.Chapters), doc.Chapters)
|
||
}
|
||
for i, want := range []string{"第一章。", "第二章。", "第三章。"} {
|
||
if strings.TrimSpace(doc.Chapters[i]) != want {
|
||
t.Fatalf("chapter %d = %q, want %q (spine order broken)", i+1, doc.Chapters[i], want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBStripsTagsAndEntities(t *testing.T) {
|
||
body := `<p>Hello & <b>bold</b> world.</p><p>Second paragraph.</p>`
|
||
doc, err := Ingest(buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: body}}, []string{"c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
txt := doc.Chapters[0]
|
||
if !strings.Contains(txt, "Hello & bold world.") {
|
||
t.Fatalf("entities/tags not handled: %q", txt)
|
||
}
|
||
if strings.Contains(txt, "<b>") || strings.Contains(txt, "color:red") || strings.Contains(txt, "<title>") {
|
||
t.Fatalf("markup/style/head leaked into text: %q", txt)
|
||
}
|
||
// Two <p> blocks must be separated by a blank line so the chunker sees paragraphs.
|
||
if paras := splitParagraphs(NormalizeSource(txt)); len(paras) != 2 {
|
||
t.Fatalf("want 2 paragraphs from 2 <p>, got %d: %#v", len(paras), paras)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBRubyCaptureAndBaseInBody(t *testing.T) {
|
||
body := `<p><ruby>漢字<rt>かんじ</rt></ruby>を読む。</p>` +
|
||
`<p><ruby>東<rp>(</rp><rt>とう</rt><rp>)</rp>京<rp>(</rp><rt>きょう</rt><rp>)</rp></ruby>タワー。</p>` +
|
||
`<p><ruby>字<rt></rt></ruby>だけ。</p>` // empty reading → not captured, base kept
|
||
doc, err := Ingest(buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: body}}, []string{"c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
txt := doc.Chapters[0]
|
||
// Base stays in the body; readings and rp parens do NOT.
|
||
for _, want := range []string{"漢字を読む", "東京タワー", "字だけ"} {
|
||
if !strings.Contains(txt, want) {
|
||
t.Fatalf("base missing from body %q (want %q)", txt, want)
|
||
}
|
||
}
|
||
for _, bad := range []string{"かんじ", "とう", "きょう", "(", ")"} {
|
||
if strings.Contains(txt, bad) {
|
||
t.Fatalf("reading/paren %q leaked into body %q", bad, txt)
|
||
}
|
||
}
|
||
// Two readings captured (mono-ruby merged to one base+reading); empty rt skipped.
|
||
got := map[string]string{}
|
||
for _, r := range doc.Ruby {
|
||
got[r.Base] = r.Reading
|
||
if r.Chapter != 1 {
|
||
t.Fatalf("ruby chapter = %d, want 1", r.Chapter)
|
||
}
|
||
}
|
||
if len(doc.Ruby) != 2 || got["漢字"] != "かんじ" || got["東京"] != "とうきょう" {
|
||
t.Fatalf("ruby capture = %#v", doc.Ruby)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBRubyChapterMatchesDenseNumbering(t *testing.T) {
|
||
// An empty cover page in the spine before a ruby-bearing chapter must NOT shift
|
||
// ruby's first_chapter off the number SplitChunks assigns (dense — cover skipped).
|
||
chapters := []epubChapter{
|
||
{id: "cover", href: "cover.xhtml", body: `<div> </div>`}, // whitespace only → no chapter number
|
||
{id: "c1", href: "ch1.xhtml", body: `<p>ふつうの文。</p>`},
|
||
{id: "c2", href: "ch2.xhtml", body: `<p><ruby>朱雀<rt>すざく</rt></ruby>が舞う。</p>`},
|
||
}
|
||
doc, err := Ingest(buildEPUB(t, chapters, []string{"cover", "c1", "c2"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The ruby is in the 3rd spine item but the 2nd NON-empty chapter → chapter 2,
|
||
// exactly what SplitChunks(doc.Chapters) labels it.
|
||
if len(doc.Ruby) != 1 || doc.Ruby[0].Chapter != 2 {
|
||
t.Fatalf("ruby dense chapter = %#v, want chapter 2", doc.Ruby)
|
||
}
|
||
chunks := SplitChunks(doc.Chapters, testSeg())
|
||
var rubyChunkChapter int
|
||
for _, c := range chunks {
|
||
if strings.Contains(c.Text, "朱雀") {
|
||
rubyChunkChapter = c.Chapter
|
||
}
|
||
}
|
||
if rubyChunkChapter != doc.Ruby[0].Chapter {
|
||
t.Fatalf("ruby first_chapter %d != chunk chapter %d — memory-v2 since_ch would be off",
|
||
doc.Ruby[0].Chapter, rubyChunkChapter)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBAcceptsGenericAndParameterizedMediaTypes(t *testing.T) {
|
||
// Real epubs mislabel xhtml chapters as application/xml or text/xml (incl. a .xml
|
||
// href — external-review #2), or add a "; charset=utf-8" parameter. All must be
|
||
// read, not silently dropped (a dropped chapter also shifts every since_ch after).
|
||
chapters := []epubChapter{
|
||
{id: "c1", href: "ch1.xml", mtype: "application/xml", body: `<p>第一章。</p>`},
|
||
{id: "c2", href: "ch2.xhtml", mtype: "text/xml", body: `<p>第二章。</p>`},
|
||
{id: "c3", href: "ch3.xhtml", mtype: "application/xhtml+xml; charset=utf-8", body: `<p>第三章。</p>`},
|
||
}
|
||
doc, err := Ingest(buildEPUB(t, chapters, []string{"c1", "c2", "c3"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Chapters) != 3 {
|
||
t.Fatalf("mislabeled/parameterized media-types must all be read, got %d: %#v", len(doc.Chapters), doc.Chapters)
|
||
}
|
||
for i, want := range []string{"第一章。", "第二章。", "第三章。"} {
|
||
if strings.TrimSpace(doc.Chapters[i]) != want {
|
||
t.Fatalf("chapter %d = %q, want %q", i+1, doc.Chapters[i], want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBDanglingIdrefFailsLoud(t *testing.T) {
|
||
// A spine idref with no manifest item, AMONG valid chapters, must fail loud — not
|
||
// silently drop that chapter (which would also shift every later since_ch).
|
||
chapters := []epubChapter{
|
||
{id: "c1", href: "ch1.xhtml", body: `<p>第一章。</p>`},
|
||
{id: "c2", href: "ch2.xhtml", body: `<p>第二章。</p>`},
|
||
{id: "c3", href: "ch3.xhtml", body: `<p>第三章。</p>`},
|
||
}
|
||
_, err := Ingest(buildEPUB(t, chapters, []string{"c1", "ghost", "c3"}))
|
||
if err == nil {
|
||
t.Fatal("a dangling spine idref among valid chapters must fail loud, not silently lose a chapter")
|
||
}
|
||
if !strings.Contains(err.Error(), "ghost") {
|
||
t.Fatalf("error should name the dangling idref, got: %v", err)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBBrInsideRubyDoesNotLeak(t *testing.T) {
|
||
// A <br> inside <ruby> must not inject a newline between base glyphs (it would
|
||
// desync base from body and pollute ch.Text — external-review #5).
|
||
body := `<p><ruby>漢<br/>字<rt>かんじ</rt></ruby>を読む。</p>`
|
||
doc, err := Ingest(buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: body}}, []string{"c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "漢字" || doc.Ruby[0].Reading != "かんじ" {
|
||
t.Fatalf("ruby base/reading = %#v", doc.Ruby)
|
||
}
|
||
if !strings.Contains(doc.Chapters[0], "漢字を読む") || strings.Contains(doc.Chapters[0], "漢\n字") {
|
||
t.Fatalf("<br> leaked a newline between base glyphs: %q", doc.Chapters[0])
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBRubyBaseWhitespaceCollapsed(t *testing.T) {
|
||
// Pretty-printed jukugo ruby: whitespace BETWEEN <rb> base segments must not
|
||
// enter the captured base (mis-keys the glossary) nor the body (pollutes ch.Text).
|
||
body := "<p><ruby>\n <rb>旧</rb>\n <rb>字</rb>\n <rb>体</rb>\n <rt>きゅう</rt><rt>じ</rt><rt>たい</rt>\n</ruby>の話。</p>"
|
||
doc, err := Ingest(buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: body}}, []string{"c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "旧字体" || doc.Ruby[0].Reading != "きゅうじたい" {
|
||
t.Fatalf("ruby base/reading not clean: %#v", doc.Ruby)
|
||
}
|
||
if !strings.Contains(doc.Chapters[0], "旧字体の話") {
|
||
t.Fatalf("clean base missing from body: %q", doc.Chapters[0])
|
||
}
|
||
for _, bad := range []string{"旧 字", "旧\n"} {
|
||
if strings.Contains(doc.Chapters[0], bad) {
|
||
t.Fatalf("ruby-internal whitespace leaked into body: %q", doc.Chapters[0])
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBSkipsNonXHTML(t *testing.T) {
|
||
chapters := []epubChapter{
|
||
{id: "img", href: "cover.jpg", mtype: "image/jpeg", body: "\xff\xd8not-really-jpeg"},
|
||
{id: "c1", href: "ch1.xhtml", body: `<p>本文。</p>`},
|
||
}
|
||
doc, err := Ingest(buildEPUB(t, chapters, []string{"img", "c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Chapters) != 1 || strings.TrimSpace(doc.Chapters[0]) != "本文。" {
|
||
t.Fatalf("non-xhtml spine item must be skipped: %#v", doc.Chapters)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBBrokenFailsLoud(t *testing.T) {
|
||
t.Run("not a zip", func(t *testing.T) {
|
||
p := filepath.Join(t.TempDir(), "bad.epub")
|
||
os.WriteFile(p, []byte("this is not a zip"), 0o644)
|
||
if _, err := Ingest(p); err == nil {
|
||
t.Fatal("a non-zip .epub must error, not panic")
|
||
}
|
||
})
|
||
t.Run("missing container", func(t *testing.T) {
|
||
p := filepath.Join(t.TempDir(), "noc.epub")
|
||
f, _ := os.Create(p)
|
||
zw := zip.NewWriter(f)
|
||
w, _ := zw.Create("OEBPS/content.opf")
|
||
w.Write([]byte("<package/>"))
|
||
zw.Close()
|
||
f.Close()
|
||
if _, err := Ingest(p); err == nil {
|
||
t.Fatal("missing container.xml must error")
|
||
}
|
||
})
|
||
t.Run("empty spine", func(t *testing.T) {
|
||
p := buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: `<p>x</p>`}}, nil)
|
||
if _, err := Ingest(p); err == nil {
|
||
t.Fatal("an empty spine must error")
|
||
}
|
||
})
|
||
t.Run("dangling spine idref", func(t *testing.T) {
|
||
// spine references a missing manifest id → resolves to zero chapters → error.
|
||
p := buildEPUB(t, []epubChapter{{id: "c1", href: "ch1.xhtml", body: `<p>x</p>`}}, []string{"ghost"})
|
||
if _, err := Ingest(p); err == nil {
|
||
t.Fatal("a spine resolving to no readable chapters must error")
|
||
}
|
||
})
|
||
}
|