290 lines
11 KiB
Go
290 lines
11 KiB
Go
package chunk
|
||
|
||
import (
|
||
"archive/zip"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/chunk/chunktest"
|
||
"textmachine/backend/internal/text"
|
||
)
|
||
|
||
// --- txt ------------------------------------------------------------------------
|
||
|
||
func TestIngestTXTSingleChapter(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "src.txt")
|
||
if err := os.WriteFile(path, []byte("静かな図書館の朝。"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
doc, err := ingest(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Chapters) != 1 || doc.Chapters[0] != "静かな図書館の朝。" {
|
||
t.Fatalf("chapters = %#v", doc.Chapters)
|
||
}
|
||
if len(doc.Ruby) != 0 {
|
||
t.Fatalf("txt has no ruby, got %#v", doc.Ruby)
|
||
}
|
||
}
|
||
|
||
func TestIngestTXTFormFeedChapters(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "src.txt")
|
||
if err := os.WriteFile(path, []byte("Глава A"+chapterSep+"Глава B"+chapterSep+"Глава C"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
doc, err := ingest(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
want := []string{"Глава A", "Глава B", "Глава C"}
|
||
if strings.Join(doc.Chapters, "|") != strings.Join(want, "|") {
|
||
t.Fatalf("chapters = %#v, want %#v", doc.Chapters, want)
|
||
}
|
||
}
|
||
|
||
func TestIngestTXTNormalizes(t *testing.T) {
|
||
path := filepath.Join(t.TempDir(), "src.txt")
|
||
// UTF-8 BOM + CRLF + surrounding whitespace must be normalized away.
|
||
if err := os.WriteFile(path, []byte("\uFEFF строка один\r\nстрока два "), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
doc, err := ingest(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if doc.Chapters[0] != "строка один\nстрока два" {
|
||
t.Fatalf("normalized = %q", doc.Chapters[0])
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBSpineOrder(t *testing.T) {
|
||
// Manifest order (c3,c1,c2) differs from spine order (c1,c2,c3): the spine wins.
|
||
chapters := []chunktest.Chapter{
|
||
{ID: "c3", Href: "ch3.xhtml", Body: `<p>第三章。</p>`},
|
||
{ID: "c1", Href: "ch1.xhtml", Body: `<p>第一章。</p>`},
|
||
{ID: "c2", Href: "ch2.xhtml", Body: `<p>第二章。</p>`},
|
||
}
|
||
doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "c2", "c3"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Chapters) != 3 {
|
||
t.Fatalf("want 3 chapters, got %d: %#v", len(doc.Chapters), doc.Chapters)
|
||
}
|
||
for i, want := range []string{"第一章。", "第二章。", "第三章。"} {
|
||
if strings.TrimSpace(doc.Chapters[i]) != want {
|
||
t.Fatalf("chapter %d = %q, want %q (spine order broken)", i+1, doc.Chapters[i], want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBStripsTagsAndEntities(t *testing.T) {
|
||
body := `<p>Hello & <b>bold</b> world.</p><p>Second paragraph.</p>`
|
||
doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
txt := doc.Chapters[0]
|
||
if !strings.Contains(txt, "Hello & bold world.") {
|
||
t.Fatalf("entities/tags not handled: %q", txt)
|
||
}
|
||
if strings.Contains(txt, "<b>") || strings.Contains(txt, "color:red") || strings.Contains(txt, "<title>") {
|
||
t.Fatalf("markup/style/head leaked into text: %q", txt)
|
||
}
|
||
// Two <p> blocks must be separated by a blank line so the chunker sees paragraphs.
|
||
if paras := splitParagraphs(text.NormalizeSource(txt)); len(paras) != 2 {
|
||
t.Fatalf("want 2 paragraphs from 2 <p>, got %d: %#v", len(paras), paras)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBRubyCaptureAndBaseInBody(t *testing.T) {
|
||
body := `<p><ruby>漢字<rt>かんじ</rt></ruby>を読む。</p>` +
|
||
`<p><ruby>東<rp>(</rp><rt>とう</rt><rp>)</rp>京<rp>(</rp><rt>きょう</rt><rp>)</rp></ruby>タワー。</p>` +
|
||
`<p><ruby>字<rt></rt></ruby>だけ。</p>` // empty reading → not captured, base kept
|
||
doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
txt := doc.Chapters[0]
|
||
// Base stays in the body; readings and rp parens do NOT.
|
||
for _, want := range []string{"漢字を読む", "東京タワー", "字だけ"} {
|
||
if !strings.Contains(txt, want) {
|
||
t.Fatalf("base missing from body %q (want %q)", txt, want)
|
||
}
|
||
}
|
||
for _, bad := range []string{"かんじ", "とう", "きょう", "(", ")"} {
|
||
if strings.Contains(txt, bad) {
|
||
t.Fatalf("reading/paren %q leaked into body %q", bad, txt)
|
||
}
|
||
}
|
||
// Two readings captured (mono-ruby merged to one base+reading); empty rt skipped.
|
||
got := map[string]string{}
|
||
for _, r := range doc.Ruby {
|
||
got[r.Base] = r.Reading
|
||
if r.Chapter != 1 {
|
||
t.Fatalf("ruby chapter = %d, want 1", r.Chapter)
|
||
}
|
||
}
|
||
if len(doc.Ruby) != 2 || got["漢字"] != "かんじ" || got["東京"] != "とうきょう" {
|
||
t.Fatalf("ruby capture = %#v", doc.Ruby)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBRubyChapterMatchesDenseNumbering(t *testing.T) {
|
||
// An empty cover page in the spine before a ruby-bearing chapter must NOT shift
|
||
// ruby's first_chapter off the number SplitChunks assigns (dense — cover skipped).
|
||
chapters := []chunktest.Chapter{
|
||
{ID: "cover", Href: "cover.xhtml", Body: `<div> </div>`}, // whitespace only → no chapter number
|
||
{ID: "c1", Href: "ch1.xhtml", Body: `<p>ふつうの文。</p>`},
|
||
{ID: "c2", Href: "ch2.xhtml", Body: `<p><ruby>朱雀<rt>すざく</rt></ruby>が舞う。</p>`},
|
||
}
|
||
doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"cover", "c1", "c2"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The ruby is in the 3rd spine item but the 2nd NON-empty chapter → chapter 2,
|
||
// exactly what SplitChunks(doc.Chapters) labels it.
|
||
if len(doc.Ruby) != 1 || doc.Ruby[0].Chapter != 2 {
|
||
t.Fatalf("ruby dense chapter = %#v, want chapter 2", doc.Ruby)
|
||
}
|
||
chunks := SplitChunks(doc.Chapters, testSeg(), nil)
|
||
var rubyChunkChapter int
|
||
for _, c := range chunks {
|
||
if strings.Contains(c.Text, "朱雀") {
|
||
rubyChunkChapter = c.Chapter
|
||
}
|
||
}
|
||
if rubyChunkChapter != doc.Ruby[0].Chapter {
|
||
t.Fatalf("ruby first_chapter %d != chunk chapter %d — memory-v2 since_ch would be off",
|
||
doc.Ruby[0].Chapter, rubyChunkChapter)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBAcceptsGenericAndParameterizedMediaTypes(t *testing.T) {
|
||
// Real epubs mislabel xhtml chapters as application/xml or text/xml (incl. a .xml
|
||
// href — external-review #2), or add a "; charset=utf-8" parameter. All must be
|
||
// read, not silently dropped (a dropped chapter also shifts every since_ch after).
|
||
chapters := []chunktest.Chapter{
|
||
{ID: "c1", Href: "ch1.xml", MType: "application/xml", Body: `<p>第一章。</p>`},
|
||
{ID: "c2", Href: "ch2.xhtml", MType: "text/xml", Body: `<p>第二章。</p>`},
|
||
{ID: "c3", Href: "ch3.xhtml", MType: "application/xhtml+xml; charset=utf-8", Body: `<p>第三章。</p>`},
|
||
}
|
||
doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "c2", "c3"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Chapters) != 3 {
|
||
t.Fatalf("mislabeled/parameterized media-types must all be read, got %d: %#v", len(doc.Chapters), doc.Chapters)
|
||
}
|
||
for i, want := range []string{"第一章。", "第二章。", "第三章。"} {
|
||
if strings.TrimSpace(doc.Chapters[i]) != want {
|
||
t.Fatalf("chapter %d = %q, want %q", i+1, doc.Chapters[i], want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBDanglingIdrefFailsLoud(t *testing.T) {
|
||
// A spine idref with no manifest item, AMONG valid chapters, must fail loud — not
|
||
// silently drop that chapter (which would also shift every later since_ch).
|
||
chapters := []chunktest.Chapter{
|
||
{ID: "c1", Href: "ch1.xhtml", Body: `<p>第一章。</p>`},
|
||
{ID: "c2", Href: "ch2.xhtml", Body: `<p>第二章。</p>`},
|
||
{ID: "c3", Href: "ch3.xhtml", Body: `<p>第三章。</p>`},
|
||
}
|
||
_, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "ghost", "c3"}))
|
||
if err == nil {
|
||
t.Fatal("a dangling spine idref among valid chapters must fail loud, not silently lose a chapter")
|
||
}
|
||
if !strings.Contains(err.Error(), "ghost") {
|
||
t.Fatalf("error should name the dangling idref, got: %v", err)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBBrInsideRubyDoesNotLeak(t *testing.T) {
|
||
// A <br> inside <ruby> must not inject a newline between base glyphs (it would
|
||
// desync base from body and pollute ch.Text — external-review #5).
|
||
body := `<p><ruby>漢<br/>字<rt>かんじ</rt></ruby>を読む。</p>`
|
||
doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "漢字" || doc.Ruby[0].Reading != "かんじ" {
|
||
t.Fatalf("ruby base/reading = %#v", doc.Ruby)
|
||
}
|
||
if !strings.Contains(doc.Chapters[0], "漢字を読む") || strings.Contains(doc.Chapters[0], "漢\n字") {
|
||
t.Fatalf("<br> leaked a newline between base glyphs: %q", doc.Chapters[0])
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBRubyBaseWhitespaceCollapsed(t *testing.T) {
|
||
// Pretty-printed jukugo ruby: whitespace BETWEEN <rb> base segments must not
|
||
// enter the captured base (mis-keys the glossary) nor the body (pollutes ch.Text).
|
||
body := "<p><ruby>\n <rb>旧</rb>\n <rb>字</rb>\n <rb>体</rb>\n <rt>きゅう</rt><rt>じ</rt><rt>たい</rt>\n</ruby>の話。</p>"
|
||
doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "旧字体" || doc.Ruby[0].Reading != "きゅうじたい" {
|
||
t.Fatalf("ruby base/reading not clean: %#v", doc.Ruby)
|
||
}
|
||
if !strings.Contains(doc.Chapters[0], "旧字体の話") {
|
||
t.Fatalf("clean base missing from Body: %q", doc.Chapters[0])
|
||
}
|
||
for _, bad := range []string{"旧 字", "旧\n"} {
|
||
if strings.Contains(doc.Chapters[0], bad) {
|
||
t.Fatalf("ruby-internal whitespace leaked into Body: %q", doc.Chapters[0])
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBSkipsNonXHTML(t *testing.T) {
|
||
chapters := []chunktest.Chapter{
|
||
{ID: "img", Href: "cover.jpg", MType: "image/jpeg", Body: "\xff\xd8not-really-jpeg"},
|
||
{ID: "c1", Href: "ch1.xhtml", Body: `<p>本文。</p>`},
|
||
}
|
||
doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"img", "c1"}))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(doc.Chapters) != 1 || strings.TrimSpace(doc.Chapters[0]) != "本文。" {
|
||
t.Fatalf("non-xhtml spine item must be skipped: %#v", doc.Chapters)
|
||
}
|
||
}
|
||
|
||
func TestIngestEPUBBrokenFailsLoud(t *testing.T) {
|
||
t.Run("not a zip", func(t *testing.T) {
|
||
p := filepath.Join(t.TempDir(), "bad.epub")
|
||
os.WriteFile(p, []byte("this is not a zip"), 0o644)
|
||
if _, err := ingest(p); err == nil {
|
||
t.Fatal("a non-zip .epub must error, not panic")
|
||
}
|
||
})
|
||
t.Run("missing container", func(t *testing.T) {
|
||
p := filepath.Join(t.TempDir(), "noc.epub")
|
||
f, _ := os.Create(p)
|
||
zw := zip.NewWriter(f)
|
||
w, _ := zw.Create("OEBPS/content.opf")
|
||
w.Write([]byte("<package/>"))
|
||
zw.Close()
|
||
f.Close()
|
||
if _, err := ingest(p); err == nil {
|
||
t.Fatal("missing container.xml must error")
|
||
}
|
||
})
|
||
t.Run("empty spine", func(t *testing.T) {
|
||
p := chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: `<p>x</p>`}}, nil)
|
||
if _, err := ingest(p); err == nil {
|
||
t.Fatal("an empty spine must error")
|
||
}
|
||
})
|
||
t.Run("dangling spine idref", func(t *testing.T) {
|
||
// spine references a missing manifest id → resolves to zero chapters → error.
|
||
p := chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: `<p>x</p>`}}, []string{"ghost"})
|
||
if _, err := ingest(p); err == nil {
|
||
t.Fatal("a spine resolving to no readable chapters must error")
|
||
}
|
||
})
|
||
}
|