package chunk import ( "archive/zip" "os" "path/filepath" "strings" "testing" "textmachine/backend/internal/chunk/chunktest" "textmachine/backend/internal/text" ) // --- txt ------------------------------------------------------------------------ func TestIngestTXTSingleChapter(t *testing.T) { path := filepath.Join(t.TempDir(), "src.txt") if err := os.WriteFile(path, []byte("静かな図書館の朝。"), 0o644); err != nil { t.Fatal(err) } doc, err := ingest(path) if err != nil { t.Fatal(err) } if len(doc.Chapters) != 1 || doc.Chapters[0] != "静かな図書館の朝。" { t.Fatalf("chapters = %#v", doc.Chapters) } if len(doc.Ruby) != 0 { t.Fatalf("txt has no ruby, got %#v", doc.Ruby) } } func TestIngestTXTFormFeedChapters(t *testing.T) { path := filepath.Join(t.TempDir(), "src.txt") if err := os.WriteFile(path, []byte("Глава A"+chapterSep+"Глава B"+chapterSep+"Глава C"), 0o644); err != nil { t.Fatal(err) } doc, err := ingest(path) if err != nil { t.Fatal(err) } want := []string{"Глава A", "Глава B", "Глава C"} if strings.Join(doc.Chapters, "|") != strings.Join(want, "|") { t.Fatalf("chapters = %#v, want %#v", doc.Chapters, want) } } func TestIngestTXTNormalizes(t *testing.T) { path := filepath.Join(t.TempDir(), "src.txt") // UTF-8 BOM + CRLF + surrounding whitespace must be normalized away. if err := os.WriteFile(path, []byte("\uFEFF строка один\r\nстрока два "), 0o644); err != nil { t.Fatal(err) } doc, err := ingest(path) if err != nil { t.Fatal(err) } if doc.Chapters[0] != "строка один\nстрока два" { t.Fatalf("normalized = %q", doc.Chapters[0]) } } func TestIngestEPUBSpineOrder(t *testing.T) { // Manifest order (c3,c1,c2) differs from spine order (c1,c2,c3): the spine wins. chapters := []chunktest.Chapter{ {ID: "c3", Href: "ch3.xhtml", Body: `

第三章。

`}, {ID: "c1", Href: "ch1.xhtml", Body: `

第一章。

`}, {ID: "c2", Href: "ch2.xhtml", Body: `

第二章。

`}, } doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "c2", "c3"})) if err != nil { t.Fatal(err) } if len(doc.Chapters) != 3 { t.Fatalf("want 3 chapters, got %d: %#v", len(doc.Chapters), doc.Chapters) } for i, want := range []string{"第一章。", "第二章。", "第三章。"} { if strings.TrimSpace(doc.Chapters[i]) != want { t.Fatalf("chapter %d = %q, want %q (spine order broken)", i+1, doc.Chapters[i], want) } } } func TestIngestEPUBStripsTagsAndEntities(t *testing.T) { body := `

Hello & bold world.

Second paragraph.

` doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"})) if err != nil { t.Fatal(err) } txt := doc.Chapters[0] if !strings.Contains(txt, "Hello & bold world.") { t.Fatalf("entities/tags not handled: %q", txt) } if strings.Contains(txt, "") || strings.Contains(txt, "color:red") || strings.Contains(txt, "") { t.Fatalf("markup/style/head leaked into text: %q", txt) } // Two <p> blocks must be separated by a blank line so the chunker sees paragraphs. if paras := splitParagraphs(text.NormalizeSource(txt)); len(paras) != 2 { t.Fatalf("want 2 paragraphs from 2 <p>, got %d: %#v", len(paras), paras) } } func TestIngestEPUBRubyCaptureAndBaseInBody(t *testing.T) { body := `<p><ruby>漢字<rt>かんじ</rt></ruby>を読む。</p>` + `<p><ruby>東<rp>(</rp><rt>とう</rt><rp>)</rp>京<rp>(</rp><rt>きょう</rt><rp>)</rp></ruby>タワー。</p>` + `<p><ruby>字<rt></rt></ruby>だけ。</p>` // empty reading → not captured, base kept doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"})) if err != nil { t.Fatal(err) } txt := doc.Chapters[0] // Base stays in the body; readings and rp parens do NOT. for _, want := range []string{"漢字を読む", "東京タワー", "字だけ"} { if !strings.Contains(txt, want) { t.Fatalf("base missing from body %q (want %q)", txt, want) } } for _, bad := range []string{"かんじ", "とう", "きょう", "(", ")"} { if strings.Contains(txt, bad) { t.Fatalf("reading/paren %q leaked into body %q", bad, txt) } } // Two readings captured (mono-ruby merged to one base+reading); empty rt skipped. got := map[string]string{} for _, r := range doc.Ruby { got[r.Base] = r.Reading if r.Chapter != 1 { t.Fatalf("ruby chapter = %d, want 1", r.Chapter) } } if len(doc.Ruby) != 2 || got["漢字"] != "かんじ" || got["東京"] != "とうきょう" { t.Fatalf("ruby capture = %#v", doc.Ruby) } } func TestIngestEPUBRubyChapterMatchesDenseNumbering(t *testing.T) { // An empty cover page in the spine before a ruby-bearing chapter must NOT shift // ruby's first_chapter off the number SplitChunks assigns (dense — cover skipped). chapters := []chunktest.Chapter{ {ID: "cover", Href: "cover.xhtml", Body: `<div> </div>`}, // whitespace only → no chapter number {ID: "c1", Href: "ch1.xhtml", Body: `<p>ふつうの文。</p>`}, {ID: "c2", Href: "ch2.xhtml", Body: `<p><ruby>朱雀<rt>すざく</rt></ruby>が舞う。</p>`}, } doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"cover", "c1", "c2"})) if err != nil { t.Fatal(err) } // The ruby is in the 3rd spine item but the 2nd NON-empty chapter → chapter 2, // exactly what SplitChunks(doc.Chapters) labels it. if len(doc.Ruby) != 1 || doc.Ruby[0].Chapter != 2 { t.Fatalf("ruby dense chapter = %#v, want chapter 2", doc.Ruby) } chunks := SplitChunks(doc.Chapters, testSeg(), nil) var rubyChunkChapter int for _, c := range chunks { if strings.Contains(c.Text, "朱雀") { rubyChunkChapter = c.Chapter } } if rubyChunkChapter != doc.Ruby[0].Chapter { t.Fatalf("ruby first_chapter %d != chunk chapter %d — memory-v2 since_ch would be off", doc.Ruby[0].Chapter, rubyChunkChapter) } } func TestIngestEPUBAcceptsGenericAndParameterizedMediaTypes(t *testing.T) { // Real epubs mislabel xhtml chapters as application/xml or text/xml (incl. a .xml // href — external-review #2), or add a "; charset=utf-8" parameter. All must be // read, not silently dropped (a dropped chapter also shifts every since_ch after). chapters := []chunktest.Chapter{ {ID: "c1", Href: "ch1.xml", MType: "application/xml", Body: `<p>第一章。</p>`}, {ID: "c2", Href: "ch2.xhtml", MType: "text/xml", Body: `<p>第二章。</p>`}, {ID: "c3", Href: "ch3.xhtml", MType: "application/xhtml+xml; charset=utf-8", Body: `<p>第三章。</p>`}, } doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "c2", "c3"})) if err != nil { t.Fatal(err) } if len(doc.Chapters) != 3 { t.Fatalf("mislabeled/parameterized media-types must all be read, got %d: %#v", len(doc.Chapters), doc.Chapters) } for i, want := range []string{"第一章。", "第二章。", "第三章。"} { if strings.TrimSpace(doc.Chapters[i]) != want { t.Fatalf("chapter %d = %q, want %q", i+1, doc.Chapters[i], want) } } } func TestIngestEPUBDanglingIdrefFailsLoud(t *testing.T) { // A spine idref with no manifest item, AMONG valid chapters, must fail loud — not // silently drop that chapter (which would also shift every later since_ch). chapters := []chunktest.Chapter{ {ID: "c1", Href: "ch1.xhtml", Body: `<p>第一章。</p>`}, {ID: "c2", Href: "ch2.xhtml", Body: `<p>第二章。</p>`}, {ID: "c3", Href: "ch3.xhtml", Body: `<p>第三章。</p>`}, } _, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "ghost", "c3"})) if err == nil { t.Fatal("a dangling spine idref among valid chapters must fail loud, not silently lose a chapter") } if !strings.Contains(err.Error(), "ghost") { t.Fatalf("error should name the dangling idref, got: %v", err) } } func TestIngestEPUBBrInsideRubyDoesNotLeak(t *testing.T) { // A <br> inside <ruby> must not inject a newline between base glyphs (it would // desync base from body and pollute ch.Text — external-review #5). body := `<p><ruby>漢<br/>字<rt>かんじ</rt></ruby>を読む。</p>` doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"})) if err != nil { t.Fatal(err) } if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "漢字" || doc.Ruby[0].Reading != "かんじ" { t.Fatalf("ruby base/reading = %#v", doc.Ruby) } if !strings.Contains(doc.Chapters[0], "漢字を読む") || strings.Contains(doc.Chapters[0], "漢\n字") { t.Fatalf("<br> leaked a newline between base glyphs: %q", doc.Chapters[0]) } } func TestIngestEPUBRubyBaseWhitespaceCollapsed(t *testing.T) { // Pretty-printed jukugo ruby: whitespace BETWEEN <rb> base segments must not // enter the captured base (mis-keys the glossary) nor the body (pollutes ch.Text). body := "<p><ruby>\n <rb>旧</rb>\n <rb>字</rb>\n <rb>体</rb>\n <rt>きゅう</rt><rt>じ</rt><rt>たい</rt>\n</ruby>の話。</p>" doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"})) if err != nil { t.Fatal(err) } if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "旧字体" || doc.Ruby[0].Reading != "きゅうじたい" { t.Fatalf("ruby base/reading not clean: %#v", doc.Ruby) } if !strings.Contains(doc.Chapters[0], "旧字体の話") { t.Fatalf("clean base missing from Body: %q", doc.Chapters[0]) } for _, bad := range []string{"旧 字", "旧\n"} { if strings.Contains(doc.Chapters[0], bad) { t.Fatalf("ruby-internal whitespace leaked into Body: %q", doc.Chapters[0]) } } } func TestIngestEPUBSkipsNonXHTML(t *testing.T) { chapters := []chunktest.Chapter{ {ID: "img", Href: "cover.jpg", MType: "image/jpeg", Body: "\xff\xd8not-really-jpeg"}, {ID: "c1", Href: "ch1.xhtml", Body: `<p>本文。</p>`}, } doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"img", "c1"})) if err != nil { t.Fatal(err) } if len(doc.Chapters) != 1 || strings.TrimSpace(doc.Chapters[0]) != "本文。" { t.Fatalf("non-xhtml spine item must be skipped: %#v", doc.Chapters) } } func TestIngestEPUBBrokenFailsLoud(t *testing.T) { t.Run("not a zip", func(t *testing.T) { p := filepath.Join(t.TempDir(), "bad.epub") os.WriteFile(p, []byte("this is not a zip"), 0o644) if _, err := ingest(p); err == nil { t.Fatal("a non-zip .epub must error, not panic") } }) t.Run("missing container", func(t *testing.T) { p := filepath.Join(t.TempDir(), "noc.epub") f, _ := os.Create(p) zw := zip.NewWriter(f) w, _ := zw.Create("OEBPS/content.opf") w.Write([]byte("<package/>")) zw.Close() f.Close() if _, err := ingest(p); err == nil { t.Fatal("missing container.xml must error") } }) t.Run("empty spine", func(t *testing.T) { p := chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: `<p>x</p>`}}, nil) if _, err := ingest(p); err == nil { t.Fatal("an empty spine must error") } }) t.Run("dangling spine idref", func(t *testing.T) { // spine references a missing manifest id → resolves to zero chapters → error. p := chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: `<p>x</p>`}}, []string{"ghost"}) if _, err := ingest(p); err == nil { t.Fatal("a spine resolving to no readable chapters must error") } }) }