textmachine/backend/internal/chunk/ingest_test.go

package chunk

import (
	"archive/zip"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"textmachine/backend/internal/chunk/chunktest"
	"textmachine/backend/internal/text"
)

// --- txt ------------------------------------------------------------------------

func TestIngestTXTSingleChapter(t *testing.T) {
	path := filepath.Join(t.TempDir(), "src.txt")
	if err := os.WriteFile(path, []byte("静かな図書館の朝。"), 0o644); err != nil {
		t.Fatal(err)
	}
	doc, err := ingest(path)
	if err != nil {
		t.Fatal(err)
	}
	if len(doc.Chapters) != 1 || doc.Chapters[0] != "静かな図書館の朝。" {
		t.Fatalf("chapters = %#v", doc.Chapters)
	}
	if len(doc.Ruby) != 0 {
		t.Fatalf("txt has no ruby, got %#v", doc.Ruby)
	}
}

func TestIngestTXTFormFeedChapters(t *testing.T) {
	path := filepath.Join(t.TempDir(), "src.txt")
	if err := os.WriteFile(path, []byte("Глава A"+chapterSep+"Глава B"+chapterSep+"Глава C"), 0o644); err != nil {
		t.Fatal(err)
	}
	doc, err := ingest(path)
	if err != nil {
		t.Fatal(err)
	}
	want := []string{"Глава A", "Глава B", "Глава C"}
	if strings.Join(doc.Chapters, "|") != strings.Join(want, "|") {
		t.Fatalf("chapters = %#v, want %#v", doc.Chapters, want)
	}
}

// TestTXTProvenanceNamesTheWinningPath pins the provenance of the cut, which is what a consumer offering an
// order «through chapter N» trusts when it decides whether N names something the file said or something this
// engine guessed.
//
// ⛔ THE PATHS COMPETE. The old name asked whether the format drew EVERY boundary, a question that no longer
// resolves: with competition exactly one path draws all of them, so there is no mixed case to judge. A form
// feed now witnesses `delimited` — it is the ASCII PAGE break, not a statement about chapters — and matched
// headers witness `detected` whatever else is in the bytes.
func TestTXTProvenanceNamesTheWinningPath(t *testing.T) {
	// A header line the matcher recognises, followed by enough prose to be a chapter.
	chap := func(n string) string { return n + "\n" + strings.Repeat("蛊", 200) }

	for _, tc := range []struct {
		name string
		body string
		want string
	}{
		// Every boundary is a form feed, and no header was matched: the file carried a SEPARATOR, and calling
		// it a chapter is this package's assumption about a page break.
		{"a page break is a separator, not a chapter", "ГЛАВА A" + chapterSep + "ГЛАВА B", StructureDelimited},
		// No form feed at all: every boundary is this package's inference from prose.
		{"the matcher found them all", chap("第一章") + "\n" + chap("第二章"), StructureDetected},
		// ⛔ THE CASE THE ORDER OF OPERATIONS DECIDES: a header sits immediately after the form feed, sharing
		// a LINE with the previous chapter's prose. Detect before neutralising and it is invisible — see
		// TestTXTHeaderRightAfterAFormFeedIsNotLost, which holds the COUNT this case cannot.
		{"one stray form feed among matched headers", chap("第一章") + "\n" + chap("第二章") + chapterSep + chap("第三章"), StructureDetected},
	} {
		t.Run(tc.name, func(t *testing.T) {
			path := filepath.Join(t.TempDir(), "src.txt")
			if err := os.WriteFile(path, []byte(tc.body), 0o644); err != nil {
				t.Fatal(err)
			}
			doc, err := ingest(path)
			if err != nil {
				t.Fatal(err)
			}
			if doc.Structure != tc.want {
				t.Fatalf("structure = %q, want %q (chapters: %d)", doc.Structure, tc.want, len(doc.Chapters))
			}
		})
	}
}

// TestTXTHeaderRightAfterAFormFeedIsNotLost holds the number this package's provenance test cannot: with the
// naive order the label stays `detected` and is CORRECT while a whole chapter has vanished, so a test that
// checks only Structure passes over a silent loss. Chapter three lives on the same LINE as chapter two's
// prose (there is no \n before a \f), 204 runes long and opening with 蛊 — rejected by both the length bound
// and the ^ anchor unless the form feed became a paragraph break first.
func TestTXTHeaderRightAfterAFormFeedIsNotLost(t *testing.T) {
	chap := func(n string) string { return n + "\n" + strings.Repeat("蛊", 200) }
	body := chap("第一章") + "\n" + chap("第二章") + chapterSep + chap("第三章")
	path := filepath.Join(t.TempDir(), "src.txt")
	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
		t.Fatal(err)
	}
	doc, err := ingest(path)
	if err != nil {
		t.Fatal(err)
	}
	if len(doc.Chapters) != 3 {
		t.Fatalf("chapters = %d, want 3 — the header after the form feed was swallowed", len(doc.Chapters))
	}
	if !strings.Contains(doc.Chapters[2], "第三章") {
		t.Fatalf("chapter 3 does not open with its header: %.40q", doc.Chapters[2])
	}
	// And the ignored page break must not survive into the text the model and the content hash see.
	for i, ch := range doc.Chapters {
		if strings.Contains(ch, "\f") {
			t.Fatalf("chapter %d still carries a form feed", i+1)
		}
	}
	if doc.FormFeedsIgnored != 1 {
		t.Fatalf("form feeds ignored = %d, want 1", doc.FormFeedsIgnored)
	}
}

func TestIngestTXTNormalizes(t *testing.T) {
	path := filepath.Join(t.TempDir(), "src.txt")
	// UTF-8 BOM + CRLF + surrounding whitespace must be normalized away.
	if err := os.WriteFile(path, []byte("\uFEFF  строка один\r\nстрока два  "), 0o644); err != nil {
		t.Fatal(err)
	}
	doc, err := ingest(path)
	if err != nil {
		t.Fatal(err)
	}
	if doc.Chapters[0] != "строка один\nстрока два" {
		t.Fatalf("normalized = %q", doc.Chapters[0])
	}
}

func TestIngestEPUBSpineOrder(t *testing.T) {
	// Manifest order (c3,c1,c2) differs from spine order (c1,c2,c3): the spine wins.
	chapters := []chunktest.Chapter{
		{ID: "c3", Href: "ch3.xhtml", Body: `<p>第三章。</p>`},
		{ID: "c1", Href: "ch1.xhtml", Body: `<p>第一章。</p>`},
		{ID: "c2", Href: "ch2.xhtml", Body: `<p>第二章。</p>`},
	}
	doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "c2", "c3"}))
	if err != nil {
		t.Fatal(err)
	}
	if len(doc.Chapters) != 3 {
		t.Fatalf("want 3 chapters, got %d: %#v", len(doc.Chapters), doc.Chapters)
	}
	for i, want := range []string{"第一章。", "第二章。", "第三章。"} {
		if strings.TrimSpace(doc.Chapters[i]) != want {
			t.Fatalf("chapter %d = %q, want %q (spine order broken)", i+1, doc.Chapters[i], want)
		}
	}
}

func TestIngestEPUBStripsTagsAndEntities(t *testing.T) {
	body := `<p>Hello &amp; <b>bold</b> world.</p><p>Second&#160;paragraph.</p>`
	doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"}))
	if err != nil {
		t.Fatal(err)
	}
	txt := doc.Chapters[0]
	if !strings.Contains(txt, "Hello & bold world.") {
		t.Fatalf("entities/tags not handled: %q", txt)
	}
	if strings.Contains(txt, "<b>") || strings.Contains(txt, "color:red") || strings.Contains(txt, "<title>") {
		t.Fatalf("markup/style/head leaked into text: %q", txt)
	}
	// Two <p> blocks must be separated by a blank line so the chunker sees paragraphs.
	if paras := splitParagraphs(text.NormalizeSource(txt)); len(paras) != 2 {
		t.Fatalf("want 2 paragraphs from 2 <p>, got %d: %#v", len(paras), paras)
	}
}

func TestIngestEPUBRubyCaptureAndBaseInBody(t *testing.T) {
	body := `<p><ruby>漢字<rt>かんじ</rt></ruby>を読む。</p>` +
		`<p><ruby>東<rp>(</rp><rt>とう</rt><rp>)</rp>京<rp>(</rp><rt>きょう</rt><rp>)</rp></ruby>タワー。</p>` +
		`<p><ruby>字<rt></rt></ruby>だけ。</p>` // empty reading → not captured, base kept
	doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"}))
	if err != nil {
		t.Fatal(err)
	}
	txt := doc.Chapters[0]
	// Base stays in the body; readings and rp parens do NOT.
	for _, want := range []string{"漢字を読む", "東京タワー", "字だけ"} {
		if !strings.Contains(txt, want) {
			t.Fatalf("base missing from body %q (want %q)", txt, want)
		}
	}
	for _, bad := range []string{"かんじ", "とう", "きょう", "(", ")"} {
		if strings.Contains(txt, bad) {
			t.Fatalf("reading/paren %q leaked into body %q", bad, txt)
		}
	}
	// Two readings captured (mono-ruby merged to one base+reading); empty rt skipped.
	got := map[string]string{}
	for _, r := range doc.Ruby {
		got[r.Base] = r.Reading
		if r.Chapter != 1 {
			t.Fatalf("ruby chapter = %d, want 1", r.Chapter)
		}
	}
	if len(doc.Ruby) != 2 || got["漢字"] != "かんじ" || got["東京"] != "とうきょう" {
		t.Fatalf("ruby capture = %#v", doc.Ruby)
	}
}

func TestIngestEPUBRubyChapterMatchesDenseNumbering(t *testing.T) {
	// An empty cover page in the spine before a ruby-bearing chapter must NOT shift
	// ruby's first_chapter off the number SplitChunks assigns (dense — cover skipped).
	chapters := []chunktest.Chapter{
		{ID: "cover", Href: "cover.xhtml", Body: `<div>   </div>`}, // whitespace only → no chapter number
		{ID: "c1", Href: "ch1.xhtml", Body: `<p>ふつうの文。</p>`},
		{ID: "c2", Href: "ch2.xhtml", Body: `<p><ruby>朱雀<rt>すざく</rt></ruby>が舞う。</p>`},
	}
	doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"cover", "c1", "c2"}))
	if err != nil {
		t.Fatal(err)
	}
	// The ruby is in the 3rd spine item but the 2nd NON-empty chapter → chapter 2,
	// exactly what SplitChunks(doc.Chapters) labels it.
	if len(doc.Ruby) != 1 || doc.Ruby[0].Chapter != 2 {
		t.Fatalf("ruby dense chapter = %#v, want chapter 2", doc.Ruby)
	}
	chunks := SplitChunks(doc.Chapters, testSeg(), nil, testAbbrevs())
	var rubyChunkChapter int
	for _, c := range chunks {
		if strings.Contains(c.Text, "朱雀") {
			rubyChunkChapter = c.Chapter
		}
	}
	if rubyChunkChapter != doc.Ruby[0].Chapter {
		t.Fatalf("ruby first_chapter %d != chunk chapter %d — memory-v2 since_ch would be off",
			doc.Ruby[0].Chapter, rubyChunkChapter)
	}
}

func TestIngestEPUBAcceptsGenericAndParameterizedMediaTypes(t *testing.T) {
	// Real epubs mislabel xhtml chapters as application/xml or text/xml (incl. a .xml
	// href — external-review #2), or add a "; charset=utf-8" parameter. All must be
	// read, not silently dropped (a dropped chapter also shifts every since_ch after).
	chapters := []chunktest.Chapter{
		{ID: "c1", Href: "ch1.xml", MType: "application/xml", Body: `<p>第一章。</p>`},
		{ID: "c2", Href: "ch2.xhtml", MType: "text/xml", Body: `<p>第二章。</p>`},
		{ID: "c3", Href: "ch3.xhtml", MType: "application/xhtml+xml; charset=utf-8", Body: `<p>第三章。</p>`},
	}
	doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "c2", "c3"}))
	if err != nil {
		t.Fatal(err)
	}
	if len(doc.Chapters) != 3 {
		t.Fatalf("mislabeled/parameterized media-types must all be read, got %d: %#v", len(doc.Chapters), doc.Chapters)
	}
	for i, want := range []string{"第一章。", "第二章。", "第三章。"} {
		if strings.TrimSpace(doc.Chapters[i]) != want {
			t.Fatalf("chapter %d = %q, want %q", i+1, doc.Chapters[i], want)
		}
	}
}

// The four ways real dirty epubs used to kill an import outright: encoding/xml aborted the whole
// chapter on each, so ONE malformed document lost the book. The tokenizer reads prose out of all
// four. Each case asserts the prose survives AND that the markup noise does not enter it.
func TestIngestEPUBDirtyXHTMLImports(t *testing.T) {
	cases := []struct {
		name, body string
		wantIn     []string
		wantNotIn  []string
	}{{
		name:      "bare < in prose",
		body:      `<p>Если a < b, то дальше.</p><p>Второй абзац.</p>`,
		wantIn:    []string{"Если a < b, то дальше.", "Второй абзац."},
		wantNotIn: []string{"<p>"},
	}, {
		name:      "overlapping tags",
		body:      `<p><b>жирный <i>оба</b> курсив</i> хвост.</p>`,
		wantIn:    []string{"жирный", "оба", "курсив", "хвост."},
		wantNotIn: []string{"<b>", "<i>"},
	}, {
		name:      "script content looks like markup",
		body:      `<script>if (a<b && c>d) { document.write("</p>"); }</script><p>Настоящий текст.</p>`,
		wantIn:    []string{"Настоящий текст."},
		wantNotIn: []string{"document.write", "a<b"},
	}, {
		name:      "-- inside a comment",
		body:      `<!-- v2 (26.07) -- черновая заметка -- --><p>Настоящий текст.</p>`,
		wantIn:    []string{"Настоящий текст."},
		wantNotIn: []string{"черновая заметка", "v2 (26.07)"},
	}}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			doc, err := ingest(chunktest.BuildEPUB(t,
				[]chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: c.body}}, []string{"c1"}))
			if err != nil {
				t.Fatalf("a dirty xhtml chapter must still import: %v", err)
			}
			if len(doc.Chapters) != 1 {
				t.Fatalf("want 1 chapter, got %#v", doc.Chapters)
			}
			got := doc.Chapters[0]
			for _, w := range c.wantIn {
				if !strings.Contains(got, w) {
					t.Errorf("prose %q lost from extraction: %q", w, got)
				}
			}
			for _, w := range c.wantNotIn {
				if strings.Contains(got, w) {
					t.Errorf("noise %q leaked into extraction: %q", w, got)
				}
			}
		})
	}
}

// The <head> subtree is skipped by NAME, not by raw nesting depth: its void children (<meta>,
// <link>) emit no end tag, so a depth counter would never unwind and the whole chapter would
// vanish. Both spellings — self-closed and bare — must leave the body intact.
func TestIngestEPUBVoidTagsInHeadDoNotSwallowChapter(t *testing.T) {
	for _, head := range []string{
		`<meta charset="utf-8"/><link rel="stylesheet" href="s.css"/><title>T</title>`,
		`<meta charset="utf-8"><link rel="stylesheet" href="s.css"><title>T</title>`,
	} {
		raw := `<?xml version="1.0" encoding="utf-8"?><html xmlns="http://www.w3.org/1999/xhtml">` +
			`<head>` + head + `</head><body><p>Тело главы.</p></body></html>`
		body, _, err := extractXHTML([]byte(raw))
		if err != nil {
			t.Fatalf("head %q: %v", head, err)
		}
		if !strings.Contains(body, "Тело главы.") {
			t.Fatalf("head %q swallowed the body: %q", head, body)
		}
		if strings.Contains(body, "T") && strings.Contains(body, "<title>") {
			t.Fatalf("head content leaked: %q", body)
		}
	}
	// An UNCLOSED <head> must not eat the chapter either — <body> ends it.
	raw := `<html><head><meta charset="utf-8"><body><p>Тело главы.</p></body></html>`
	body, _, err := extractXHTML([]byte(raw))
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(body, "Тело главы.") {
		t.Fatalf("an unclosed <head> swallowed the chapter: %q", body)
	}
}

// Byte-parity with the previous encoding/xml reader on the constructs where the tokenizer differs
// most: it emits ONE token for a void or self-closed element where the xml decoder synthesised a
// start AND an end (HTMLAutoClose). The expected strings below were measured against that reader,
// so a regression here is a silent re-chunk of every book carrying <hr> or <br>.
func TestExtractXHTMLVoidTagByteParity(t *testing.T) {
	for _, c := range []struct{ name, doc, want string }{
		// <hr> is a BLOCK tag: the old reader wrote a break for its start and another for its
		// synthesised end. Both spellings must produce the same bytes.
		{"hr self-closed", `<html><body><p>A</p><hr/><p>B</p></body></html>`, "\n\nA\n\n\n\n\n\n\n\nB\n\n"},
		{"hr bare", `<html><body><p>A</p><hr><p>B</p></body></html>`, "\n\nA\n\n\n\n\n\n\n\nB\n\n"},
		// <br> is not a block tag: one newline, no end-tag break.
		{"br self-closed", `<html><body><p>A<br/>B</p></body></html>`, "\n\nA\nB\n\n"},
		// A void child inside a skipped subtree must not unbalance the skip and eat the rest.
		{"void inside style", `<html><body><p>A</p><style>x<br/>y</style><p>B</p></body></html>`, "\n\nA\n\n\n\nB\n\n"},
		{"void inside script", `<html><body><p>A</p><script>var i=0;<br/></script><p>B</p></body></html>`, "\n\nA\n\n\n\nB\n\n"},
		{"void inside head", `<html><head><meta charset="utf-8"><style>.a{}</style></head><body><p>A</p></body></html>`, "\n\nA\n\n"},
		// No <body> wrapper: nothing rescues a skip that failed to unwind, so this is what pins
		// that the <head> subtree is tracked by NAME. Its void children emit no end tag, and a
		// blind depth counter would still be inside <head> here and drop the prose entirely.
		{"head without body, bare void", `<html><head><meta charset="utf-8"><link rel="s"></head><p>Проза.</p></html>`, "\n\nПроза.\n\n"},
		{"head without body, self-closed void", `<html><head><meta charset="utf-8"/><title>T</title></head><p>Проза.</p></html>`, "\n\nПроза.\n\n"},
	} {
		t.Run(c.name, func(t *testing.T) {
			got, _, err := extractXHTML([]byte(c.doc))
			if err != nil {
				t.Fatal(err)
			}
			if got != c.want {
				t.Fatalf("extraction drifted from the previous reader\n got  %q\n want %q", got, c.want)
			}
		})
	}
}

// UNBALANCED ruby. HTML5 makes </rt>, </rp> and </rb> optional, and the tokenizer — unlike the xml
// decoder — does not synthesise the implied end tags. Tracking the sub-element as a nesting depth
// leaks on the first omission and routes every LATER base into the reading buffer, deleting it from
// the prose; an unclosed <ruby> likewise suppresses every later paragraph break. Both are content
// loss on the wire, and both are invisible to a clean-corpus comparison. Expected values measured
// against the previous reader.
func TestExtractXHTMLUnbalancedRuby(t *testing.T) {
	t.Run("omitted </rt> does not eat the next ruby", func(t *testing.T) {
		body, ruby, err := extractXHTML([]byte(
			`<html><body><p><ruby>漢<rt>かん</ruby>текст</p><p><ruby>字<rt>じ</rt></ruby>ещё</p></body></html>`))
		if err != nil {
			t.Fatal(err)
		}
		if body != "\n\n漢текст\n\n\n\n字ещё\n\n" {
			t.Fatalf("prose lost after an omitted </rt>: %q", body)
		}
		if len(ruby) != 2 || ruby[0].Base != "漢" || ruby[1].Base != "字" || ruby[1].Reading != "じ" {
			t.Fatalf("ruby capture broken after an omitted </rt>: %#v", ruby)
		}
	})
	t.Run("unclosed <ruby> does not swallow the chapter", func(t *testing.T) {
		body, ruby, err := extractXHTML([]byte(
			`<html><body><p><ruby>漢<rt>かん</rt></p><p>Второй абзац.</p><p>Третий.</p></body></html>`))
		if err != nil {
			t.Fatal(err)
		}
		if body != "\n\n漢\n\n\n\nВторой абзац.\n\n\n\nТретий.\n\n" {
			t.Fatalf("an unclosed <ruby> suppressed the paragraph breaks: %q", body)
		}
		if paras := splitParagraphs(text.NormalizeSource(body)); len(paras) != 3 {
			t.Fatalf("want 3 paragraphs, got %d: %#v", len(paras), paras)
		}
		if len(ruby) != 1 || ruby[0].Base != "漢" || ruby[0].Reading != "かん" {
			t.Fatalf("ruby lost when <ruby> was closed by its block: %#v", ruby)
		}
	})
	t.Run("nested ruby with an unclosed inner </rt>", func(t *testing.T) {
		// The inner </ruby> must clear the sub-mode even though it does not close the OUTER ruby,
		// or the outer base 字 is routed into the reading and disappears from the prose.
		body, ruby, err := extractXHTML([]byte(
			`<html><body><p><ruby><ruby>漢<rt>かん</ruby>字<rt>じ</rt></ruby>текст</p></body></html>`))
		if err != nil {
			t.Fatal(err)
		}
		if body != "\n\n漢字текст\n\n" {
			t.Fatalf("nested ruby lost the outer base: %q", body)
		}
		if len(ruby) != 1 || ruby[0].Base != "漢字" || ruby[0].Reading != "かんじ" {
			t.Fatalf("nested ruby capture = %#v", ruby)
		}
	})
	t.Run("omitted </rp> keeps the parenthesis fallback out of the prose", func(t *testing.T) {
		body, ruby, err := extractXHTML([]byte(
			`<html><body><p><ruby>東<rp>(<rt>とう<rp>)</ruby>текст</p></body></html>`))
		if err != nil {
			t.Fatal(err)
		}
		if body != "\n\n東текст\n\n" {
			t.Fatalf("rp fallback leaked or base lost: %q", body)
		}
		if len(ruby) != 1 || ruby[0].Base != "東" || ruby[0].Reading != "とう" {
			t.Fatalf("ruby capture = %#v", ruby)
		}
	})
}

// Raw-text elements are the tokenizer's sharpest edge. Its raw mode is what makes markup-shaped
// <script> content harmless, but it applies to a whole family of tags, and for a SELF-CLOSED one no
// closing tag ever arrives — raw mode then runs to end of file and dumps the rest of the chapter
// onto the wire as literal markup. Expected strings measured against the previous reader.
func TestExtractXHTMLRawTextElementsByteParity(t *testing.T) {
	const proseTail = "\n\n\n\nПроза.\n\n"
	for _, c := range []struct{ name, doc, want string }{
		// Not in skipRoots: their text is prose and their tags must still be stripped, not emitted.
		{"noscript", `<html><body><noscript><p>Включите JS</p></noscript><p>Проза.</p></body></html>`, "\n\nВключите JS" + proseTail},
		{"textarea", `<html><body><textarea><p>шаблон</p></textarea><p>Проза.</p></body></html>`, "\n\nшаблон" + proseTail},
		{"iframe", `<html><body><iframe><p>фрейм</p></iframe><p>Проза.</p></body></html>`, "\n\nфрейм" + proseTail},
		{"xmp", `<html><body><xmp><p>xmp</p></xmp><p>Проза.</p></body></html>`, "\n\nxmp" + proseTail},
		{"noembed", `<html><body><noembed><p>ne</p></noembed><p>Проза.</p></body></html>`, "\n\nne" + proseTail},
		{"noframes", `<html><body><noframes><p>nf</p></noframes><p>Проза.</p></body></html>`, "\n\nnf" + proseTail},
		{"plaintext", `<html><body><plaintext><p>pt</p></plaintext><p>Проза.</p></body></html>`, "\n\npt" + proseTail},
		// Self-closed raw-text tags: the whole rest of the chapter used to survive; it must still.
		{"self-closed script", `<html><body><p>До.</p><script src="x.js"/><p>После.</p></body></html>`, "\n\nДо.\n\n\n\nПосле.\n\n"},
		{"self-closed style", `<html><body><p>До.</p><style/><p>После.</p></body></html>`, "\n\nДо.\n\n\n\nПосле.\n\n"},
		{"self-closed title", `<html><body><title/><p>Проза.</p></body></html>`, "\n\nПроза.\n\n"},
	} {
		t.Run(c.name, func(t *testing.T) {
			got, _, err := extractXHTML([]byte(c.doc))
			if err != nil {
				t.Fatal(err)
			}
			if got != c.want {
				t.Fatalf("raw-text handling drifted from the previous reader\n got  %q\n want %q", got, c.want)
			}
			if strings.Contains(got, "<p>") || strings.Contains(got, "</body>") {
				t.Fatalf("literal markup reached the prose: %q", got)
			}
		})
	}
	// …while a NON-self-closed <script>/<style> keeps raw mode, which is what makes markup-shaped
	// JS harmless (dirty class iii). Both properties have to hold at once.
	got, _, err := extractXHTML([]byte(`<html><body><script>if(a<b){x("</p>")}</script><p>Проза.</p></body></html>`))
	if err != nil {
		t.Fatal(err)
	}
	if strings.TrimSpace(got) != "Проза." {
		t.Fatalf("script raw-text mode broken: %q", got)
	}
}

// The tokenizer resolves HTML5 legacy entities the xml decoder left literal — a bare «&» followed
// by a known name (&amp, &copy, &nbsp …) with no semicolon. That is the correct HTML reading, but it
// means a bare ampersand in prose is now interpreted, so the ordinary prose case is pinned here:
// «AT&T», «R&D» and «Р&Б» must survive untouched, because the letter after & starts no entity name.
func TestExtractXHTMLBareAmpersandInProseSurvives(t *testing.T) {
	for _, s := range []string{"AT&T", "R&D", "Тим &  Ко", "1 & 2"} {
		body, _, err := extractXHTML([]byte(`<html><body><p>` + s + `</p></body></html>`))
		if err != nil {
			t.Fatal(err)
		}
		if !strings.Contains(body, s) {
			t.Errorf("bare ampersand prose %q was altered: %q", s, body)
		}
	}
}

// A namespace-prefixed block tag must still break paragraphs: the xml decoder matched on the LOCAL
// name (<epub:p> → "p"), and the tokenizer reports the prefixed name, so the prefix is dropped
// before lookup. Without that an epub3 chapter using prefixed markup collapses into one paragraph.
func TestExtractXHTMLStripsNamespacePrefix(t *testing.T) {
	body, _, err := extractXHTML([]byte(`<html xmlns:e="u"><body><e:p>Раз.</e:p><e:p>Два.</e:p></body></html>`))
	if err != nil {
		t.Fatal(err)
	}
	if paras := splitParagraphs(text.NormalizeSource(body)); len(paras) != 2 {
		t.Fatalf("prefixed block tags must still split paragraphs, got %d: %#v", len(paras), paras)
	}
}

// CDATA is literal text in XHTML, and the previous xml-based reader delivered it as such. The HTML
// tokenizer has no CDATA — left alone it drops part of the section and leaks «]]>» into the prose —
// so the section is unwrapped first. A chapter that imported cleanly before must not silently
// re-chunk (which on the next run would re-pay for it).
func TestExtractXHTMLUnwrapsCDATA(t *testing.T) {
	for _, c := range []struct{ name, doc, want string }{
		{"literal text", `<html><body><p>До.</p><![CDATA[ сырой <текст> & амперсанд ]]><p>После.</p></body></html>`,
			"сырой <текст> & амперсанд"},
		{"markup kept literal", `<html><body><![CDATA[<p>не разметка</p>]]><p>Проза.</p></body></html>`,
			"<p>не разметка</p>"},
	} {
		t.Run(c.name, func(t *testing.T) {
			body, _, err := extractXHTML([]byte(c.doc))
			if err != nil {
				t.Fatal(err)
			}
			if !strings.Contains(body, c.want) {
				t.Fatalf("CDATA content lost or mangled: got %q, want it to contain %q", body, c.want)
			}
			if strings.Contains(body, "]]>") || strings.Contains(body, "CDATA") {
				t.Fatalf("CDATA delimiters leaked into prose: %q", body)
			}
		})
	}
	// The real-world spelling: CDATA guards inside a <style> block stay dropped with the subtree.
	body, _, err := extractXHTML([]byte("<html><head><style>/*<![CDATA[*/\n.x{color:red}\n/*]]>*/</style></head><body><p>Проза.</p></body></html>"))
	if err != nil {
		t.Fatal(err)
	}
	if strings.TrimSpace(body) != "Проза." {
		t.Fatalf("style CDATA leaked: %q", body)
	}
}

// A declared non-UTF-8 charset stays a LOUD error (epub v1 = UTF-8) — the guard the xml decoder's
// CharsetReader used to provide. Silent mojibake in ch.Text is the failure this prevents.
func TestExtractXHTMLRejectsNonUTF8Charset(t *testing.T) {
	bad := []byte(`<?xml version="1.0" encoding="gb18030"?><html><body><p>текст</p></body></html>`)
	if _, _, err := extractXHTML(bad); err == nil {
		t.Fatal("a declared non-UTF-8 xhtml charset must fail loud")
	} else if !strings.Contains(err.Error(), "gb18030") {
		t.Fatalf("the error must name the charset, got: %v", err)
	}
	for _, ok := range []string{`<?xml version="1.0" encoding="utf-8"?>`, `<?xml version="1.0"?>`, ``} {
		if _, _, err := extractXHTML([]byte(ok + `<html><body><p>текст</p></body></html>`)); err != nil {
			t.Fatalf("declaration %q must be accepted: %v", ok, err)
		}
	}
}

func TestIngestEPUBPercentEncodedHref(t *testing.T) {
	// Real epubs percent-encode non-ASCII (and spaced) filenames in the manifest while the zip
	// entry carries the decoded name. Without resolveHref's url.PathUnescape the entry is missed
	// and ingest fails loud on a chapter that is actually present.
	chapters := []chunktest.Chapter{
		{ID: "c1", Href: "%E7%AC%AC%E4%B8%80%E7%AB%A0.xhtml", EntryName: "第一章.xhtml", Body: `<p>第一章。</p>`},
		{ID: "c2", Href: "ch%202.xhtml", EntryName: "ch 2.xhtml", Body: `<p>第二章。</p>`},
	}
	doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "c2"}))
	if err != nil {
		t.Fatalf("percent-encoded hrefs must resolve to their decoded zip entries: %v", err)
	}
	if len(doc.Chapters) != 2 {
		t.Fatalf("want 2 chapters, got %d: %#v", len(doc.Chapters), doc.Chapters)
	}
	for i, want := range []string{"第一章。", "第二章。"} {
		if strings.TrimSpace(doc.Chapters[i]) != want {
			t.Fatalf("chapter %d = %q, want %q", i+1, doc.Chapters[i], want)
		}
	}
}

func TestIngestEPUBHrefFragmentIgnored(t *testing.T) {
	// A spine href may point at an anchor inside a document (chapter.xhtml#part2). The fragment
	// addresses a position, not a file: it must be dropped before the zip lookup, or the chapter
	// is reported missing.
	chapters := []chunktest.Chapter{
		{ID: "c1", Href: "ch1.xhtml#part2", EntryName: "ch1.xhtml", Body: `<p>第一章。</p>`},
	}
	doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1"}))
	if err != nil {
		t.Fatalf("an href #fragment must be dropped before the zip lookup: %v", err)
	}
	if len(doc.Chapters) != 1 || strings.TrimSpace(doc.Chapters[0]) != "第一章。" {
		t.Fatalf("chapters = %#v", doc.Chapters)
	}
}

func TestEPUBFixtureMimetypeIsOCFConformant(t *testing.T) {
	// OCF: "mimetype" must be the FIRST entry and STORED (uncompressed). All three stand epubs
	// ship it that way; the fixture must too, or it is not the file shape the reader will meet.
	p := chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: `<p>x</p>`}}, []string{"c1"})
	zr, err := zip.OpenReader(p)
	if err != nil {
		t.Fatal(err)
	}
	defer zr.Close()
	if len(zr.File) == 0 || zr.File[0].Name != "mimetype" {
		t.Fatalf("mimetype must be the first zip entry, got %q", zr.File[0].Name)
	}
	if zr.File[0].Method != zip.Store {
		t.Fatalf("mimetype must be STORED (method %d), got method %d", zip.Store, zr.File[0].Method)
	}
	rc, err := zr.File[0].Open()
	if err != nil {
		t.Fatal(err)
	}
	defer rc.Close()
	b, _ := io.ReadAll(rc)
	if string(b) != "application/epub+zip" {
		t.Fatalf("mimetype content = %q", b)
	}
}

func TestIngestEPUBDanglingIdrefFailsLoud(t *testing.T) {
	// A spine idref with no manifest item, AMONG valid chapters, must fail loud — not
	// silently drop that chapter (which would also shift every later since_ch).
	chapters := []chunktest.Chapter{
		{ID: "c1", Href: "ch1.xhtml", Body: `<p>第一章。</p>`},
		{ID: "c2", Href: "ch2.xhtml", Body: `<p>第二章。</p>`},
		{ID: "c3", Href: "ch3.xhtml", Body: `<p>第三章。</p>`},
	}
	_, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"c1", "ghost", "c3"}))
	if err == nil {
		t.Fatal("a dangling spine idref among valid chapters must fail loud, not silently lose a chapter")
	}
	if !strings.Contains(err.Error(), "ghost") {
		t.Fatalf("error should name the dangling idref, got: %v", err)
	}
}

func TestIngestEPUBBrInsideRubyDoesNotLeak(t *testing.T) {
	// A <br> inside <ruby> must not inject a newline between base glyphs (it would
	// desync base from body and pollute ch.Text — external-review #5).
	body := `<p><ruby>漢<br/>字<rt>かんじ</rt></ruby>を読む。</p>`
	doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"}))
	if err != nil {
		t.Fatal(err)
	}
	if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "漢字" || doc.Ruby[0].Reading != "かんじ" {
		t.Fatalf("ruby base/reading = %#v", doc.Ruby)
	}
	if !strings.Contains(doc.Chapters[0], "漢字を読む") || strings.Contains(doc.Chapters[0], "漢\n字") {
		t.Fatalf("<br> leaked a newline between base glyphs: %q", doc.Chapters[0])
	}
}

func TestIngestEPUBRubyBaseWhitespaceCollapsed(t *testing.T) {
	// Pretty-printed jukugo ruby: whitespace BETWEEN <rb> base segments must not
	// enter the captured base (mis-keys the glossary) nor the body (pollutes ch.Text).
	body := "<p><ruby>\n  <rb>旧</rb>\n  <rb>字</rb>\n  <rb>体</rb>\n  <rt>きゅう</rt><rt>じ</rt><rt>たい</rt>\n</ruby>の話。</p>"
	doc, err := ingest(chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: body}}, []string{"c1"}))
	if err != nil {
		t.Fatal(err)
	}
	if len(doc.Ruby) != 1 || doc.Ruby[0].Base != "旧字体" || doc.Ruby[0].Reading != "きゅうじたい" {
		t.Fatalf("ruby base/reading not clean: %#v", doc.Ruby)
	}
	if !strings.Contains(doc.Chapters[0], "旧字体の話") {
		t.Fatalf("clean base missing from Body: %q", doc.Chapters[0])
	}
	for _, bad := range []string{"旧 字", "旧\n"} {
		if strings.Contains(doc.Chapters[0], bad) {
			t.Fatalf("ruby-internal whitespace leaked into Body: %q", doc.Chapters[0])
		}
	}
}

func TestIngestEPUBSkipsNonXHTML(t *testing.T) {
	chapters := []chunktest.Chapter{
		{ID: "img", Href: "cover.jpg", MType: "image/jpeg", Body: "\xff\xd8not-really-jpeg"},
		{ID: "c1", Href: "ch1.xhtml", Body: `<p>本文。</p>`},
	}
	doc, err := ingest(chunktest.BuildEPUB(t, chapters, []string{"img", "c1"}))
	if err != nil {
		t.Fatal(err)
	}
	if len(doc.Chapters) != 1 || strings.TrimSpace(doc.Chapters[0]) != "本文。" {
		t.Fatalf("non-xhtml spine item must be skipped: %#v", doc.Chapters)
	}
}

func TestIngestEPUBBrokenFailsLoud(t *testing.T) {
	t.Run("not a zip", func(t *testing.T) {
		p := filepath.Join(t.TempDir(), "bad.epub")
		os.WriteFile(p, []byte("this is not a zip"), 0o644)
		if _, err := ingest(p); err == nil {
			t.Fatal("a non-zip .epub must error, not panic")
		}
	})
	t.Run("missing container", func(t *testing.T) {
		p := filepath.Join(t.TempDir(), "noc.epub")
		f, _ := os.Create(p)
		zw := zip.NewWriter(f)
		w, _ := zw.Create("OEBPS/content.opf")
		w.Write([]byte("<package/>"))
		zw.Close()
		f.Close()
		if _, err := ingest(p); err == nil {
			t.Fatal("missing container.xml must error")
		}
	})
	t.Run("empty spine", func(t *testing.T) {
		p := chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: `<p>x</p>`}}, nil)
		if _, err := ingest(p); err == nil {
			t.Fatal("an empty spine must error")
		}
	})
	t.Run("dangling spine idref", func(t *testing.T) {
		// spine references a missing manifest id → resolves to zero chapters → error.
		p := chunktest.BuildEPUB(t, []chunktest.Chapter{{ID: "c1", Href: "ch1.xhtml", Body: `<p>x</p>`}}, []string{"ghost"})
		if _, err := ingest(p); err == nil {
			t.Fatal("a spine resolving to no readable chapters must error")
		}
	})
}

// TestTitleRawIsTakenWhereTheCutAllowsIt holds both halves of one rule, and the halves pull in opposite
// directions — which is why narrowing it to "headers won" alone was wrong and cost a real title.
//
//   - a cut the HEADERS won may hide chapter one's header deep inside it (the frozen preamble rule folds
//     everything before the first header into chapter one), so the whole chapter is scanned;
//   - any other cut was drawn by page breaks, and a header-shaped line buried in the prose is a lone
//     occurrence that never met the ≥2 floor. Only the OPENING line counts there — which still gives a
//     one-chapter book its own real title instead of throwing it away.
func TestTitleRawIsTakenWhereTheCutAllowsIt(t *testing.T) {
	body := strings.Repeat("蛊", 200)
	for _, tc := range []struct {
		name, source string
		wantStruct   string
		wantTitles   []string
	}{
		{
			name:       "one chapter keeps its own opening header",
			source:     "第一节 начало\n" + body,
			wantStruct: StructureNone,
			wantTitles: []string{"第一节 начало"},
		},
		{
			name:       "a header buried in a page-break cut is not a title",
			source:     "Проза первой страницы.\n\n第九节 это предложение, а не шапка\n\n" + body + chapterSep + "Вторая страница.\n" + body,
			wantStruct: StructureDelimited,
			wantTitles: []string{"", ""},
		},
		{
			name:       "the preamble rule still hands chapter one its swallowed header",
			source:     "Предисловие без шапки.\n" + "第一节 первая\n" + body + "\n第二节 вторая\n" + body,
			wantStruct: StructureDetected,
			wantTitles: []string{"第一节 первая", "第二节 вторая"},
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			path := filepath.Join(t.TempDir(), "src.txt")
			if err := os.WriteFile(path, []byte(tc.source), 0o644); err != nil {
				t.Fatal(err)
			}
			doc, err := ingest(path)
			if err != nil {
				t.Fatal(err)
			}
			if doc.Structure != tc.wantStruct {
				t.Fatalf("structure = %q, want %q (chapters %d)", doc.Structure, tc.wantStruct, len(doc.Chapters))
			}
			if len(doc.Titles) != len(tc.wantTitles) {
				t.Fatalf("titles = %d, want %d", len(doc.Titles), len(tc.wantTitles))
			}
			for i, w := range tc.wantTitles {
				if doc.Titles[i] != w {
					t.Fatalf("title %d = %q, want %q", i+1, doc.Titles[i], w)
				}
			}
		})
	}
}