textmachine/backend/internal/bookfile/bookfile_test.go

package bookfile

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"io"
	"regexp"
	"strings"
	"testing"
	"time"
)

// bookfile_test.go pins the writer's two load-bearing properties — determinism and a container the
// engine's own reader round-trips — and the EPUB 3 shape epubcheck was run against (the stand proof).

func sampleBook() *Book {
	return &Book{
		Identifier: "urn:textmachine:book:test",
		Title:      "Тест & <проба>",
		Language:   "ru",
		Modified:   time.Date(2026, 8, 24, 1, 15, 0, 0, time.UTC),
		Chapters: []Chapter{
			{Title: "Глава 1", Paragraphs: []string{"Первый абзац.", "Второй — с «кавычками» & <тегом>."}},
			{Title: "2", Paragraphs: []string{"Одна строка."}},
		},
	}
}

func epubBytes(t *testing.T, b *Book) []byte {
	t.Helper()
	var buf bytes.Buffer
	if err := WriteEPUB(&buf, b); err != nil {
		t.Fatal(err)
	}
	return buf.Bytes()
}

func zipEntries(t *testing.T, data []byte) (*zip.Reader, map[string]string) {
	t.Helper()
	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
	if err != nil {
		t.Fatal(err)
	}
	out := map[string]string{}
	for _, f := range zr.File {
		rc, err := f.Open()
		if err != nil {
			t.Fatal(err)
		}
		b, _ := io.ReadAll(rc)
		rc.Close()
		out[f.Name] = string(b)
	}
	return zr, out
}

func TestContainerIsOCFConformant(t *testing.T) {
	data := epubBytes(t, sampleBook())
	zr, entries := zipEntries(t, data)
	// The mimetype's LOCAL header carries its CRC and sizes itself: general-purpose flag bit 3 (a data
	// descriptor after the entry) is clear, and the extra-field length is zero — the plain form a reader
	// that identifies the file by its first bytes expects. Bytes 6–7 are the flags, 28–29 the extra length.
	if flags := uint16(data[6]) | uint16(data[7])<<8; flags&0x8 != 0 {
		t.Errorf("mimetype local header has the data-descriptor flag set (flags=%#x)", flags)
	}
	if extra := uint16(data[28]) | uint16(data[29])<<8; extra != 0 {
		t.Errorf("mimetype local header carries an extra field of %d bytes", extra)
	}
	if string(data[30:38]) != "mimetype" || string(data[38:58]) != "application/epub+zip" {
		t.Errorf("the mimetype entry is not at the fixed OCF offsets: %q", data[30:58])
	}
	if zr.File[0].Name != "mimetype" || zr.File[0].Method != zip.Store {
		t.Fatalf("mimetype must be the first, STORED entry; got %q method %d", zr.File[0].Name, zr.File[0].Method)
	}
	if entries["mimetype"] != "application/epub+zip" {
		t.Fatalf("mimetype = %q", entries["mimetype"])
	}
	if !strings.Contains(entries["META-INF/container.xml"], `full-path="OEBPS/content.opf"`) {
		t.Fatalf("container.xml must point at the OPF:\n%s", entries["META-INF/container.xml"])
	}
	for _, f := range zr.File {
		if !f.Modified.IsZero() && f.Modified.Unix() != -2208988800 { // zero MS-DOS time decodes to 1980-00-00 → 1979/1980 epochs vary; a real clock reading is what is forbidden
			if f.Modified.Year() > 1981 {
				t.Errorf("entry %s carries a clock timestamp %v — the file would differ between builds", f.Name, f.Modified)
			}
		}
	}
}

type opfDoc struct {
	UniqueID string `xml:"unique-identifier,attr"`
	Metadata struct {
		Identifier struct {
			ID    string `xml:"id,attr"`
			Value string `xml:",chardata"`
		} `xml:"identifier"`
		Title       string `xml:"title"`
		Language    string `xml:"language"`
		Description string `xml:"description"`
		Meta        []struct {
			Property string `xml:"property,attr"`
			Value    string `xml:",chardata"`
		} `xml:"meta"`
	} `xml:"metadata"`
	Manifest []struct {
		ID         string `xml:"id,attr"`
		Href       string `xml:"href,attr"`
		MediaType  string `xml:"media-type,attr"`
		Properties string `xml:"properties,attr"`
	} `xml:"manifest>item"`
	Spine []struct {
		IDRef string `xml:"idref,attr"`
	} `xml:"spine>itemref"`
}

func parseOPF(t *testing.T, entries map[string]string) opfDoc {
	t.Helper()
	var doc opfDoc
	if err := xml.Unmarshal([]byte(entries["OEBPS/content.opf"]), &doc); err != nil {
		t.Fatalf("OPF does not parse: %v\n%s", err, entries["OEBPS/content.opf"])
	}
	return doc
}

func TestEPUBCarriesTheFiveRequiredThingsAndKeepsNavOutOfTheSpine(t *testing.T) {
	b := sampleBook()
	b.Description = "⚠ 1/2"
	_, entries := zipEntries(t, epubBytes(t, b))
	doc := parseOPF(t, entries)
	if doc.Metadata.Identifier.Value != b.Identifier || doc.Metadata.Identifier.ID != doc.UniqueID {
		t.Errorf("dc:identifier %+v must be the one unique-identifier=%q names", doc.Metadata.Identifier, doc.UniqueID)
	}
	if doc.Metadata.Title != b.Title {
		t.Errorf("dc:title = %q, want %q (escaped in the file, intact when parsed)", doc.Metadata.Title, b.Title)
	}
	padded := sampleBook()
	padded.Title = "  T  "
	if _, e := zipEntries(t, epubBytes(t, padded)); !strings.Contains(e["OEBPS/content.opf"], "<dc:title>T</dc:title>") {
		t.Errorf("dc:title must be trimmed like the text file's title line:\n%s", e["OEBPS/content.opf"])
	}
	if doc.Metadata.Language != "ru" {
		t.Errorf("dc:language = %q", doc.Metadata.Language)
	}
	if doc.Metadata.Description != b.Description {
		t.Errorf("dc:description = %q, want %q", doc.Metadata.Description, b.Description)
	}
	modified := ""
	for _, m := range doc.Metadata.Meta {
		if m.Property == "dcterms:modified" {
			modified = m.Value
		}
	}
	if modified != "2026-08-24T01:15:00Z" {
		t.Errorf("dcterms:modified = %q, want the CCYY-MM-DDThh:mm:ssZ render of Book.Modified", modified)
	}
	navID := ""
	for _, it := range doc.Manifest {
		if it.Properties == "nav" {
			navID = it.ID
			if _, ok := entries["OEBPS/"+it.Href]; !ok {
				t.Errorf("the nav item points at %q, which is not in the zip", it.Href)
			}
		}
	}
	if navID == "" {
		t.Fatal("no manifest item carries properties=\"nav\" — EPUB 3 requires a navigation document")
	}
	if len(doc.Spine) != len(b.Chapters) {
		t.Fatalf("spine has %d itemrefs, want one per chapter (%d) and nothing else", len(doc.Spine), len(b.Chapters))
	}
	for _, ref := range doc.Spine {
		if ref.IDRef == navID {
			t.Fatal("the nav document is in the spine — the engine's reader would read it back as a chapter")
		}
	}
	nav := entries["OEBPS/nav.xhtml"]
	for _, ch := range b.Chapters {
		if !strings.Contains(nav, ">"+xmlText(ch.Title)+"</a>") {
			t.Errorf("nav lacks an entry titled %q:\n%s", ch.Title, nav)
		}
	}
}

// blocksOf pulls the text of every <h1>/<p> of a chapter document, in order — what the engine's reader
// turns into paragraphs.
var blockRE = regexp.MustCompile(`(?s)<(h1|p)>(.*?)</(?:h1|p)>`)

func blocksOf(doc string) []string {
	var out []string
	for _, m := range blockRE.FindAllStringSubmatch(doc, -1) {
		out = append(out, m[2])
	}
	return out
}

func TestChapterDocumentsCarryExactlyBlocks(t *testing.T) {
	b := sampleBook()
	b.Notice = []string{"⚠ 1/2", "⚠ +3"}
	_, entries := zipEntries(t, epubBytes(t, b))
	for i := range b.Chapters {
		got := blocksOf(entries["OEBPS/"+chapterEntry(i)])
		want := b.Blocks(i)
		if len(got) != len(want) {
			t.Fatalf("chapter %d: %d blocks in the document, Blocks() says %d\n%v\n%v", i+1, len(got), len(want), got, want)
		}
		for j := range want {
			if got[j] != xmlText(want[j]) {
				t.Errorf("chapter %d block %d = %q, want %q", i+1, j, got[j], xmlText(want[j]))
			}
		}
	}
	// The notice is in the FIRST document only, ahead of its title.
	if first := blocksOf(entries["OEBPS/ch1.xhtml"]); first[0] != "⚠ 1/2" || first[2] != xmlText("Глава 1") {
		t.Errorf("first document must open with the notice, then the title: %v", first)
	}
	if second := blocksOf(entries["OEBPS/ch2.xhtml"]); second[0] != "2" {
		t.Errorf("second document must open with its title, no notice: %v", second)
	}
}

func TestEPUBAndTXTAreDeterministic(t *testing.T) {
	b := sampleBook()
	first, second := epubBytes(t, b), epubBytes(t, b)
	if !bytes.Equal(first, second) {
		t.Fatal("two EPUB builds of one Book differ")
	}
	var t1, t2 bytes.Buffer
	if err := WriteTXT(&t1, b); err != nil {
		t.Fatal(err)
	}
	if err := WriteTXT(&t2, b); err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(t1.Bytes(), t2.Bytes()) {
		t.Fatal("two TXT builds of one Book differ")
	}
}

func TestTextIsEscapedAndControlCharactersAreDropped(t *testing.T) {
	b := sampleBook()
	b.Chapters[0].Paragraphs = []string{"a < b & c > d \"q\"", "bad\x00\x0bchars\x7f kept"}
	if got := CleanText("bad\x00\x0bchars\x7f kept\xff"); got != "badchars\x7f kept\uFFFD" {
		t.Errorf("CleanText = %q", got)
	}
	_, entries := zipEntries(t, epubBytes(t, b))
	doc := entries["OEBPS/ch1.xhtml"]
	for _, want := range []string{"a &lt; b &amp; c &gt; d &quot;q&quot;", "badchars\x7f kept"} {
		if !strings.Contains(doc, want) {
			t.Errorf("document lacks %q:\n%s", want, doc)
		}
	}
	if strings.ContainsAny(doc, "\x00\x0b") {
		t.Error("an XML-illegal control character reached the document")
	}
	if !strings.Contains(entries["OEBPS/content.opf"], "<dc:title>Тест &amp; &lt;проба&gt;</dc:title>") {
		t.Errorf("title not escaped:\n%s", entries["OEBPS/content.opf"])
	}
}

func TestWritersRefuseAnInvalidBook(t *testing.T) {
	cases := map[string]func(*Book){
		"no chapters": func(b *Book) { b.Chapters = nil },
		"empty title": func(b *Book) { b.Chapters[1].Title = "  " },
		// Judged after CleanText: a title of control characters only would render as an empty <h1> and
		// an empty nav link — the invalid EPUB validate exists to refuse, not something to write.
		"control-only chapter title": func(b *Book) { b.Chapters[1].Title = "\x01\x02 \x1f" },
		"control-only book title":    func(b *Book) { b.Title = "\x03\x0b" },
		"zero modified":              func(b *Book) { b.Modified = time.Time{} },
		"no language":                func(b *Book) { b.Language = "" },
		"no identifier":              func(b *Book) { b.Identifier = "" },
		"empty book title":           func(b *Book) { b.Title = "" },
	}
	for name, mutate := range cases {
		b := sampleBook()
		mutate(b)
		if err := WriteEPUB(io.Discard, b); err == nil {
			t.Errorf("%s: WriteEPUB must refuse", name)
		}
		if err := WriteTXT(io.Discard, b); err == nil {
			t.Errorf("%s: WriteTXT must refuse", name)
		}
	}
}

func TestTXTShape(t *testing.T) {
	b := sampleBook()
	b.Notice = []string{"⚠ 1/2"}
	var buf bytes.Buffer
	if err := WriteTXT(&buf, b); err != nil {
		t.Fatal(err)
	}
	want := "Тест & <проба>\n\n⚠ 1/2\n\n\nГлава 1\n\nПервый абзац.\n\nВторой — с «кавычками» & <тегом>.\n\n\n2\n\nОдна строка.\n"
	if buf.String() != want {
		t.Fatalf("txt =\n%q\nwant\n%q", buf.String(), want)
	}
	for _, banned := range []string{"=== CHAPTER", "TEXT MISSING", "pending", "flagged"} {
		if strings.Contains(buf.String(), banned) {
			t.Errorf("operator vocabulary %q in the reader's text", banned)
		}
	}
}

// TestTitlesAreCleanedTheSameInBothFormats pins format parity on the strings the assembler does NOT
// pre-clean — the book title, the chapter titles, the notice: the EPUB drops a control character through
// xmlText, and the text file must drop it the same way (acceptance finding: TXT wrote titles raw).
func TestTitlesAreCleanedTheSameInBothFormats(t *testing.T) {
	b := sampleBook()
	b.Title = "Заглавие\x01книги"
	b.Notice = []string{"⚠\x03 1/2"}
	b.Chapters[0].Title = "Гла\x02ва 1"
	b.Chapters[0].Paragraphs = []string{"Абзац\x1fс контролем."}
	_, entries := zipEntries(t, epubBytes(t, b))
	doc := parseOPF(t, entries)
	if doc.Metadata.Title != "Заглавиекниги" {
		t.Errorf("EPUB dc:title = %q", doc.Metadata.Title)
	}
	if blocks := blocksOf(entries["OEBPS/ch1.xhtml"]); len(blocks) < 3 || blocks[0] != "⚠ 1/2" || blocks[1] != "Глава 1" || blocks[2] != "Абзацс контролем." {
		t.Errorf("EPUB blocks = %q", blocks)
	}
	var txt bytes.Buffer
	if err := WriteTXT(&txt, b); err != nil {
		t.Fatal(err)
	}
	want := "Заглавиекниги\n\n⚠ 1/2\n\n\nГлава 1\n\nАбзацс контролем.\n\n\n2\n\nОдна строка.\n"
	if txt.String() != want {
		t.Errorf("TXT =\n%q\nwant\n%q", txt.String(), want)
	}
	for name, out := range map[string]string{"epub opf": entries["OEBPS/content.opf"], "epub nav": entries["OEBPS/nav.xhtml"], "epub ch1": entries["OEBPS/ch1.xhtml"], "txt": txt.String()} {
		if strings.ContainsAny(out, "\x01\x02\x03\x1f") {
			t.Errorf("%s carries a control character the other format drops", name)
		}
	}
}

func TestOneChapterBook(t *testing.T) {
	b := &Book{Identifier: "urn:x", Title: "T", Language: "en", Modified: time.Unix(0, 0).UTC(),
		Chapters: []Chapter{{Title: "1"}}} // a chapter with a title and no paragraphs is a legal, empty chapter
	_, entries := zipEntries(t, epubBytes(t, b))
	doc := parseOPF(t, entries)
	if len(doc.Spine) != 1 || len(blocksOf(entries["OEBPS/ch1.xhtml"])) != 1 {
		t.Fatalf("one-chapter book: spine %d, blocks %v", len(doc.Spine), blocksOf(entries["OEBPS/ch1.xhtml"]))
	}
}

// TestALostMarkerCannotInvertAParagraph is the reader-facing half of the emphasis contract, and the one
// that cost the most to find: with one glyph on both sides, pairing left to right without asking WHICH
// marker may open turns a single marker lost by the model into emphasis on the words between it and the
// next one. The file parses, the run reports success, and the book emphasises text nobody marked.
//
// ⛔ THE FIRST VERSION OF THIS TEST HELD CONSTANT THE EXACT FORM THAT BREAKS — in every case the opening
// marker stood after a space or at the start of the line — and it went green over a rule that lost 68 of
// the production book's 4842 runs. The cases below therefore LEAD with the shapes measured in that book:
// emphasis that lands on a full stop, on a comma, or inside a word, so the opener follows a letter.
func TestALostMarkerCannotInvertAParagraph(t *testing.T) {
	b := &Book{ItalicOpen: "*", ItalicClose: "*"}
	render := func(s string) string {
		var sb strings.Builder
		b.emphasisSpans(s, func(text string, italic bool) {
			if text == "" {
				return
			}
			if italic {
				sb.WriteString("<i>" + text + "</i>")
				return
			}
			sb.WriteString(text)
		})
		return sb.String()
	}
	for _, tc := range []struct{ name, in, want string }{
		// MEASURED IN THE PRODUCTION BOOK: 68 of its 4842 runs open right after a letter or a mark.
		{"emphasis on a full stop after a word", "HIM*.*", "HIM<i>.</i>"},
		{"emphasis inside a word", "Fucks*sakes*, Baptiste", "Fucks<i>sakes</i>, Baptiste"},
		{"emphasis on a comma", "heat*,* that blessed", "heat<i>,</i> that blessed"},
		{"several runs in one line, one of them mid-word",
			"*Word* all*.*’ and then *more* text.", "<i>Word</i> all<i>.</i>’ and then <i>more</i> text."},
		// AND THE DEFECT THE RULE EXISTS FOR: a marker the model dropped must not move the emphasis.
		{"a marker the model lost does not move the emphasis",
			"Он поставил сноску* и сказал *важно* потом.", "Он поставил сноску* и сказал <i>важно</i> потом."},
		{"a scene break is the author's, not a marker pair", "* * *", "* * *"},
		{"the model's doubled markers stay literal", "**слово**", "**слово**"},
		{"and ordinary emphasis still works", "Она *почти* ушла, но *вернулась*.",
			"Она <i>почти</i> ушла, но <i>вернулась</i>."},
	} {
		t.Run(tc.name, func(t *testing.T) {
			if got := render(tc.in); got != tc.want {
				t.Errorf("got  %q\nwant %q", got, tc.want)
			}
		})
	}
	// ⚠ THE LIMIT, ASSERTED RATHER THAN LEFT TO BE DISCOVERED: when BOTH sides of the author's own glyph
	// are text, nothing here can tell it from ours. The answer is the ingest's `source_marker_glyphs`
	// count (0 on the production book, 12 on the other English one), not a cleverer predicate — the
	// cleverer predicate is what lost the 68 runs above.
	if got := render("5*5=25 и 3*3=9"); got != "5<i>5=25 и 3</i>3=9" {
		t.Errorf("the known limit changed shape (%q) — if this is now handled, the comment above is stale", got)
	}
	// The premise the rule rests on: with DIFFERENT glyphs there is no ambiguity to resolve, and the
	// pairing must stay exactly as permissive as before.
	d := &Book{ItalicOpen: "⟦i⟧", ItalicClose: "⟦/i⟧"}
	var sb strings.Builder
	d.emphasisSpans("слово⟦i⟧выделено⟦/i⟧", func(text string, italic bool) {
		if italic {
			sb.WriteString("<i>" + text + "</i>")
			return
		}
		sb.WriteString(text)
	})
	if got := sb.String(); got != "слово<i>выделено</i>" {
		t.Errorf("distinct glyphs need no flanking rule, got %q", got)
	}
}