package books import ( "archive/zip" "bytes" "io" "strings" "testing" "unicode" "unicode/utf8" ) // TestTheIntakeCounterCountsTheWriteStreamAndNotCharacters is documentation the battery enforces. // // ⚠ WHY IT EXISTS. The number this counter produces leaves the platform as `character_count` and is // labelled «Знаков» on the product screen (unified backlog row 282). For a UTF-8 text source that is // exactly true; for an EPUB — which this intake accepts, because the engine dispatches an EPUB reader // by the extension — it is the non-continuation-byte count of a ZIP ARCHIVE and is not a character // count in any sense. Two sessions have now read this counter as "characters, roughly", and the // second one wrote it into a document. So the fact is pinned here rather than described anywhere: // the next reader meets it in a test that fails if it stops being true. // // The cure is NOT in this package and is deliberately not attempted here — see `counter`. func TestTheIntakeCounterCountsTheWriteStreamAndNotCharacters(t *testing.T) { count := func(body []byte) int64 { var sink bytes.Buffer c := &counter{w: &sink} if _, err := io.Copy(c, bytes.NewReader(body)); err != nil { t.Fatal(err) } if sink.Len() != len(body) { t.Fatalf("the counter did not write through: %d of %d bytes", sink.Len(), len(body)) } return c.runes } // (1) A UTF-8 text source: the number IS the character count, exactly — including across the // chunk boundaries io.Copy hands the writer, which is why the count is stateless. text := strings.Repeat("蛊真人 — глава первая.\n", 500) if got, want := count([]byte(text)), int64(utf8.RuneCountInString(text)); got != want { t.Errorf("a UTF-8 text source counted %d, want %d characters", got, want) } // (2) An EPUB: a ZIP archive, whose stream has nothing to do with the number of characters in // the book inside it. The assertion is deliberately the INEQUALITY — this test does not bless a // particular wrong number, it records that the number is not the one the field's name promises. var epub bytes.Buffer zw := zip.NewWriter(&epub) f, err := zw.Create("OEBPS/ch1.xhtml") if err != nil { t.Fatal(err) } if _, err := f.Write([]byte("

" + text + "

")); err != nil { t.Fatal(err) } if err := zw.Close(); err != nil { t.Fatal(err) } inside := int64(utf8.RuneCountInString(text)) if got := count(epub.Bytes()); got == inside { t.Errorf("an EPUB counted %d, which happens to equal the characters inside it — the pin below is meaningless if that is now true by construction", got) } else if got > inside { t.Errorf("an EPUB counted %d, MORE than the %d characters it contains: the number is a property of the container", got, inside) } // (3) UTF-16, which the engine accepts and decodes itself. Every ASCII character occupies two // bytes and the high byte is a zero, so the count is roughly DOUBLE the characters. utf16 := make([]byte, 0, 40) for _, r := range "hello world" { utf16 = append(utf16, byte(r), 0x00) } if got, want := count(utf16), int64(len("hello world")); got == want { t.Errorf("a UTF-16 source counted %d, exactly its character count — that is not what this counter does and the note on `counter` would be wrong", got) } } // ⛔ BOTH WRITERS OF `books.title` MUST AGREE ABOUT WHAT MAY BE IN IT. This file's own `MaxTitle` // exists because "two copies of the number is how the intake and the patch come to disagree" — and // the intake then disagreed with the rename door about something worse than a number: a title the // user TYPED went through length bounding alone, so U+0000 reached Postgres, which cannot hold it in // a `text` column, and U+2028 landed in a book's name. // // Found by acceptance, not by this session, in the very place the session had argued the border must // be single — the class "fixed the reader, left the writer" applied to two writers of one column. // // Mutation caught: dropping the withoutControls call from Intake.title. func TestBothTitleWritersAgreeAboutControlCharacters(t *testing.T) { for name, bad := range map[string]string{ "NUL": "\u0000", "newline": "\u000a", "escape": "\u001b", "line separator": "\u2028", "paragraph separator": "\u2029", } { got := (Intake{Title: "be" + bad + "fore", Filename: "fallback.txt"}).title() for _, r := range got { if unicode.IsControl(r) || r == 0x2028 || r == 0x2029 { t.Errorf("%s: the intake stored %q, carrying U+%04X — the rename door refuses this", name, got, r) } } if got != "before" { t.Errorf("%s: got %q, want the title with only the offending rune removed", name, got) } } // A title that is NOTHING BUT control characters leaves nothing legible, so the book is named from // its file — the same answer as naming nothing at all. if got := (Intake{Title: "\u0000\u000a", Filename: "Мастер Гу.txt"}).title(); got != "Мастер Гу" { t.Errorf("a title of only control characters gave %q, want the name derived from the file", got) } // ⚠ And the half that must NOT be cleaned: bidi marks and the zero-width joiners are ordinary // content in Hebrew, Arabic, Devanagari and Persian. Dropping "everything invisible" would mangle // a legitimate title in a pair this repository does not carry yet. for name, title := range map[string]string{ "right-to-left mark": "\u200fdavar", "zero-width joiner": "ka\u200dsha", "soft hyphen": "a\u00adb", } { if got := (Intake{Title: title, Filename: "f.txt"}).title(); got != title { t.Errorf("%s: a legitimate title was altered: %q -> %q", name, title, got) } } }