124 lines
5.6 KiB
Go
124 lines
5.6 KiB
Go
package books
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// TestTheIntakeCounterCountsTheWriteStreamAndNotCharacters is documentation the battery enforces.
|
|
//
|
|
// ⚠ WHY IT EXISTS. The number this counter produces leaves the platform as `character_count` and is
|
|
// labelled «Знаков» on the product screen (unified backlog row 282). For a UTF-8 text source that is
|
|
// exactly true; for an EPUB — which this intake accepts, because the engine dispatches an EPUB reader
|
|
// by the extension — it is the non-continuation-byte count of a ZIP ARCHIVE and is not a character
|
|
// count in any sense. Two sessions have now read this counter as "characters, roughly", and the
|
|
// second one wrote it into a document. So the fact is pinned here rather than described anywhere:
|
|
// the next reader meets it in a test that fails if it stops being true.
|
|
//
|
|
// The cure is NOT in this package and is deliberately not attempted here — see `counter`.
|
|
func TestTheIntakeCounterCountsTheWriteStreamAndNotCharacters(t *testing.T) {
|
|
count := func(body []byte) int64 {
|
|
var sink bytes.Buffer
|
|
c := &counter{w: &sink}
|
|
if _, err := io.Copy(c, bytes.NewReader(body)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if sink.Len() != len(body) {
|
|
t.Fatalf("the counter did not write through: %d of %d bytes", sink.Len(), len(body))
|
|
}
|
|
return c.runes
|
|
}
|
|
|
|
// (1) A UTF-8 text source: the number IS the character count, exactly — including across the
|
|
// chunk boundaries io.Copy hands the writer, which is why the count is stateless.
|
|
text := strings.Repeat("蛊真人 — глава первая.\n", 500)
|
|
if got, want := count([]byte(text)), int64(utf8.RuneCountInString(text)); got != want {
|
|
t.Errorf("a UTF-8 text source counted %d, want %d characters", got, want)
|
|
}
|
|
|
|
// (2) An EPUB: a ZIP archive, whose stream has nothing to do with the number of characters in
|
|
// the book inside it. The assertion is deliberately the INEQUALITY — this test does not bless a
|
|
// particular wrong number, it records that the number is not the one the field's name promises.
|
|
var epub bytes.Buffer
|
|
zw := zip.NewWriter(&epub)
|
|
f, err := zw.Create("OEBPS/ch1.xhtml")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.Write([]byte("<html><body><p>" + text + "</p></body></html>")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := zw.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inside := int64(utf8.RuneCountInString(text))
|
|
if got := count(epub.Bytes()); got == inside {
|
|
t.Errorf("an EPUB counted %d, which happens to equal the characters inside it — the pin below is meaningless if that is now true by construction", got)
|
|
} else if got > inside {
|
|
t.Errorf("an EPUB counted %d, MORE than the %d characters it contains: the number is a property of the container", got, inside)
|
|
}
|
|
|
|
// (3) UTF-16, which the engine accepts and decodes itself. Every ASCII character occupies two
|
|
// bytes and the high byte is a zero, so the count is roughly DOUBLE the characters.
|
|
utf16 := make([]byte, 0, 40)
|
|
for _, r := range "hello world" {
|
|
utf16 = append(utf16, byte(r), 0x00)
|
|
}
|
|
if got, want := count(utf16), int64(len("hello world")); got == want {
|
|
t.Errorf("a UTF-16 source counted %d, exactly its character count — that is not what this counter does and the note on `counter` would be wrong", got)
|
|
}
|
|
}
|
|
|
|
// ⛔ BOTH WRITERS OF `books.title` MUST AGREE ABOUT WHAT MAY BE IN IT. This file's own `MaxTitle`
|
|
// exists because "two copies of the number is how the intake and the patch come to disagree" — and
|
|
// the intake then disagreed with the rename door about something worse than a number: a title the
|
|
// user TYPED went through length bounding alone, so U+0000 reached Postgres, which cannot hold it in
|
|
// a `text` column, and U+2028 landed in a book's name.
|
|
//
|
|
// Found by acceptance, not by this session, in the very place the session had argued the border must
|
|
// be single — the class "fixed the reader, left the writer" applied to two writers of one column.
|
|
//
|
|
// Mutation caught: dropping the withoutControls call from Intake.title.
|
|
func TestBothTitleWritersAgreeAboutControlCharacters(t *testing.T) {
|
|
for name, bad := range map[string]string{
|
|
"NUL": "\u0000",
|
|
"newline": "\u000a",
|
|
"escape": "\u001b",
|
|
"line separator": "\u2028",
|
|
"paragraph separator": "\u2029",
|
|
} {
|
|
got := (Intake{Title: "be" + bad + "fore", Filename: "fallback.txt"}).title()
|
|
for _, r := range got {
|
|
if unicode.IsControl(r) || r == 0x2028 || r == 0x2029 {
|
|
t.Errorf("%s: the intake stored %q, carrying U+%04X — the rename door refuses this", name, got, r)
|
|
}
|
|
}
|
|
if got != "before" {
|
|
t.Errorf("%s: got %q, want the title with only the offending rune removed", name, got)
|
|
}
|
|
}
|
|
|
|
// A title that is NOTHING BUT control characters leaves nothing legible, so the book is named from
|
|
// its file — the same answer as naming nothing at all.
|
|
if got := (Intake{Title: "\u0000\u000a", Filename: "Мастер Гу.txt"}).title(); got != "Мастер Гу" {
|
|
t.Errorf("a title of only control characters gave %q, want the name derived from the file", got)
|
|
}
|
|
|
|
// ⚠ And the half that must NOT be cleaned: bidi marks and the zero-width joiners are ordinary
|
|
// content in Hebrew, Arabic, Devanagari and Persian. Dropping "everything invisible" would mangle
|
|
// a legitimate title in a pair this repository does not carry yet.
|
|
for name, title := range map[string]string{
|
|
"right-to-left mark": "\u200fdavar",
|
|
"zero-width joiner": "ka\u200dsha",
|
|
"soft hyphen": "a\u00adb",
|
|
} {
|
|
if got := (Intake{Title: title, Filename: "f.txt"}).title(); got != title {
|
|
t.Errorf("%s: a legitimate title was altered: %q -> %q", name, title, got)
|
|
}
|
|
}
|
|
}
|