286 lines
12 KiB
Go
286 lines
12 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"unicode/utf8"
|
||
|
||
"golang.org/x/text/encoding/japanese"
|
||
"golang.org/x/text/encoding/simplifiedchinese"
|
||
xunicode "golang.org/x/text/encoding/unicode"
|
||
"golang.org/x/text/transform"
|
||
)
|
||
|
||
// A realistic zh chapter: Han + CJK punctuation + fullwidth quotes + a stray Latin name + digits.
|
||
const zhChapter = "第一章 天空之陨\n\n方源睁开眼睛，“这是哪里？”他喃喃自语。三万年后，Gu 之力遍布天地。"
|
||
|
||
func gb18030Bytes(t *testing.T, s string) []byte {
|
||
t.Helper()
|
||
b, _, err := transform.Bytes(simplifiedchinese.GB18030.NewEncoder(), []byte(s))
|
||
if err != nil {
|
||
t.Fatalf("encode gb18030: %v", err)
|
||
}
|
||
return b
|
||
}
|
||
|
||
func utf16Bytes(t *testing.T, s string, endian xunicode.Endianness) []byte {
|
||
t.Helper()
|
||
b, _, err := transform.Bytes(xunicode.UTF16(endian, xunicode.UseBOM).NewEncoder(), []byte(s))
|
||
if err != nil {
|
||
t.Fatalf("encode utf16: %v", err)
|
||
}
|
||
return b
|
||
}
|
||
|
||
func TestDecodeSourceAutoGB18030(t *testing.T) {
|
||
raw := gb18030Bytes(t, zhChapter)
|
||
got, err := decodeSourceBytes(raw, "auto", "zh")
|
||
if err != nil {
|
||
t.Fatalf("auto-detect of a GB18030 chapter failed: %v", err)
|
||
}
|
||
if got != zhChapter {
|
||
t.Fatalf("GB18030 round-trip mismatch:\n got %q\nwant %q", got, zhChapter)
|
||
}
|
||
// The whole point: a GB18030 file is NOT valid UTF-8, so the naive string(raw) would be mojibake.
|
||
if strings.ContainsRune(got, '<27>') {
|
||
t.Error("decoded text contains U+FFFD — the lossless invariant is broken")
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceAutoUTF8NoBOM(t *testing.T) {
|
||
got, err := decodeSourceBytes([]byte(zhChapter), "auto", "zh")
|
||
if err != nil {
|
||
t.Fatalf("valid UTF-8 must pass through: %v", err)
|
||
}
|
||
if got != zhChapter {
|
||
t.Fatalf("UTF-8 passthrough mismatch: %q", got)
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceAutoUTF8BOM(t *testing.T) {
|
||
raw := append(append([]byte(nil), 0xEF, 0xBB, 0xBF), []byte(zhChapter)...)
|
||
got, err := decodeSourceBytes(raw, "auto", "zh")
|
||
if err != nil {
|
||
t.Fatalf("UTF-8 BOM file must decode: %v", err)
|
||
}
|
||
// The BOM byte may survive as U+FEFF here; NormalizeSource strips it downstream. What matters is
|
||
// the body decoded correctly and no mojibake appeared.
|
||
if !strings.Contains(got, "第一章 天空之陨") {
|
||
t.Fatalf("UTF-8 BOM body mis-decoded: %q", got)
|
||
}
|
||
if strings.ContainsRune(got, '<27>') {
|
||
t.Error("U+FFFD after BOM decode")
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceAutoUTF16(t *testing.T) {
|
||
for _, tc := range []struct {
|
||
name string
|
||
endian xunicode.Endianness
|
||
}{{"LE", xunicode.LittleEndian}, {"BE", xunicode.BigEndian}} {
|
||
raw := utf16Bytes(t, zhChapter, tc.endian)
|
||
got, err := decodeSourceBytes(raw, "auto", "zh")
|
||
if err != nil {
|
||
t.Fatalf("UTF-16 %s must decode: %v", tc.name, err)
|
||
}
|
||
if !strings.Contains(got, "方源睁开眼睛") {
|
||
t.Fatalf("UTF-16 %s mis-decoded: %q", tc.name, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceAutoRejectsGarbage(t *testing.T) {
|
||
// Random high bytes: not valid UTF-8, GB18030 yields U+FFFD → fail loud, not silent junk.
|
||
raw := []byte{0x00, 0x01, 0xFF, 0xFE, 0x80, 0x81, 0x00, 0x9F, 0xC0}
|
||
if got, err := decodeSourceBytes(raw, "auto", "zh"); err == nil {
|
||
t.Fatalf("garbage bytes must fail loud, got %q", got)
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceAutoRejectsLatin1(t *testing.T) {
|
||
// Latin-1 "Café déjà vu — naïve" is invalid UTF-8; GB18030 decodes it into mojibake that here
|
||
// contains U+FFFD, so it is rejected by the strict U+FFFD check (NOT the Han-density guard — see
|
||
// TestPlausibleCJKGuard for direct density coverage). Either way it must fail loud.
|
||
raw := []byte{0x43, 0x61, 0x66, 0xE9, 0x20, 0x64, 0xE9, 0x6A, 0xE0, 0x20, 0x76, 0x75, 0x20, 0x2D, 0x20, 0x6E, 0x61, 0xEF, 0x76, 0x65}
|
||
if got, err := decodeSourceBytes(raw, "auto", "zh"); err == nil {
|
||
t.Fatalf("Latin-1 mojibake must fail loud, got %q", got)
|
||
}
|
||
}
|
||
|
||
// TestPlausibleCJKGuard directly locks the Han-density guard (self-review: it had zero coverage —
|
||
// every mojibake fixture was caught earlier by the U+FFFD check). A clean-decoding low-Han string
|
||
// must be rejected; a Han-dense string must pass.
|
||
func TestPlausibleCJKGuard(t *testing.T) {
|
||
if err := plausibleCJK("Hello world, ordinary Latin prose with one 一 ideograph in it."); err == nil {
|
||
t.Error("low-Han text must fail the plausibility guard")
|
||
}
|
||
if err := plausibleCJK("方源睁开眼睛，这是哪里？天地无穷，万物玄妙。"); err != nil {
|
||
t.Errorf("Han-dense Chinese must pass the plausibility guard: %v", err)
|
||
}
|
||
}
|
||
|
||
// TestDecodeSourceRejectsShiftJISForJa is the self-review CRITICAL: a Japanese Shift-JIS source must
|
||
// NOT be silently accepted as GB18030 mojibake. With source_lang=ja the GB18030 auto-probe is off,
|
||
// so it fails loud (the operator converts to UTF-8). Shift-JIS is invalid UTF-8, so the ladder does
|
||
// reach the (now gated) GB18030 step.
|
||
func TestDecodeSourceRejectsShiftJISForJa(t *testing.T) {
|
||
ja := "彼女は静かに窓の外を眺めていた。"
|
||
sj, _, err := transform.Bytes(japanese.ShiftJIS.NewEncoder(), []byte(ja))
|
||
if err != nil {
|
||
t.Fatalf("encode shift-jis: %v", err)
|
||
}
|
||
if utf8.Valid(sj) {
|
||
t.Fatal("setup: shift-jis bytes unexpectedly valid UTF-8")
|
||
}
|
||
if got, derr := decodeSourceBytes(sj, "auto", "ja"); derr == nil {
|
||
t.Fatalf("a ja Shift-JIS source must fail loud under auto, got %q", got)
|
||
}
|
||
}
|
||
|
||
// TestDecodeSourceRejectsUTF16NoBOM is the self-review major: ASCII text saved as UTF-16 without a
|
||
// BOM is valid UTF-8 with interleaved NUL — the NUL guard must reject it, not pass silent corruption.
|
||
func TestDecodeSourceRejectsUTF16NoBOM(t *testing.T) {
|
||
enc := xunicode.UTF16(xunicode.LittleEndian, xunicode.IgnoreBOM).NewEncoder()
|
||
raw, _, err := transform.Bytes(enc, []byte("Hello world, a plain English sentence."))
|
||
if err != nil {
|
||
t.Fatalf("encode utf16-no-bom: %v", err)
|
||
}
|
||
if got, derr := decodeSourceBytes(raw, "auto", "en"); derr == nil {
|
||
t.Fatalf("UTF-16 without a BOM must fail loud (NUL guard), got %q", got)
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceExplicitGB18030(t *testing.T) {
|
||
raw := gb18030Bytes(t, zhChapter)
|
||
got, err := decodeSourceBytes(raw, "gb18030", "zh")
|
||
if err != nil {
|
||
t.Fatalf("explicit gb18030 must decode: %v", err)
|
||
}
|
||
if got != zhChapter {
|
||
t.Fatalf("explicit gb18030 mismatch: %q", got)
|
||
}
|
||
}
|
||
|
||
// TestDecodeSourceGB18030NULGuard is the D20.4 fix: a UTF-16/binary file mis-DECLARED as gb18030
|
||
// decodes each 0x00 byte to a VALID U+0000 (not U+FFFD), so the replacement-char check passes it —
|
||
// the NUL guard on the GB18030 path must catch it. Bytes "H\0i\0" (ASCII saved UTF-16LE-no-BOM, then
|
||
// declared gb18030). Reverting the checkNoNUL on the GB18030 return lets the corruption through.
|
||
func TestDecodeSourceGB18030NULGuard(t *testing.T) {
|
||
raw := []byte{0x48, 0x00, 0x69, 0x00} // "H\0i\0"
|
||
if got, err := decodeSourceBytes(raw, "gb18030", "zh"); err == nil {
|
||
t.Fatalf("NUL-carrying bytes declared gb18030 must fail loud (NUL guard), got %q", got)
|
||
} else if !strings.Contains(err.Error(), "NUL") {
|
||
t.Fatalf("expected a NUL-guard error, got %v", err)
|
||
}
|
||
}
|
||
|
||
// TestDecodeSourceUTF16NULGuard covers the sibling of the GB18030 NUL guard: a BOM-prefixed UTF-16LE
|
||
// stream carrying an embedded U+0000 decodes cleanly (no U+FFFD) via decodeUTF16, so only the NUL
|
||
// guard on that path catches it. Reverting checkNoNUL on the decodeUTF16 return lets it through.
|
||
func TestDecodeSourceUTF16NULGuard(t *testing.T) {
|
||
// BOM(FF FE) + "H"(48 00) + U+0000(00 00) + "i"(69 00) — valid UTF-16LE, decodes to "H\0i".
|
||
raw := []byte{0xFF, 0xFE, 0x48, 0x00, 0x00, 0x00, 0x69, 0x00}
|
||
if got, err := decodeSourceBytes(raw, "auto", "en"); err == nil {
|
||
t.Fatalf("a BOM'd UTF-16 stream with an embedded NUL must fail loud (NUL guard), got %q", got)
|
||
} else if !strings.Contains(err.Error(), "NUL") {
|
||
t.Fatalf("expected a NUL-guard error, got %v", err)
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceExplicitUTF8OnGBFailsLoud(t *testing.T) {
|
||
raw := gb18030Bytes(t, zhChapter) // not valid UTF-8
|
||
if _, err := decodeSourceBytes(raw, "utf8", "zh"); err == nil {
|
||
t.Fatal("declaring utf8 on GB18030 bytes must fail loud, not mojibake")
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceExplicitGBOnCorruptFailsLoud(t *testing.T) {
|
||
// A valid GB18030 chapter truncated mid-multibyte → U+FFFD → strict decode fails loud.
|
||
raw := gb18030Bytes(t, zhChapter)
|
||
if got, err := decodeSourceBytes(raw[:len(raw)-1], "gb18030", "zh"); err == nil {
|
||
t.Fatalf("truncated GB18030 must fail loud (U+FFFD), got %q", got)
|
||
}
|
||
}
|
||
|
||
func TestDecodeSourceUnknownEncoding(t *testing.T) {
|
||
if _, err := decodeSourceBytes([]byte("hi"), "shift-jis", ""); err == nil {
|
||
t.Fatal("an unsupported explicit encoding must be rejected")
|
||
}
|
||
}
|
||
|
||
// TestIngestTXTGB18030EndToEnd exercises the whole txt path: a GB18030 file with a \f chapter break
|
||
// decodes, splits, and normalizes — the D18 acceptance-book ingest case (蛊真人 is GB18030).
|
||
func TestIngestTXTGB18030EndToEnd(t *testing.T) {
|
||
dir := t.TempDir()
|
||
p := filepath.Join(dir, "book.txt")
|
||
twoChapters := zhChapter + "\f" + "第二章 洞窟\n\n他走进了洞窟深处。"
|
||
if err := os.WriteFile(p, gb18030Bytes(t, twoChapters), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
doc, err := IngestEncoded(p, "gb18030", "zh")
|
||
if err != nil {
|
||
t.Fatalf("ingest gb18030 txt: %v", err)
|
||
}
|
||
if len(doc.Chapters) != 2 {
|
||
t.Fatalf("want 2 chapters (split on \\f), got %d", len(doc.Chapters))
|
||
}
|
||
if !strings.Contains(doc.Chapters[0], "方源") || !strings.Contains(doc.Chapters[1], "洞窟") {
|
||
t.Fatalf("chapters mis-decoded: %q | %q", doc.Chapters[0], doc.Chapters[1])
|
||
}
|
||
for _, ch := range doc.Chapters {
|
||
if strings.ContainsRune(ch, '<27>') {
|
||
t.Errorf("U+FFFD in a normalized chapter: %q", ch)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestSplitTextChapters covers the CJK chapter-header splitting (D18: 蛊真人 uses «第N节» headers).
|
||
func TestSplitTextChapters(t *testing.T) {
|
||
jie := "第一节：начало\n方源开口。\n第二节：середина\n他走了。\n第三节：конец\n完。"
|
||
got := splitTextChapters(jie)
|
||
if len(got) != 3 {
|
||
t.Fatalf("第N节 split: want 3 chapters, got %d: %q", len(got), got)
|
||
}
|
||
if !strings.HasPrefix(got[0], "第一节") || !strings.HasPrefix(got[1], "第二节") {
|
||
t.Errorf("headers must start their chapter: %q", got)
|
||
}
|
||
|
||
zhang := "第1章 первая\nтекст\n第2章 вторая\nтекст"
|
||
if got := splitTextChapters(zhang); len(got) != 2 {
|
||
t.Fatalf("第N章 (arabic) split: want 2, got %d", len(got))
|
||
}
|
||
|
||
// Preamble before the first header merges into chapter 1 (第一节 == chapter 1).
|
||
withPreamble := "书名：Тест\n作者：Автор\n第一节：глава\nтело первой главы\n第二节：вторая\nтело"
|
||
got = splitTextChapters(withPreamble)
|
||
if len(got) != 2 {
|
||
t.Fatalf("preamble merge: want 2 chapters, got %d: %q", len(got), got)
|
||
}
|
||
if !strings.Contains(got[0], "书名") || !strings.Contains(got[0], "第一节") {
|
||
t.Errorf("preamble must merge into chapter 1: %q", got[0])
|
||
}
|
||
|
||
// No headers → a single chapter (backward compat).
|
||
if got := splitTextChapters("Просто текст без глав.\nВторая строка."); len(got) != 1 {
|
||
t.Fatalf("no-header text must be one chapter, got %d", len(got))
|
||
}
|
||
|
||
// A long prose line that merely opens with «第三章…» is NOT a header (length guard).
|
||
prose := "第三章的内容非常精彩，讲述了一个漫长而复杂的故事，充满了各种各样的细节和转折点情节。"
|
||
if got := splitTextChapters(prose); len(got) != 1 {
|
||
t.Fatalf("prose starting with 第三章 must not split, got %d", len(got))
|
||
}
|
||
|
||
// Self-review major: 回 is a measure word, so «第一回见面…» (a content glyph glued after the unit)
|
||
// must NOT split; a real «第N回：» header (separator after the unit) does.
|
||
huiProse := "第一回见面时他就心动了。\n第二回重逢依然如此。"
|
||
if got := splitTextChapters(huiProse); len(got) != 1 {
|
||
t.Fatalf("回 measure-word prose must not split (separator guard), got %d: %q", len(got), got)
|
||
}
|
||
huiHeader := "第一回：初见\nтекст первой.\n第二回：重逢\nтекст второй."
|
||
if got := splitTextChapters(huiHeader); len(got) != 2 {
|
||
t.Fatalf("第N回： headers must split, got %d", len(got))
|
||
}
|
||
}
|