96 lines
3 KiB
Go
96 lines
3 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"reflect"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
func TestSplitChunksSingleParagraph(t *testing.T) {
|
||
got := SplitChunks("静かな図書館の朝。")
|
||
want := []Chunk{{Chapter: 1, ChunkIdx: 0, Text: "静かな図書館の朝。"}}
|
||
if !reflect.DeepEqual(got, want) {
|
||
t.Fatalf("single paragraph = %+v, want %+v", got, want)
|
||
}
|
||
}
|
||
|
||
func TestSplitChunksChapters(t *testing.T) {
|
||
src := "Глава один." + chapterSep + "Глава два." + chapterSep + "Глава три."
|
||
got := SplitChunks(src)
|
||
want := []Chunk{
|
||
{Chapter: 1, ChunkIdx: 0, Text: "Глава один."},
|
||
{Chapter: 2, ChunkIdx: 0, Text: "Глава два."},
|
||
{Chapter: 3, ChunkIdx: 0, Text: "Глава три."},
|
||
}
|
||
if !reflect.DeepEqual(got, want) {
|
||
t.Fatalf("chapters = %+v, want %+v", got, want)
|
||
}
|
||
}
|
||
|
||
func TestSplitChunksPacksToTarget(t *testing.T) {
|
||
// Two paragraphs that together exceed the target must split into two chunks,
|
||
// each never splitting a paragraph.
|
||
p1 := strings.Repeat("a", 800)
|
||
p2 := strings.Repeat("b", 800)
|
||
got := SplitChunks(p1 + "\n\n" + p2)
|
||
want := []Chunk{
|
||
{Chapter: 1, ChunkIdx: 0, Text: p1},
|
||
{Chapter: 1, ChunkIdx: 1, Text: p2},
|
||
}
|
||
if !reflect.DeepEqual(got, want) {
|
||
t.Fatalf("packing = chapters/chunks %d, want 2 chunks; got %+v", len(got), summarize(got))
|
||
}
|
||
}
|
||
|
||
func TestSplitChunksPacksSmallParagraphsTogether(t *testing.T) {
|
||
// Small paragraphs pack into one chunk (under the target), joined by a blank line.
|
||
got := SplitChunks("Абзац один.\n\nАбзац два.\n\nАбзац три.")
|
||
if len(got) != 1 {
|
||
t.Fatalf("small paragraphs must pack into one chunk, got %d: %+v", len(got), summarize(got))
|
||
}
|
||
if got[0].Text != "Абзац один.\n\nАбзац два.\n\nАбзац три." {
|
||
t.Fatalf("packed text = %q", got[0].Text)
|
||
}
|
||
}
|
||
|
||
func TestSplitChunksOversizeParagraphIsOwnChunk(t *testing.T) {
|
||
big := strings.Repeat("ы", 2000) // > targetChunkChars but never split
|
||
got := SplitChunks(big)
|
||
if len(got) != 1 || got[0].Text != big {
|
||
t.Fatalf("an oversize paragraph must be one un-split chunk, got %+v", summarize(got))
|
||
}
|
||
}
|
||
|
||
func TestSplitChunksDropsEmpties(t *testing.T) {
|
||
// Empty chapter blocks and whitespace-only paragraphs vanish; chapter numbers
|
||
// only advance for non-empty chapters.
|
||
src := "A" + chapterSep + " \n\n " + chapterSep + "B"
|
||
got := SplitChunks(src)
|
||
want := []Chunk{
|
||
{Chapter: 1, ChunkIdx: 0, Text: "A"},
|
||
{Chapter: 2, ChunkIdx: 0, Text: "B"},
|
||
}
|
||
if !reflect.DeepEqual(got, want) {
|
||
t.Fatalf("empties = %+v, want %+v", got, want)
|
||
}
|
||
}
|
||
|
||
func TestSplitChunksDeterministic(t *testing.T) {
|
||
src := "Абзац.\n\nЕщё абзац." + chapterSep + strings.Repeat("длинный ", 300)
|
||
a, b := SplitChunks(src), SplitChunks(src)
|
||
if !reflect.DeepEqual(a, b) {
|
||
t.Fatal("SplitChunks must be deterministic")
|
||
}
|
||
}
|
||
|
||
func summarize(cs []Chunk) []string {
|
||
out := make([]string, len(cs))
|
||
for i, c := range cs {
|
||
txt := c.Text
|
||
if len(txt) > 12 {
|
||
txt = txt[:12] + "…"
|
||
}
|
||
out[i] = strings.TrimSpace(txt)
|
||
}
|
||
return out
|
||
}
|