package pipeline import ( "reflect" "strings" "testing" ) func TestSplitChunksSingleParagraph(t *testing.T) { got := SplitChunks("静かな図書館の朝。") want := []Chunk{{Chapter: 1, ChunkIdx: 0, Text: "静かな図書館の朝。"}} if !reflect.DeepEqual(got, want) { t.Fatalf("single paragraph = %+v, want %+v", got, want) } } func TestSplitChunksChapters(t *testing.T) { src := "Глава один." + chapterSep + "Глава два." + chapterSep + "Глава три." got := SplitChunks(src) want := []Chunk{ {Chapter: 1, ChunkIdx: 0, Text: "Глава один."}, {Chapter: 2, ChunkIdx: 0, Text: "Глава два."}, {Chapter: 3, ChunkIdx: 0, Text: "Глава три."}, } if !reflect.DeepEqual(got, want) { t.Fatalf("chapters = %+v, want %+v", got, want) } } func TestSplitChunksPacksToTarget(t *testing.T) { // Two paragraphs that together exceed the target must split into two chunks, // each never splitting a paragraph. p1 := strings.Repeat("a", 800) p2 := strings.Repeat("b", 800) got := SplitChunks(p1 + "\n\n" + p2) want := []Chunk{ {Chapter: 1, ChunkIdx: 0, Text: p1}, {Chapter: 1, ChunkIdx: 1, Text: p2}, } if !reflect.DeepEqual(got, want) { t.Fatalf("packing = chapters/chunks %d, want 2 chunks; got %+v", len(got), summarize(got)) } } func TestSplitChunksPacksSmallParagraphsTogether(t *testing.T) { // Small paragraphs pack into one chunk (under the target), joined by a blank line. got := SplitChunks("Абзац один.\n\nАбзац два.\n\nАбзац три.") if len(got) != 1 { t.Fatalf("small paragraphs must pack into one chunk, got %d: %+v", len(got), summarize(got)) } if got[0].Text != "Абзац один.\n\nАбзац два.\n\nАбзац три." { t.Fatalf("packed text = %q", got[0].Text) } } func TestSplitChunksOversizeParagraphIsOwnChunk(t *testing.T) { big := strings.Repeat("ы", 2000) // > targetChunkChars but never split got := SplitChunks(big) if len(got) != 1 || got[0].Text != big { t.Fatalf("an oversize paragraph must be one un-split chunk, got %+v", summarize(got)) } } func TestSplitChunksDropsEmpties(t *testing.T) { // Empty chapter blocks and whitespace-only paragraphs vanish; chapter numbers // only advance for non-empty chapters. src := "A" + chapterSep + " \n\n " + chapterSep + "B" got := SplitChunks(src) want := []Chunk{ {Chapter: 1, ChunkIdx: 0, Text: "A"}, {Chapter: 2, ChunkIdx: 0, Text: "B"}, } if !reflect.DeepEqual(got, want) { t.Fatalf("empties = %+v, want %+v", got, want) } } func TestSplitChunksDeterministic(t *testing.T) { src := "Абзац.\n\nЕщё абзац." + chapterSep + strings.Repeat("длинный ", 300) a, b := SplitChunks(src), SplitChunks(src) if !reflect.DeepEqual(a, b) { t.Fatal("SplitChunks must be deterministic") } } func summarize(cs []Chunk) []string { out := make([]string, len(cs)) for i, c := range cs { txt := c.Text if len(txt) > 12 { txt = txt[:12] + "…" } out[i] = strings.TrimSpace(txt) } return out }