diff --git a/backend/internal/pipeline/chunker.go b/backend/internal/pipeline/chunker.go
index 783f3e7..abe0381 100644
--- a/backend/internal/pipeline/chunker.go
+++ b/backend/internal/pipeline/chunker.go
@@ -1,18 +1,32 @@
package pipeline
-import "strings"
+import (
+ "strings"
+ "unicode"
+)
-// chunker.go: the Веха-2 segmenter SEAM. Its only job right now is to turn a
-// normalized source into an ordered list of (chapter, chunk) units so the runner
-// loop, resume and disposition machinery can be exercised over MANY chunks — the
-// hardcoded "chapter=1/chunk=0" of Фаза 0 is gone.
+// chunker.go: the шаг-3a SOURCE segmenter. It turns the per-chapter normalized
+// text produced by ingest.go into an ordered list of (chapter, chunk) units for
+// the runner loop. It replaced the Веха-2 placeholder (chapter split on \f +
+// blank-line paragraph packing to a CHAR budget) with real segmentation: pack
+// whole paragraphs to a ~1–2k-TOKEN budget, and when a single paragraph exceeds
+// the budget descend to SENTENCE boundaries — never splitting a sentence.
//
-// This is a deliberate PLACEHOLDER. The real linguistic chunker of шаг 3
-// (sentence segmentation, read-only overlap, ruby name-reading extraction for
-// ja) replaces SplitChunks WITHOUT touching the loop — it only has to keep the
-// Chunk contract and bump chunkerVersion. chunkerVersion is folded into the
-// snapshot (render.go), so changing the segmentation is an explicit
-// re-translation (a loud --resnapshot), never a silent cache miss (R6/D5.2).
+// Contract kept intact for the loop (runner.go depends on it): SplitChunks emits
+// Chunk{Chapter,ChunkIdx,Text}; ChunkIdx resets to 0 per chapter; the function is
+// a PURE, deterministic function of its input (no time/rand, no map iteration).
+// Its behavior is versioned by chunkerVersion (render.go), folded into the job
+// snapshot — so changing the segmentation is a loud --resnapshot re-translation,
+// never a silent cache miss (§3.4/R6/D5.2). The chunk target is a CONST covered by
+// that version (Р2 = code); were it ever made config-tunable it MUST be folded into
+// the top-level snapshot like coverageSnap (§7d), not left to the version string.
+//
+// IMPORTANT: this sentence splitter is a SEPARATE, independent piece of logic from
+// coverage.go's splitSentences. That one is a bit-for-bit port of the Python oracle
+// (refusal_bench.py) over the RUSSIAN OUTPUT and is Python-locked (D12 Q1). THIS one
+// segments the SOURCE (zh/ja/en), is versioned by chunkerVersion (Р2 = code), and is
+// free to be more faithful (closing-bracket absorption, an abbreviation guard). Do
+// not couple them.
// Chunk is one ordered unit of translation work.
type Chunk struct {
@@ -21,68 +35,357 @@ type Chunk struct {
Text string
}
-// chapterSep splits the source into chapters. Form feed (U+000C) is the ASCII
-// "page/section break" — semantically a chapter boundary, invisible in prose,
-// and untouched by NormalizeSource (BOM/CRLF/NFC/outer-trim leave an internal
-// \f intact). A source with no \f is a single chapter (Фаза-0 backward compat:
-// the one-file example stays one chapter).
-const chapterSep = "\f"
+// targetChunkTokens is the packing budget: paragraphs (or, inside an oversize
+// paragraph, sentences) are greedily packed until the WHOLE chunk's estimated
+// tokens (EstimateTokens' formula, applied once — see appendChapterChunks) would
+// exceed this. ~1500 sits in the plan's "1–2k токенов по границам абзацев" corridor
+// (02-mvp Фаза 1); a chunk of ordinary prose lands well above the coverage gate's
+// 500-char floor, so the excision gate is not silently disabled (§7b). A chunk can
+// still fall under the floor only when a whole chapter (or its tail) is genuinely
+// that short — a documented boundary. A const, not config: the size is a code
+// segmentation rule (Р2), covered by chunkerVersion.
+const targetChunkTokens = 1500
-// targetChunkChars is the placeholder chunk size (≈ the plan's "1–2k tokens";
-// for a CJK source ~1 char ≈ 1 token). Paragraphs are packed up to this size and
-// never split, so the 264-char example remains a single chunk.
-const targetChunkChars = 1500
-
-// SplitChunks segments a NORMALIZED source (post-NormalizeSource) into an ordered
-// chunk list. Rule: split on \f into 1-based chapters; within a chapter pack
-// blank-line paragraphs greedily into chunks of ≤ targetChunkChars, never
-// splitting a paragraph (a paragraph larger than the target is its own chunk);
-// drop empty pieces. Fully deterministic.
-func SplitChunks(source string) []Chunk {
+// SplitChunks segments the ordered, per-chapter NORMALIZED text of a Document
+// (ingest.go already applied NormalizeSource and split chapters) into an ordered
+// chunk list. Chapters that yield no text (blank/whitespace-only, e.g. an epub
+// cover/nav page) do NOT consume a chapter number, so numbering stays dense over
+// real content (Фаза-0 backward compat). Fully deterministic.
+func SplitChunks(chapters []string) []Chunk {
var out []Chunk
chapterNo := 0
- for _, chapRaw := range strings.Split(source, chapterSep) {
- paras := splitParagraphs(chapRaw)
+ for _, chapText := range chapters {
+ paras := splitParagraphs(chapText)
if len(paras) == 0 {
- continue // an empty chapter block does not consume a chapter number
+ continue // an empty chapter does not consume a chapter number
}
chapterNo++
- chunkIdx := 0
- var buf strings.Builder
- flush := func() {
- if buf.Len() == 0 {
- return
- }
- out = append(out, Chunk{Chapter: chapterNo, ChunkIdx: chunkIdx, Text: buf.String()})
- chunkIdx++
- buf.Reset()
- }
- for _, p := range paras {
- // Start a new chunk if adding this paragraph would exceed the target
- // and the current chunk is non-empty (never split a paragraph).
- if buf.Len() > 0 && buf.Len()+len("\n\n")+len(p) > targetChunkChars {
- flush()
- }
- if buf.Len() > 0 {
- buf.WriteString("\n\n")
- }
- buf.WriteString(p)
- }
- flush()
+ out = appendChapterChunks(out, chapterNo, paras)
}
return out
}
-// splitParagraphs breaks a chapter block on blank lines and trims each paragraph,
-// dropping empties. Deterministic; whitespace-only paragraphs vanish.
-func splitParagraphs(block string) []string {
+// appendChapterChunks packs one chapter's paragraphs into chunks and appends them.
+// Rule: pack whole paragraphs while the running token estimate stays within
+// targetChunkTokens (prefer paragraph boundaries); a paragraph that alone exceeds
+// the budget is flushed on its own and then SENTENCE-packed (never splitting a
+// sentence; a single sentence over budget is its own chunk).
+//
+// The running budget is tracked as ADDITIVE char-class counts (CJK vs other),
+// applying EstimateTokens' formula (and its 16-token floor) ONCE to the whole
+// chunk — NOT by summing per-unit EstimateTokens. Summing per-unit floors every
+// tiny paragraph up to 16 tokens, so a dialogue chapter of many short lines would
+// flush a chunk at ~⅓ its true token content, needlessly dropping real content
+// under the coverage gate's 500-char floor (self-review finding). Counting classes
+// and flooring once reproduces EstimateTokens(joined text) exactly (the "\n\n"
+// separators are whitespace, which EstimateTokens ignores).
+func appendChapterChunks(out []Chunk, chapterNo int, paras []string) []Chunk {
+ chunkIdx := 0
+ var buf []string // paragraphs accumulated for the current chunk
+ var bufCJK, bufOther int
+ flush := func() {
+ if len(buf) == 0 {
+ return
+ }
+ if text := strings.TrimSpace(strings.Join(buf, "\n\n")); text != "" {
+ out = append(out, Chunk{Chapter: chapterNo, ChunkIdx: chunkIdx, Text: text})
+ chunkIdx++
+ }
+ buf = buf[:0]
+ bufCJK, bufOther = 0, 0
+ }
+ for _, p := range paras {
+ pCJK, pOther := tokenClassCounts(p)
+ if estTokensFrom(pCJK, pOther) > targetChunkTokens {
+ // Oversize paragraph: close any open chunk at this paragraph boundary,
+ // then split the paragraph at sentence boundaries into its own chunks.
+ flush()
+ for _, sub := range packSentences(p) {
+ if text := strings.TrimSpace(sub); text != "" {
+ out = append(out, Chunk{Chapter: chapterNo, ChunkIdx: chunkIdx, Text: text})
+ chunkIdx++
+ }
+ }
+ continue
+ }
+ // Normal paragraph: start a new chunk if adding it would exceed the budget
+ // and the current chunk is non-empty (never split a paragraph here).
+ if len(buf) > 0 && estTokensFrom(bufCJK+pCJK, bufOther+pOther) > targetChunkTokens {
+ flush()
+ }
+ buf = append(buf, p)
+ bufCJK, bufOther = bufCJK+pCJK, bufOther+pOther
+ }
+ flush()
+ return out
+}
+
+// packSentences splits one oversize paragraph into sentence segments (lossless
+// tiling — concatenation reproduces the paragraph) and packs consecutive segments
+// into ≤ targetChunkTokens sub-chunks, never splitting a sentence. A single
+// sentence larger than the budget becomes its own (oversize) sub-chunk — the only
+// way to honor "never split a sentence". Returns the sub-chunk texts in order. Uses
+// the same additive char-class budget as appendChapterChunks (floor applied once).
+func packSentences(paragraph string) []string {
+ segs := splitSourceSentences(paragraph)
+ var chunks []string
+ var buf strings.Builder
+ var bufCJK, bufOther int
+ flush := func() {
+ if buf.Len() == 0 {
+ return
+ }
+ chunks = append(chunks, buf.String())
+ buf.Reset()
+ bufCJK, bufOther = 0, 0
+ }
+ for _, s := range segs {
+ sCJK, sOther := tokenClassCounts(s)
+ if buf.Len() > 0 && estTokensFrom(bufCJK+sCJK, bufOther+sOther) > targetChunkTokens {
+ flush()
+ }
+ buf.WriteString(s)
+ bufCJK, bufOther = bufCJK+sCJK, bufOther+sOther
+ }
+ flush()
+ return chunks
+}
+
+// tokenClassCounts counts the two token-estimate character classes of EstimateTokens
+// (render.go): CJK ideographs/kana/hangul, and "other" non-space runes (whitespace
+// folds into neighbours). Kept separate from EstimateTokens so the packer can sum
+// counts ADDITIVELY across units and floor once at the chunk level. Mirrors
+// EstimateTokens' classes; a recalibration there that changes packing granularity
+// is itself a chunkerVersion bump (both are behaviour changes covered by the snapshot).
+func tokenClassCounts(s string) (cjk, other int) {
+ for _, r := range s {
+ switch {
+ case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul):
+ cjk++
+ case unicode.IsSpace(r):
+ default:
+ other++
+ }
+ }
+ return cjk, other
+}
+
+// estTokensFrom applies EstimateTokens' formula (cjk + other/3, floored at 16) to
+// pre-counted class totals — identical to EstimateTokens over the joined text.
+func estTokensFrom(cjk, other int) int {
+ est := cjk + other/3
+ if est < 16 {
+ est = 16
+ }
+ return est
+}
+
+// splitParagraphs breaks a chapter into paragraphs on blank lines and trims each,
+// dropping empties. Deterministic; whitespace-only paragraphs vanish. (Ingest emits
+// a blank line between block-level epub elements, so paragraphs survive extraction.)
+func splitParagraphs(chapter string) []string {
var paras []string
- for _, raw := range strings.Split(block, "\n\n") {
- // Collapse a run of blank lines: a "paragraph" that is itself only blank
- // lines (from 3+ consecutive newlines) trims to "".
+ for _, raw := range strings.Split(chapter, "\n\n") {
if p := strings.TrimSpace(raw); p != "" {
paras = append(paras, p)
}
}
return paras
}
+
+// --- source sentence segmentation (zh/ja/en) — chunkerVersion-versioned ---------
+
+// splitSourceSentences segments one paragraph at sentence boundaries, returning
+// substrings that TILE the input (their concatenation is exactly the paragraph —
+// no character is lost or moved), so a packed sub-chunk is an exact source slice.
+// A boundary is a run of sentence terminators, plus any closing quotes/brackets, at:
+//
+// (1) a CJK terminator (。!?) — zero-width, CJK prose has no inter-sentence space —
+// but ONLY at quote-depth 0. A terminator inside a quote is dialogue-internal:
+// 「止まれ!」と言った。 is ONE sentence ending at the outer 。, not two (the same
+// for zh “…!”)— this is the dialogue case §3.7/D12-Q1 defers for the coverage
+// oracle, but our source splitter is free to get it right;
+// (2) an ASCII terminator (.!?) or ellipsis (…) — only when followed by whitespace
+// or end-of-paragraph (so a decimal/URL dot mid-token does not split, and en
+// dialogue "Stop!" She ran. still splits on the space after the quote).
+//
+// Trailing whitespace after a boundary is attached to the LEFT segment (the next
+// segment starts at the next content char), keeping the tiling exact. An
+// abbreviation guard suppresses a split after a lone "." following a known
+// abbreviation or a single-letter initial (never split "Mr. Smith" / "J. R. R.").
+// When unsure it prefers NOT to split (over-merge is safe — units just pack
+// together; over-split cuts a sentence, the forbidden case). Unbalanced quotes are
+// tolerated (depth floors at 0), erring toward merge.
+func splitSourceSentences(paragraph string) []string {
+ runes := []rune(paragraph)
+ var segs []string
+ start, i, depth := 0, 0, 0
+ for i < len(runes) {
+ r := runes[i]
+ if isOpenQuote(r) {
+ depth++
+ i++
+ continue
+ }
+ if isCloseQuote(r) {
+ if depth > 0 {
+ depth--
+ }
+ i++
+ continue
+ }
+ if !isSourceTerminator(r) {
+ i++
+ continue
+ }
+ // Consume a run of terminators (collapse "。。。" / "!?" into one boundary).
+ j := i
+ cjk := false
+ for j < len(runes) && isSourceTerminator(runes[j]) {
+ if isCJKTerminator(runes[j]) {
+ cjk = true
+ }
+ j++
+ }
+ // The terminator's quote-depth is the depth HERE — before absorbing the
+ // closing quote that may follow it (「…!」: the ! is at depth 1, the 」 that
+ // closes the quote comes after and must not retro-promote it to depth 0).
+ depthHere := depth
+ // Absorb a run of closing quotes/brackets that belong to the sentence,
+ // decrementing quote-depth for the depth-tracked closers among them.
+ e := j
+ for e < len(runes) && isClosingBracket(runes[e]) {
+ if isCloseQuote(runes[e]) && depth > 0 {
+ depth--
+ }
+ e++
+ }
+ boundary := false
+ if cjk {
+ boundary = depthHere == 0 // dialogue-internal terminators do not end a sentence
+ } else {
+ // ASCII terminator / ellipsis: a boundary only before whitespace or EOS.
+ boundary = e >= len(runes) || isASCIISpace(runes[e])
+ // Abbreviation guard: a lone "." after an abbreviation/initial is not a end.
+ if boundary && j-i == 1 && runes[i] == '.' && isAbbrevBefore(runes[start:i]) {
+ boundary = false
+ }
+ }
+ if !boundary {
+ i = e
+ continue
+ }
+ // Attach the following whitespace run to the left segment (keeps tiling exact).
+ w := e
+ for w < len(runes) && isASCIISpace(runes[w]) {
+ w++
+ }
+ segs = append(segs, string(runes[start:w]))
+ start, i = w, w
+ }
+ if start < len(runes) {
+ segs = append(segs, string(runes[start:]))
+ }
+ return segs
+}
+
+// isOpenQuote / isCloseQuote are the DEPTH-TRACKED paired quote/bracket delimiters
+// (CJK brackets + curly quotes with distinct open/close glyphs). Straight quotes
+// (" ') are NOT tracked — same glyph opens and closes, so depth is undecidable; en
+// relies on the whitespace rule instead, which is correct for spaced prose.
+func isOpenQuote(r rune) bool {
+ switch r {
+ case '「', '『', '(', '【', '《', '〈', '[', '{', '“', '‘':
+ return true
+ }
+ return false
+}
+
+func isCloseQuote(r rune) bool {
+ switch r {
+ case '」', '』', ')', '】', '》', '〉', ']', '}', '”', '’':
+ return true
+ }
+ return false
+}
+
+// isSourceTerminator is the sentence-terminator class for the SOURCE: CJK 。!?,
+// ASCII .!?, and the ellipsis … (U+2026).
+func isSourceTerminator(r rune) bool {
+ switch r {
+ case '。', '!', '?', '.', '!', '?', '…':
+ return true
+ }
+ return false
+}
+
+// isCJKTerminator is the zero-width-splitting subset (fullwidth CJK). The ellipsis
+// is deliberately NOT here: "……" mid-CJK-sentence must not split, but "……。" still
+// splits on the 。 in the same run.
+func isCJKTerminator(r rune) bool {
+ switch r {
+ case '。', '!', '?':
+ return true
+ }
+ return false
+}
+
+// isClosingBracket is the closing-quote/bracket class absorbed into the left
+// sentence after its terminator (ja/zh brackets + straight/curly quotes).
+func isClosingBracket(r rune) bool {
+ switch r {
+ case '」', '』', '】', ')', ')', '》', '〉', ']', ']', '}', '}',
+ '"', '\'', '”', '’':
+ return true
+ }
+ return false
+}
+
+// isASCIISpace is the whitespace used for ASCII-terminator boundary detection and
+// tiling — the ordinary prose separators (space, tab, newline, CR, form feed,
+// vertical tab, and the ideographic space U+3000).
+func isASCIISpace(r rune) bool {
+ switch r {
+ case ' ', '\t', '\n', '\r', '\f', '\v', ' ':
+ return true
+ }
+ return false
+}
+
+// sourceAbbrevs are trailing tokens after which a lone "." is treated as an
+// abbreviation, not a sentence end (case-insensitive). A pragmatic set for prose;
+// the acceptance path is ja→ru where CJK terminators dominate, so this only guards
+// the en source. Single-letter initials are handled separately (isAbbrevBefore).
+var sourceAbbrevs = map[string]bool{
+ "mr": true, "mrs": true, "ms": true, "dr": true, "prof": true, "st": true,
+ "jr": true, "sr": true, "vs": true, "no": true, "vol": true, "ch": true,
+ "fig": true, "col": true, "gen": true, "sgt": true, "capt": true, "lt": true,
+ "rev": true, "gov": true, "sen": true, "rep": true, "etc": true, "inc": true,
+ "ltd": true, "co": true, "mt": true, "ave": true, "rd": true,
+}
+
+// isAbbrevBefore reports whether the text immediately before a lone "." ends in a
+// known abbreviation or a single-letter initial (so the "." is not a boundary).
+func isAbbrevBefore(left []rune) bool {
+ // Trailing run of ASCII letters (the "word" before the period).
+ end := len(left)
+ k := end
+ for k > 0 && isASCIILetter(left[k-1]) {
+ k--
+ }
+ word := left[k:end]
+ if len(word) == 0 {
+ return false
+ }
+ // Single-letter initial ("J.") — only when it stands alone (start or preceded by
+ // a space/opening punctuation), never a one-letter word ending a real sentence
+ // mid-flow would be rare; over-merge is the safe side.
+ if len(word) == 1 {
+ return k == 0 || !isASCIILetter(left[k-1])
+ }
+ return sourceAbbrevs[strings.ToLower(string(word))]
+}
+
+func isASCIILetter(r rune) bool {
+ return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')
+}
diff --git a/backend/internal/pipeline/chunker_test.go b/backend/internal/pipeline/chunker_test.go
index 3a2ec0a..c011697 100644
--- a/backend/internal/pipeline/chunker_test.go
+++ b/backend/internal/pipeline/chunker_test.go
@@ -7,7 +7,7 @@ import (
)
func TestSplitChunksSingleParagraph(t *testing.T) {
- got := SplitChunks("静かな図書館の朝。")
+ got := SplitChunks([]string{"静かな図書館の朝。"})
want := []Chunk{{Chapter: 1, ChunkIdx: 0, Text: "静かな図書館の朝。"}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("single paragraph = %+v, want %+v", got, want)
@@ -15,8 +15,7 @@ func TestSplitChunksSingleParagraph(t *testing.T) {
}
func TestSplitChunksChapters(t *testing.T) {
- src := "Глава один." + chapterSep + "Глава два." + chapterSep + "Глава три."
- got := SplitChunks(src)
+ got := SplitChunks([]string{"Глава один.", "Глава два.", "Глава три."})
want := []Chunk{
{Chapter: 1, ChunkIdx: 0, Text: "Глава один."},
{Chapter: 2, ChunkIdx: 0, Text: "Глава два."},
@@ -28,23 +27,22 @@ func TestSplitChunksChapters(t *testing.T) {
}
func TestSplitChunksPacksToTarget(t *testing.T) {
- // Two paragraphs that together exceed the target must split into two chunks,
- // each never splitting a paragraph.
- p1 := strings.Repeat("a", 800)
- p2 := strings.Repeat("b", 800)
- got := SplitChunks(p1 + "\n\n" + p2)
+ // Two paragraphs that together exceed the TOKEN target (each ~1000 tokens for
+ // cyrillic: 3000 chars / 3) must split into two chunks, each kept whole.
+ p1 := strings.Repeat("а", 3000)
+ p2 := strings.Repeat("б", 3000)
+ got := SplitChunks([]string{p1 + "\n\n" + p2})
want := []Chunk{
{Chapter: 1, ChunkIdx: 0, Text: p1},
{Chapter: 1, ChunkIdx: 1, Text: p2},
}
if !reflect.DeepEqual(got, want) {
- t.Fatalf("packing = chapters/chunks %d, want 2 chunks; got %+v", len(got), summarize(got))
+ t.Fatalf("packing = %d chunks, want 2 whole paragraphs; got %+v", len(got), summarize(got))
}
}
func TestSplitChunksPacksSmallParagraphsTogether(t *testing.T) {
- // Small paragraphs pack into one chunk (under the target), joined by a blank line.
- got := SplitChunks("Абзац один.\n\nАбзац два.\n\nАбзац три.")
+ got := SplitChunks([]string{"Абзац один.\n\nАбзац два.\n\nАбзац три."})
if len(got) != 1 {
t.Fatalf("small paragraphs must pack into one chunk, got %d: %+v", len(got), summarize(got))
}
@@ -53,19 +51,62 @@ func TestSplitChunksPacksSmallParagraphsTogether(t *testing.T) {
}
}
-func TestSplitChunksOversizeParagraphIsOwnChunk(t *testing.T) {
- big := strings.Repeat("ы", 2000) // > targetChunkChars but never split
- got := SplitChunks(big)
+func TestSplitChunksOversizeParagraphSplitsAtSentences(t *testing.T) {
+ // One paragraph far over the token target, built of many whole sentences, must
+ // split into >1 chunk AND never cut a sentence: every original sentence must
+ // survive intact and in order across the chunks.
+ sentence := strings.Repeat("слово ", 40) + "конец." // ~80 tokens each
+ var sb strings.Builder
+ for i := 0; i < 40; i++ { // ~3200 tokens total ≫ 1500
+ sb.WriteString(sentence)
+ sb.WriteByte(' ')
+ }
+ para := strings.TrimSpace(sb.String())
+ got := SplitChunks([]string{para})
+ if len(got) < 2 {
+ t.Fatalf("oversize paragraph must split into >1 chunk, got %d", len(got))
+ }
+ wantSentences := trimAll(splitSourceSentences(para))
+ var gotSentences []string
+ for _, c := range got {
+ gotSentences = append(gotSentences, trimAll(splitSourceSentences(c.Text))...)
+ }
+ if !reflect.DeepEqual(gotSentences, wantSentences) {
+ t.Fatalf("sentence integrity broken:\n got %d sentences\nwant %d sentences", len(gotSentences), len(wantSentences))
+ }
+}
+
+func TestSplitChunksShortLinesPackByTrueTokenBudget(t *testing.T) {
+ // Many short dialogue paragraphs must pack by TRUE token content, not by summing
+ // the per-unit 16-token floor. 150 lines of 「はい。」 are ~450 true tokens (cjk=300,
+ // other=450 → 300+150), so they belong in ONE chunk. Summing the floor (150×16)
+ // would wrongly flush into ~2 sub-500-char chunks the coverage gate then skips.
+ var paras []string
+ for i := 0; i < 150; i++ {
+ paras = append(paras, "「はい。」")
+ }
+ got := SplitChunks([]string{strings.Join(paras, "\n\n")})
+ if len(got) != 1 {
+ t.Fatalf("short lines must pack by true token budget into 1 chunk, got %d chunks", len(got))
+ }
+ if EstimateTokens(got[0].Text) > targetChunkTokens {
+ t.Fatalf("packed chunk overshoots the budget: %d tokens", EstimateTokens(got[0].Text))
+ }
+}
+
+func TestSplitChunksOversizeSentenceIsOwnChunk(t *testing.T) {
+ // A single sentence (no internal terminator) larger than the budget can only be
+ // its own chunk — never split.
+ big := strings.Repeat("ы", 6000) // ~2000 tokens, one un-terminated run
+ got := SplitChunks([]string{big})
if len(got) != 1 || got[0].Text != big {
- t.Fatalf("an oversize paragraph must be one un-split chunk, got %+v", summarize(got))
+ t.Fatalf("an oversize sentence must be one un-split chunk, got %+v", summarize(got))
}
}
func TestSplitChunksDropsEmpties(t *testing.T) {
- // Empty chapter blocks and whitespace-only paragraphs vanish; chapter numbers
- // only advance for non-empty chapters.
- src := "A" + chapterSep + " \n\n " + chapterSep + "B"
- got := SplitChunks(src)
+ // A blank/whitespace-only chapter vanishes and does NOT consume a chapter number.
+ got := SplitChunks([]string{"A", " \n\n ", "B"})
want := []Chunk{
{Chapter: 1, ChunkIdx: 0, Text: "A"},
{Chapter: 2, ChunkIdx: 0, Text: "B"},
@@ -76,13 +117,94 @@ func TestSplitChunksDropsEmpties(t *testing.T) {
}
func TestSplitChunksDeterministic(t *testing.T) {
- src := "Абзац.\n\nЕщё абзац." + chapterSep + strings.Repeat("длинный ", 300)
- a, b := SplitChunks(src), SplitChunks(src)
+ chapters := []string{
+ "Абзац.\n\nЕщё абзац.",
+ strings.Repeat("слово ", 400) + "конец.",
+ }
+ a, b := SplitChunks(chapters), SplitChunks(chapters)
if !reflect.DeepEqual(a, b) {
t.Fatal("SplitChunks must be deterministic")
}
}
+func TestSplitChunksNeverSplitsAbbreviation(t *testing.T) {
+ // Two long en paragraphs each ending mid-flow with "Mr. Smith" must not chunk
+ // between "Mr." and "Smith" — the abbreviation guard keeps the sentence whole.
+ p := strings.Repeat("The road went on and on. ", 100) + "It was Mr. Smith who arrived."
+ got := SplitChunks([]string{p})
+ for _, c := range got {
+ if strings.HasSuffix(strings.TrimSpace(c.Text), "Mr.") {
+ t.Fatalf("chunk ended at an abbreviation 'Mr.' — a sentence was cut: %q", c.Text)
+ }
+ }
+}
+
+// --- source sentence splitter (zh/ja/en) ---------------------------------------
+
+func TestSplitSourceSentencesTilesLosslessly(t *testing.T) {
+ // The segments must TILE the input: concatenation reproduces it byte-for-byte.
+ inputs := []string{
+ "静かな朝。彼は歩いた。「止まれ!」と叫んだ。",
+ "他说。“不要!”然后离开了。",
+ "Hello world. This is a test! Really? Yes.",
+ "Mr. Smith met Dr. J. R. R. Brown. They talked.",
+ "末尾に句点なし",
+ "",
+ }
+ for _, in := range inputs {
+ segs := splitSourceSentences(in)
+ if got := strings.Join(segs, ""); got != in {
+ t.Fatalf("tiling broke for %q: rejoined to %q", in, got)
+ }
+ }
+}
+
+func TestSplitSourceSentencesJapanese(t *testing.T) {
+ // CJK terminators split zero-width; a closing quote is absorbed into its sentence.
+ got := trimAll(splitSourceSentences("静かな朝。「止まれ!」と彼は言った。"))
+ want := []string{"静かな朝。", "「止まれ!」と彼は言った。"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("ja split = %q, want %q", got, want)
+ }
+}
+
+func TestSplitSourceSentencesEnglish(t *testing.T) {
+ got := trimAll(splitSourceSentences(`He said "Stop!" She ran. Done.`))
+ want := []string{`He said "Stop!"`, "She ran.", "Done."}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("en split = %q, want %q", got, want)
+ }
+}
+
+func TestSplitSourceSentencesEllipsisNotOverSplitInCJK(t *testing.T) {
+ // "……" mid-CJK-sentence must not split; the trailing 。 still ends it.
+ got := trimAll(splitSourceSentences("そう……だった。次へ。"))
+ want := []string{"そう……だった。", "次へ。"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("ellipsis split = %q, want %q", got, want)
+ }
+}
+
+func TestSplitSourceSentencesAbbreviations(t *testing.T) {
+ // Abbreviations and single-letter initials do not end a sentence.
+ got := trimAll(splitSourceSentences("Dr. J. R. R. Tolkien wrote it. The end."))
+ want := []string{"Dr. J. R. R. Tolkien wrote it.", "The end."}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("abbrev split = %q, want %q", got, want)
+ }
+}
+
+// trimAll TrimSpaces each segment and drops empties (test convenience).
+func trimAll(segs []string) []string {
+ var out []string
+ for _, s := range segs {
+ if t := strings.TrimSpace(s); t != "" {
+ out = append(out, t)
+ }
+ }
+ return out
+}
+
func summarize(cs []Chunk) []string {
out := make([]string, len(cs))
for i, c := range cs {
diff --git a/backend/internal/pipeline/ingest.go b/backend/internal/pipeline/ingest.go
new file mode 100644
index 0000000..03d6bfd
--- /dev/null
+++ b/backend/internal/pipeline/ingest.go
@@ -0,0 +1,389 @@
+package pipeline
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/xml"
+ "fmt"
+ "io"
+ "net/url"
+ "os"
+ "path"
+ "strings"
+)
+
+// ingest.go: the шаг-3a import layer. Ingest turns a source file (txt or epub)
+// into an ordered list of per-chapter NORMALIZED text plus the ruby/furigana
+// readings captured on the way (04-unhappy §4 / D9). It is the ONLY place that
+// reads the source; the runner then feeds doc.Chapters to SplitChunks and persists
+// doc.Ruby (runner.go). It is offline and deterministic — no LLM, no time/rand — so
+// the whole path is $0 and reproducible.
+//
+// epub v1 is a text extraction (02-mvp:25): read the chapters in spine order,
+// strip tags, drop inline markup/styling. The ONE thing we do NOT drop is ruby:
+// base carries an author reading of a name/term that a
+// plain text export would silently lose (the documented hole, 04-unhappy §4). We
+// keep the BASE in the body and hand the READING to the caller for a glossary lock
+// (memory v2, шаг 4) — never injecting it into a prompt here (§7d).
+
+// chapterSep splits a TXT source into chapters. Form feed (U+000C) is the ASCII
+// "page/section break": semantically a chapter boundary, invisible in prose, and
+// untouched by NormalizeSource. A txt with no \f is a single chapter (Фаза-0 backward
+// compat: the one-file example stays one chapter). epub carries real chapter
+// structure in its spine, so it does not use this.
+const chapterSep = "\f"
+
+// RubyReading is one captured (base, reading) pair and the chapter it appeared in.
+// The pipeline aggregates these (min chapter, count) before persisting (runner.go).
+type RubyReading struct {
+ Base string // the ruby body: the kanji/base surface form
+ Reading string // the