From b26e4df4652a4ca141d34be4241950e614bc3ec1 Mon Sep 17 00:00:00 2001 From: "Claude (backend session)" Date: Sun, 5 Jul 2026 02:41:57 +0300 Subject: [PATCH] Replace the placeholder chunker with a real sentence-aware source segmenter, txt/epub ingest, and ruby-reading capture persisted to a new v4 table --- backend/internal/pipeline/chunker.go | 421 +++++++++++++++++++--- backend/internal/pipeline/chunker_test.go | 164 +++++++-- backend/internal/pipeline/ingest.go | 389 ++++++++++++++++++++ backend/internal/pipeline/ingest_test.go | 332 +++++++++++++++++ backend/internal/pipeline/render.go | 19 +- backend/internal/pipeline/runner.go | 72 +++- backend/internal/pipeline/runner_test.go | 82 ++++- backend/internal/store/migrate.go | 27 ++ backend/internal/store/ruby.go | 67 ++++ backend/internal/store/ruby_test.go | 87 +++++ docs/PROGRESS.md | 21 ++ 11 files changed, 1582 insertions(+), 99 deletions(-) create mode 100644 backend/internal/pipeline/ingest.go create mode 100644 backend/internal/pipeline/ingest_test.go create mode 100644 backend/internal/store/ruby.go create mode 100644 backend/internal/store/ruby_test.go diff --git a/backend/internal/pipeline/chunker.go b/backend/internal/pipeline/chunker.go index 783f3e7..abe0381 100644 --- a/backend/internal/pipeline/chunker.go +++ b/backend/internal/pipeline/chunker.go @@ -1,18 +1,32 @@ package pipeline -import "strings" +import ( + "strings" + "unicode" +) -// chunker.go: the Веха-2 segmenter SEAM. Its only job right now is to turn a -// normalized source into an ordered list of (chapter, chunk) units so the runner -// loop, resume and disposition machinery can be exercised over MANY chunks — the -// hardcoded "chapter=1/chunk=0" of Фаза 0 is gone. +// chunker.go: the шаг-3a SOURCE segmenter. It turns the per-chapter normalized +// text produced by ingest.go into an ordered list of (chapter, chunk) units for +// the runner loop. It replaced the Веха-2 placeholder (chapter split on \f + +// blank-line paragraph packing to a CHAR budget) with real segmentation: pack +// whole paragraphs to a ~1–2k-TOKEN budget, and when a single paragraph exceeds +// the budget descend to SENTENCE boundaries — never splitting a sentence. // -// This is a deliberate PLACEHOLDER. The real linguistic chunker of шаг 3 -// (sentence segmentation, read-only overlap, ruby name-reading extraction for -// ja) replaces SplitChunks WITHOUT touching the loop — it only has to keep the -// Chunk contract and bump chunkerVersion. chunkerVersion is folded into the -// snapshot (render.go), so changing the segmentation is an explicit -// re-translation (a loud --resnapshot), never a silent cache miss (R6/D5.2). +// Contract kept intact for the loop (runner.go depends on it): SplitChunks emits +// Chunk{Chapter,ChunkIdx,Text}; ChunkIdx resets to 0 per chapter; the function is +// a PURE, deterministic function of its input (no time/rand, no map iteration). +// Its behavior is versioned by chunkerVersion (render.go), folded into the job +// snapshot — so changing the segmentation is a loud --resnapshot re-translation, +// never a silent cache miss (§3.4/R6/D5.2). The chunk target is a CONST covered by +// that version (Р2 = code); were it ever made config-tunable it MUST be folded into +// the top-level snapshot like coverageSnap (§7d), not left to the version string. +// +// IMPORTANT: this sentence splitter is a SEPARATE, independent piece of logic from +// coverage.go's splitSentences. That one is a bit-for-bit port of the Python oracle +// (refusal_bench.py) over the RUSSIAN OUTPUT and is Python-locked (D12 Q1). THIS one +// segments the SOURCE (zh/ja/en), is versioned by chunkerVersion (Р2 = code), and is +// free to be more faithful (closing-bracket absorption, an abbreviation guard). Do +// not couple them. // Chunk is one ordered unit of translation work. type Chunk struct { @@ -21,68 +35,357 @@ type Chunk struct { Text string } -// chapterSep splits the source into chapters. Form feed (U+000C) is the ASCII -// "page/section break" — semantically a chapter boundary, invisible in prose, -// and untouched by NormalizeSource (BOM/CRLF/NFC/outer-trim leave an internal -// \f intact). A source with no \f is a single chapter (Фаза-0 backward compat: -// the one-file example stays one chapter). -const chapterSep = "\f" +// targetChunkTokens is the packing budget: paragraphs (or, inside an oversize +// paragraph, sentences) are greedily packed until the WHOLE chunk's estimated +// tokens (EstimateTokens' formula, applied once — see appendChapterChunks) would +// exceed this. ~1500 sits in the plan's "1–2k токенов по границам абзацев" corridor +// (02-mvp Фаза 1); a chunk of ordinary prose lands well above the coverage gate's +// 500-char floor, so the excision gate is not silently disabled (§7b). A chunk can +// still fall under the floor only when a whole chapter (or its tail) is genuinely +// that short — a documented boundary. A const, not config: the size is a code +// segmentation rule (Р2), covered by chunkerVersion. +const targetChunkTokens = 1500 -// targetChunkChars is the placeholder chunk size (≈ the plan's "1–2k tokens"; -// for a CJK source ~1 char ≈ 1 token). Paragraphs are packed up to this size and -// never split, so the 264-char example remains a single chunk. -const targetChunkChars = 1500 - -// SplitChunks segments a NORMALIZED source (post-NormalizeSource) into an ordered -// chunk list. Rule: split on \f into 1-based chapters; within a chapter pack -// blank-line paragraphs greedily into chunks of ≤ targetChunkChars, never -// splitting a paragraph (a paragraph larger than the target is its own chunk); -// drop empty pieces. Fully deterministic. -func SplitChunks(source string) []Chunk { +// SplitChunks segments the ordered, per-chapter NORMALIZED text of a Document +// (ingest.go already applied NormalizeSource and split chapters) into an ordered +// chunk list. Chapters that yield no text (blank/whitespace-only, e.g. an epub +// cover/nav page) do NOT consume a chapter number, so numbering stays dense over +// real content (Фаза-0 backward compat). Fully deterministic. +func SplitChunks(chapters []string) []Chunk { var out []Chunk chapterNo := 0 - for _, chapRaw := range strings.Split(source, chapterSep) { - paras := splitParagraphs(chapRaw) + for _, chapText := range chapters { + paras := splitParagraphs(chapText) if len(paras) == 0 { - continue // an empty chapter block does not consume a chapter number + continue // an empty chapter does not consume a chapter number } chapterNo++ - chunkIdx := 0 - var buf strings.Builder - flush := func() { - if buf.Len() == 0 { - return - } - out = append(out, Chunk{Chapter: chapterNo, ChunkIdx: chunkIdx, Text: buf.String()}) - chunkIdx++ - buf.Reset() - } - for _, p := range paras { - // Start a new chunk if adding this paragraph would exceed the target - // and the current chunk is non-empty (never split a paragraph). - if buf.Len() > 0 && buf.Len()+len("\n\n")+len(p) > targetChunkChars { - flush() - } - if buf.Len() > 0 { - buf.WriteString("\n\n") - } - buf.WriteString(p) - } - flush() + out = appendChapterChunks(out, chapterNo, paras) } return out } -// splitParagraphs breaks a chapter block on blank lines and trims each paragraph, -// dropping empties. Deterministic; whitespace-only paragraphs vanish. -func splitParagraphs(block string) []string { +// appendChapterChunks packs one chapter's paragraphs into chunks and appends them. +// Rule: pack whole paragraphs while the running token estimate stays within +// targetChunkTokens (prefer paragraph boundaries); a paragraph that alone exceeds +// the budget is flushed on its own and then SENTENCE-packed (never splitting a +// sentence; a single sentence over budget is its own chunk). +// +// The running budget is tracked as ADDITIVE char-class counts (CJK vs other), +// applying EstimateTokens' formula (and its 16-token floor) ONCE to the whole +// chunk — NOT by summing per-unit EstimateTokens. Summing per-unit floors every +// tiny paragraph up to 16 tokens, so a dialogue chapter of many short lines would +// flush a chunk at ~⅓ its true token content, needlessly dropping real content +// under the coverage gate's 500-char floor (self-review finding). Counting classes +// and flooring once reproduces EstimateTokens(joined text) exactly (the "\n\n" +// separators are whitespace, which EstimateTokens ignores). +func appendChapterChunks(out []Chunk, chapterNo int, paras []string) []Chunk { + chunkIdx := 0 + var buf []string // paragraphs accumulated for the current chunk + var bufCJK, bufOther int + flush := func() { + if len(buf) == 0 { + return + } + if text := strings.TrimSpace(strings.Join(buf, "\n\n")); text != "" { + out = append(out, Chunk{Chapter: chapterNo, ChunkIdx: chunkIdx, Text: text}) + chunkIdx++ + } + buf = buf[:0] + bufCJK, bufOther = 0, 0 + } + for _, p := range paras { + pCJK, pOther := tokenClassCounts(p) + if estTokensFrom(pCJK, pOther) > targetChunkTokens { + // Oversize paragraph: close any open chunk at this paragraph boundary, + // then split the paragraph at sentence boundaries into its own chunks. + flush() + for _, sub := range packSentences(p) { + if text := strings.TrimSpace(sub); text != "" { + out = append(out, Chunk{Chapter: chapterNo, ChunkIdx: chunkIdx, Text: text}) + chunkIdx++ + } + } + continue + } + // Normal paragraph: start a new chunk if adding it would exceed the budget + // and the current chunk is non-empty (never split a paragraph here). + if len(buf) > 0 && estTokensFrom(bufCJK+pCJK, bufOther+pOther) > targetChunkTokens { + flush() + } + buf = append(buf, p) + bufCJK, bufOther = bufCJK+pCJK, bufOther+pOther + } + flush() + return out +} + +// packSentences splits one oversize paragraph into sentence segments (lossless +// tiling — concatenation reproduces the paragraph) and packs consecutive segments +// into ≤ targetChunkTokens sub-chunks, never splitting a sentence. A single +// sentence larger than the budget becomes its own (oversize) sub-chunk — the only +// way to honor "never split a sentence". Returns the sub-chunk texts in order. Uses +// the same additive char-class budget as appendChapterChunks (floor applied once). +func packSentences(paragraph string) []string { + segs := splitSourceSentences(paragraph) + var chunks []string + var buf strings.Builder + var bufCJK, bufOther int + flush := func() { + if buf.Len() == 0 { + return + } + chunks = append(chunks, buf.String()) + buf.Reset() + bufCJK, bufOther = 0, 0 + } + for _, s := range segs { + sCJK, sOther := tokenClassCounts(s) + if buf.Len() > 0 && estTokensFrom(bufCJK+sCJK, bufOther+sOther) > targetChunkTokens { + flush() + } + buf.WriteString(s) + bufCJK, bufOther = bufCJK+sCJK, bufOther+sOther + } + flush() + return chunks +} + +// tokenClassCounts counts the two token-estimate character classes of EstimateTokens +// (render.go): CJK ideographs/kana/hangul, and "other" non-space runes (whitespace +// folds into neighbours). Kept separate from EstimateTokens so the packer can sum +// counts ADDITIVELY across units and floor once at the chunk level. Mirrors +// EstimateTokens' classes; a recalibration there that changes packing granularity +// is itself a chunkerVersion bump (both are behaviour changes covered by the snapshot). +func tokenClassCounts(s string) (cjk, other int) { + for _, r := range s { + switch { + case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul): + cjk++ + case unicode.IsSpace(r): + default: + other++ + } + } + return cjk, other +} + +// estTokensFrom applies EstimateTokens' formula (cjk + other/3, floored at 16) to +// pre-counted class totals — identical to EstimateTokens over the joined text. +func estTokensFrom(cjk, other int) int { + est := cjk + other/3 + if est < 16 { + est = 16 + } + return est +} + +// splitParagraphs breaks a chapter into paragraphs on blank lines and trims each, +// dropping empties. Deterministic; whitespace-only paragraphs vanish. (Ingest emits +// a blank line between block-level epub elements, so paragraphs survive extraction.) +func splitParagraphs(chapter string) []string { var paras []string - for _, raw := range strings.Split(block, "\n\n") { - // Collapse a run of blank lines: a "paragraph" that is itself only blank - // lines (from 3+ consecutive newlines) trims to "". + for _, raw := range strings.Split(chapter, "\n\n") { if p := strings.TrimSpace(raw); p != "" { paras = append(paras, p) } } return paras } + +// --- source sentence segmentation (zh/ja/en) — chunkerVersion-versioned --------- + +// splitSourceSentences segments one paragraph at sentence boundaries, returning +// substrings that TILE the input (their concatenation is exactly the paragraph — +// no character is lost or moved), so a packed sub-chunk is an exact source slice. +// A boundary is a run of sentence terminators, plus any closing quotes/brackets, at: +// +// (1) a CJK terminator (。！？) — zero-width, CJK prose has no inter-sentence space — +// but ONLY at quote-depth 0. A terminator inside a quote is dialogue-internal: +// 「止まれ！」と言った。 is ONE sentence ending at the outer 。, not two (the same +// for zh “…！”）— this is the dialogue case §3.7/D12-Q1 defers for the coverage +// oracle, but our source splitter is free to get it right; +// (2) an ASCII terminator (.!?) or ellipsis (…) — only when followed by whitespace +// or end-of-paragraph (so a decimal/URL dot mid-token does not split, and en +// dialogue "Stop!" She ran. still splits on the space after the quote). +// +// Trailing whitespace after a boundary is attached to the LEFT segment (the next +// segment starts at the next content char), keeping the tiling exact. An +// abbreviation guard suppresses a split after a lone "." following a known +// abbreviation or a single-letter initial (never split "Mr. Smith" / "J. R. R."). +// When unsure it prefers NOT to split (over-merge is safe — units just pack +// together; over-split cuts a sentence, the forbidden case). Unbalanced quotes are +// tolerated (depth floors at 0), erring toward merge. +func splitSourceSentences(paragraph string) []string { + runes := []rune(paragraph) + var segs []string + start, i, depth := 0, 0, 0 + for i < len(runes) { + r := runes[i] + if isOpenQuote(r) { + depth++ + i++ + continue + } + if isCloseQuote(r) { + if depth > 0 { + depth-- + } + i++ + continue + } + if !isSourceTerminator(r) { + i++ + continue + } + // Consume a run of terminators (collapse "。。。" / "!?" into one boundary). + j := i + cjk := false + for j < len(runes) && isSourceTerminator(runes[j]) { + if isCJKTerminator(runes[j]) { + cjk = true + } + j++ + } + // The terminator's quote-depth is the depth HERE — before absorbing the + // closing quote that may follow it (「…！」: the ！ is at depth 1, the 」 that + // closes the quote comes after and must not retro-promote it to depth 0). + depthHere := depth + // Absorb a run of closing quotes/brackets that belong to the sentence, + // decrementing quote-depth for the depth-tracked closers among them. + e := j + for e < len(runes) && isClosingBracket(runes[e]) { + if isCloseQuote(runes[e]) && depth > 0 { + depth-- + } + e++ + } + boundary := false + if cjk { + boundary = depthHere == 0 // dialogue-internal terminators do not end a sentence + } else { + // ASCII terminator / ellipsis: a boundary only before whitespace or EOS. + boundary = e >= len(runes) || isASCIISpace(runes[e]) + // Abbreviation guard: a lone "." after an abbreviation/initial is not a end. + if boundary && j-i == 1 && runes[i] == '.' && isAbbrevBefore(runes[start:i]) { + boundary = false + } + } + if !boundary { + i = e + continue + } + // Attach the following whitespace run to the left segment (keeps tiling exact). + w := e + for w < len(runes) && isASCIISpace(runes[w]) { + w++ + } + segs = append(segs, string(runes[start:w])) + start, i = w, w + } + if start < len(runes) { + segs = append(segs, string(runes[start:])) + } + return segs +} + +// isOpenQuote / isCloseQuote are the DEPTH-TRACKED paired quote/bracket delimiters +// (CJK brackets + curly quotes with distinct open/close glyphs). Straight quotes +// (" ') are NOT tracked — same glyph opens and closes, so depth is undecidable; en +// relies on the whitespace rule instead, which is correct for spaced prose. +func isOpenQuote(r rune) bool { + switch r { + case '「', '『', '（', '【', '《', '〈', '［', '｛', '“', '‘': + return true + } + return false +} + +func isCloseQuote(r rune) bool { + switch r { + case '」', '』', '）', '】', '》', '〉', '］', '｝', '”', '’': + return true + } + return false +} + +// isSourceTerminator is the sentence-terminator class for the SOURCE: CJK 。！？, +// ASCII .!?, and the ellipsis … (U+2026). +func isSourceTerminator(r rune) bool { + switch r { + case '。', '！', '？', '.', '!', '?', '…': + return true + } + return false +} + +// isCJKTerminator is the zero-width-splitting subset (fullwidth CJK). The ellipsis +// is deliberately NOT here: "……" mid-CJK-sentence must not split, but "……。" still +// splits on the 。 in the same run. +func isCJKTerminator(r rune) bool { + switch r { + case '。', '！', '？': + return true + } + return false +} + +// isClosingBracket is the closing-quote/bracket class absorbed into the left +// sentence after its terminator (ja/zh brackets + straight/curly quotes). +func isClosingBracket(r rune) bool { + switch r { + case '」', '』', '】', '）', ')', '》', '〉', '］', ']', '｝', '}', + '"', '\'', '”', '’': + return true + } + return false +} + +// isASCIISpace is the whitespace used for ASCII-terminator boundary detection and +// tiling — the ordinary prose separators (space, tab, newline, CR, form feed, +// vertical tab, and the ideographic space U+3000). +func isASCIISpace(r rune) bool { + switch r { + case ' ', '\t', '\n', '\r', '\f', '\v', '　': + return true + } + return false +} + +// sourceAbbrevs are trailing tokens after which a lone "." is treated as an +// abbreviation, not a sentence end (case-insensitive). A pragmatic set for prose; +// the acceptance path is ja→ru where CJK terminators dominate, so this only guards +// the en source. Single-letter initials are handled separately (isAbbrevBefore). +var sourceAbbrevs = map[string]bool{ + "mr": true, "mrs": true, "ms": true, "dr": true, "prof": true, "st": true, + "jr": true, "sr": true, "vs": true, "no": true, "vol": true, "ch": true, + "fig": true, "col": true, "gen": true, "sgt": true, "capt": true, "lt": true, + "rev": true, "gov": true, "sen": true, "rep": true, "etc": true, "inc": true, + "ltd": true, "co": true, "mt": true, "ave": true, "rd": true, +} + +// isAbbrevBefore reports whether the text immediately before a lone "." ends in a +// known abbreviation or a single-letter initial (so the "." is not a boundary). +func isAbbrevBefore(left []rune) bool { + // Trailing run of ASCII letters (the "word" before the period). + end := len(left) + k := end + for k > 0 && isASCIILetter(left[k-1]) { + k-- + } + word := left[k:end] + if len(word) == 0 { + return false + } + // Single-letter initial ("J.") — only when it stands alone (start or preceded by + // a space/opening punctuation), never a one-letter word ending a real sentence + // mid-flow would be rare; over-merge is the safe side. + if len(word) == 1 { + return k == 0 || !isASCIILetter(left[k-1]) + } + return sourceAbbrevs[strings.ToLower(string(word))] +} + +func isASCIILetter(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') +} diff --git a/backend/internal/pipeline/chunker_test.go b/backend/internal/pipeline/chunker_test.go index 3a2ec0a..c011697 100644 --- a/backend/internal/pipeline/chunker_test.go +++ b/backend/internal/pipeline/chunker_test.go @@ -7,7 +7,7 @@ import ( ) func TestSplitChunksSingleParagraph(t *testing.T) { - got := SplitChunks("静かな図書館の朝。") + got := SplitChunks([]string{"静かな図書館の朝。"}) want := []Chunk{{Chapter: 1, ChunkIdx: 0, Text: "静かな図書館の朝。"}} if !reflect.DeepEqual(got, want) { t.Fatalf("single paragraph = %+v, want %+v", got, want) @@ -15,8 +15,7 @@ func TestSplitChunksSingleParagraph(t *testing.T) { } func TestSplitChunksChapters(t *testing.T) { - src := "Глава один." + chapterSep + "Глава два." + chapterSep + "Глава три." - got := SplitChunks(src) + got := SplitChunks([]string{"Глава один.", "Глава два.", "Глава три."}) want := []Chunk{ {Chapter: 1, ChunkIdx: 0, Text: "Глава один."}, {Chapter: 2, ChunkIdx: 0, Text: "Глава два."}, @@ -28,23 +27,22 @@ func TestSplitChunksChapters(t *testing.T) { } func TestSplitChunksPacksToTarget(t *testing.T) { - // Two paragraphs that together exceed the target must split into two chunks, - // each never splitting a paragraph. - p1 := strings.Repeat("a", 800) - p2 := strings.Repeat("b", 800) - got := SplitChunks(p1 + "\n\n" + p2) + // Two paragraphs that together exceed the TOKEN target (each ~1000 tokens for + // cyrillic: 3000 chars / 3) must split into two chunks, each kept whole. + p1 := strings.Repeat("а", 3000) + p2 := strings.Repeat("б", 3000) + got := SplitChunks([]string{p1 + "\n\n" + p2}) want := []Chunk{ {Chapter: 1, ChunkIdx: 0, Text: p1}, {Chapter: 1, ChunkIdx: 1, Text: p2}, } if !reflect.DeepEqual(got, want) { - t.Fatalf("packing = chapters/chunks %d, want 2 chunks; got %+v", len(got), summarize(got)) + t.Fatalf("packing = %d chunks, want 2 whole paragraphs; got %+v", len(got), summarize(got)) } } func TestSplitChunksPacksSmallParagraphsTogether(t *testing.T) { - // Small paragraphs pack into one chunk (under the target), joined by a blank line. - got := SplitChunks("Абзац один.\n\nАбзац два.\n\nАбзац три.") + got := SplitChunks([]string{"Абзац один.\n\nАбзац два.\n\nАбзац три."}) if len(got) != 1 { t.Fatalf("small paragraphs must pack into one chunk, got %d: %+v", len(got), summarize(got)) } @@ -53,19 +51,62 @@ func TestSplitChunksPacksSmallParagraphsTogether(t *testing.T) { } } -func TestSplitChunksOversizeParagraphIsOwnChunk(t *testing.T) { - big := strings.Repeat("ы", 2000) // > targetChunkChars but never split - got := SplitChunks(big) +func TestSplitChunksOversizeParagraphSplitsAtSentences(t *testing.T) { + // One paragraph far over the token target, built of many whole sentences, must + // split into >1 chunk AND never cut a sentence: every original sentence must + // survive intact and in order across the chunks. + sentence := strings.Repeat("слово ", 40) + "конец." // ~80 tokens each + var sb strings.Builder + for i := 0; i < 40; i++ { // ~3200 tokens total ≫ 1500 + sb.WriteString(sentence) + sb.WriteByte(' ') + } + para := strings.TrimSpace(sb.String()) + got := SplitChunks([]string{para}) + if len(got) < 2 { + t.Fatalf("oversize paragraph must split into >1 chunk, got %d", len(got)) + } + wantSentences := trimAll(splitSourceSentences(para)) + var gotSentences []string + for _, c := range got { + gotSentences = append(gotSentences, trimAll(splitSourceSentences(c.Text))...) + } + if !reflect.DeepEqual(gotSentences, wantSentences) { + t.Fatalf("sentence integrity broken:\n got %d sentences\nwant %d sentences", len(gotSentences), len(wantSentences)) + } +} + +func TestSplitChunksShortLinesPackByTrueTokenBudget(t *testing.T) { + // Many short dialogue paragraphs must pack by TRUE token content, not by summing + // the per-unit 16-token floor. 150 lines of 「はい。」 are ~450 true tokens (cjk=300, + // other=450 → 300+150), so they belong in ONE chunk. Summing the floor (150×16) + // would wrongly flush into ~2 sub-500-char chunks the coverage gate then skips. + var paras []string + for i := 0; i < 150; i++ { + paras = append(paras, "「はい。」") + } + got := SplitChunks([]string{strings.Join(paras, "\n\n")}) + if len(got) != 1 { + t.Fatalf("short lines must pack by true token budget into 1 chunk, got %d chunks", len(got)) + } + if EstimateTokens(got[0].Text) > targetChunkTokens { + t.Fatalf("packed chunk overshoots the budget: %d tokens", EstimateTokens(got[0].Text)) + } +} + +func TestSplitChunksOversizeSentenceIsOwnChunk(t *testing.T) { + // A single sentence (no internal terminator) larger than the budget can only be + // its own chunk — never split. + big := strings.Repeat("ы", 6000) // ~2000 tokens, one un-terminated run + got := SplitChunks([]string{big}) if len(got) != 1 || got[0].Text != big { - t.Fatalf("an oversize paragraph must be one un-split chunk, got %+v", summarize(got)) + t.Fatalf("an oversize sentence must be one un-split chunk, got %+v", summarize(got)) } } func TestSplitChunksDropsEmpties(t *testing.T) { - // Empty chapter blocks and whitespace-only paragraphs vanish; chapter numbers - // only advance for non-empty chapters. - src := "A" + chapterSep + " \n\n " + chapterSep + "B" - got := SplitChunks(src) + // A blank/whitespace-only chapter vanishes and does NOT consume a chapter number. + got := SplitChunks([]string{"A", " \n\n ", "B"}) want := []Chunk{ {Chapter: 1, ChunkIdx: 0, Text: "A"}, {Chapter: 2, ChunkIdx: 0, Text: "B"}, @@ -76,13 +117,94 @@ func TestSplitChunksDropsEmpties(t *testing.T) { } func TestSplitChunksDeterministic(t *testing.T) { - src := "Абзац.\n\nЕщё абзац." + chapterSep + strings.Repeat("длинный ", 300) - a, b := SplitChunks(src), SplitChunks(src) + chapters := []string{ + "Абзац.\n\nЕщё абзац.", + strings.Repeat("слово ", 400) + "конец.", + } + a, b := SplitChunks(chapters), SplitChunks(chapters) if !reflect.DeepEqual(a, b) { t.Fatal("SplitChunks must be deterministic") } } +func TestSplitChunksNeverSplitsAbbreviation(t *testing.T) { + // Two long en paragraphs each ending mid-flow with "Mr. Smith" must not chunk + // between "Mr." and "Smith" — the abbreviation guard keeps the sentence whole. + p := strings.Repeat("The road went on and on. ", 100) + "It was Mr. Smith who arrived." + got := SplitChunks([]string{p}) + for _, c := range got { + if strings.HasSuffix(strings.TrimSpace(c.Text), "Mr.") { + t.Fatalf("chunk ended at an abbreviation 'Mr.' — a sentence was cut: %q", c.Text) + } + } +} + +// --- source sentence splitter (zh/ja/en) --------------------------------------- + +func TestSplitSourceSentencesTilesLosslessly(t *testing.T) { + // The segments must TILE the input: concatenation reproduces it byte-for-byte. + inputs := []string{ + "静かな朝。彼は歩いた。「止まれ！」と叫んだ。", + "他说。“不要！”然后离开了。", + "Hello world. This is a test! Really? Yes.", + "Mr. Smith met Dr. J. R. R. Brown. They talked.", + "末尾に句点なし", + "", + } + for _, in := range inputs { + segs := splitSourceSentences(in) + if got := strings.Join(segs, ""); got != in { + t.Fatalf("tiling broke for %q: rejoined to %q", in, got) + } + } +} + +func TestSplitSourceSentencesJapanese(t *testing.T) { + // CJK terminators split zero-width; a closing quote is absorbed into its sentence. + got := trimAll(splitSourceSentences("静かな朝。「止まれ！」と彼は言った。")) + want := []string{"静かな朝。", "「止まれ！」と彼は言った。"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ja split = %q, want %q", got, want) + } +} + +func TestSplitSourceSentencesEnglish(t *testing.T) { + got := trimAll(splitSourceSentences(`He said "Stop!" She ran. Done.`)) + want := []string{`He said "Stop!"`, "She ran.", "Done."} + if !reflect.DeepEqual(got, want) { + t.Fatalf("en split = %q, want %q", got, want) + } +} + +func TestSplitSourceSentencesEllipsisNotOverSplitInCJK(t *testing.T) { + // "……" mid-CJK-sentence must not split; the trailing 。 still ends it. + got := trimAll(splitSourceSentences("そう……だった。次へ。")) + want := []string{"そう……だった。", "次へ。"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ellipsis split = %q, want %q", got, want) + } +} + +func TestSplitSourceSentencesAbbreviations(t *testing.T) { + // Abbreviations and single-letter initials do not end a sentence. + got := trimAll(splitSourceSentences("Dr. J. R. R. Tolkien wrote it. The end.")) + want := []string{"Dr. J. R. R. Tolkien wrote it.", "The end."} + if !reflect.DeepEqual(got, want) { + t.Fatalf("abbrev split = %q, want %q", got, want) + } +} + +// trimAll TrimSpaces each segment and drops empties (test convenience). +func trimAll(segs []string) []string { + var out []string + for _, s := range segs { + if t := strings.TrimSpace(s); t != "" { + out = append(out, t) + } + } + return out +} + func summarize(cs []Chunk) []string { out := make([]string, len(cs)) for i, c := range cs { diff --git a/backend/internal/pipeline/ingest.go b/backend/internal/pipeline/ingest.go new file mode 100644 index 0000000..03d6bfd --- /dev/null +++ b/backend/internal/pipeline/ingest.go @@ -0,0 +1,389 @@ +package pipeline + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "net/url" + "os" + "path" + "strings" +) + +// ingest.go: the шаг-3a import layer. Ingest turns a source file (txt or epub) +// into an ordered list of per-chapter NORMALIZED text plus the ruby/furigana +// readings captured on the way (04-unhappy §4 / D9). It is the ONLY place that +// reads the source; the runner then feeds doc.Chapters to SplitChunks and persists +// doc.Ruby (runner.go). It is offline and deterministic — no LLM, no time/rand — so +// the whole path is $0 and reproducible. +// +// epub v1 is a text extraction (02-mvp:25): read the chapters in spine order, +// strip tags, drop inline markup/styling. The ONE thing we do NOT drop is ruby: +// basereading carries an author reading of a name/term that a +// plain text export would silently lose (the documented hole, 04-unhappy §4). We +// keep the BASE in the body and hand the READING to the caller for a glossary lock +// (memory v2, шаг 4) — never injecting it into a prompt here (§7d). + +// chapterSep splits a TXT source into chapters. Form feed (U+000C) is the ASCII +// "page/section break": semantically a chapter boundary, invisible in prose, and +// untouched by NormalizeSource. A txt with no \f is a single chapter (Фаза-0 backward +// compat: the one-file example stays one chapter). epub carries real chapter +// structure in its spine, so it does not use this. +const chapterSep = "\f" + +// RubyReading is one captured (base, reading) pair and the chapter it appeared in. +// The pipeline aggregates these (min chapter, count) before persisting (runner.go). +type RubyReading struct { + Base string // the ruby body: the kanji/base surface form + Reading string // the reading (furigana) + Chapter int // 1-based chapter where this occurrence was found +} + +// Document is the ingested source: per-chapter normalized text (in reading order) +// plus every ruby occurrence found. Chapters is what SplitChunks consumes. +type Document struct { + Chapters []string + Ruby []RubyReading +} + +// Ingest reads a source file and returns its Document. Dispatch is by extension: +// .epub → the epub reader; anything else → plain text (the example uses .txt; an +// unknown extension is treated as text rather than rejected). +func Ingest(p string) (*Document, error) { + if strings.EqualFold(extOf(p), ".epub") { + return ingestEPUB(p) + } + return ingestTXT(p) +} + +// extOf returns the lower-cased file extension incl. the dot ("" if none). +func extOf(p string) string { + i := strings.LastIndexByte(p, '.') + if i < 0 || strings.ContainsAny(p[i:], "/\\") { + return "" + } + return strings.ToLower(p[i:]) +} + +// ingestTXT reads a plain-text source: split on \f into chapters, NormalizeSource +// each (BOM/CRLF/NFC/outer-trim). No ruby in txt. +func ingestTXT(p string) (*Document, error) { + raw, err := os.ReadFile(p) + if err != nil { + return nil, fmt.Errorf("pipeline: ingest txt: %w", err) + } + var chapters []string + for _, part := range strings.Split(string(raw), chapterSep) { + chapters = append(chapters, NormalizeSource(part)) + } + return &Document{Chapters: chapters}, nil +} + +// --- epub ---------------------------------------------------------------------- + +// containerXML is META-INF/container.xml: it points at the OPF package file. Tags +// are matched by LOCAL name (no namespace in the struct tags), so a namespaced or +// prefixed container still binds. +type containerXML struct { + Rootfiles []struct { + FullPath string `xml:"full-path,attr"` + MediaType string `xml:"media-type,attr"` + } `xml:"rootfiles>rootfile"` +} + +// opfPackage is the OPF: the manifest (id→href→media-type) and the spine (the +// reading order of itemref idrefs). +type opfPackage struct { + Manifest []struct { + ID string `xml:"id,attr"` + Href string `xml:"href,attr"` + MediaType string `xml:"media-type,attr"` + } `xml:"manifest>item"` + Spine []struct { + IDRef string `xml:"idref,attr"` + } `xml:"spine>itemref"` +} + +// ingestEPUB reads chapters in spine order and captures ruby. Every structural +// problem (bad zip, missing container/opf, empty spine, no readable chapter) is a +// loud error, never a panic (a truncated/foreign epub must not crash a run). +func ingestEPUB(p string) (*Document, error) { + zr, err := zip.OpenReader(p) + if err != nil { + return nil, fmt.Errorf("pipeline: ingest epub: open zip %s: %w", p, err) + } + defer zr.Close() + + files := map[string]*zip.File{} + for _, f := range zr.File { + files[path.Clean(f.Name)] = f + } + + // 1) container.xml → OPF path. + cdata, err := readZipEntry(files, "META-INF/container.xml") + if err != nil { + return nil, fmt.Errorf("pipeline: ingest epub: %w", err) + } + var container containerXML + if err := xml.Unmarshal(cdata, &container); err != nil { + return nil, fmt.Errorf("pipeline: ingest epub: parse container.xml: %w", err) + } + opfPath := "" + for _, rf := range container.Rootfiles { + if strings.TrimSpace(rf.FullPath) != "" { + opfPath = path.Clean(rf.FullPath) + break + } + } + if opfPath == "" { + return nil, fmt.Errorf("pipeline: ingest epub: container.xml has no rootfile full-path") + } + + // 2) OPF → manifest + spine. + odata, err := readZipEntry(files, opfPath) + if err != nil { + return nil, fmt.Errorf("pipeline: ingest epub: %w", err) + } + var opf opfPackage + if err := xml.Unmarshal(odata, &opf); err != nil { + return nil, fmt.Errorf("pipeline: ingest epub: parse opf %s: %w", opfPath, err) + } + if len(opf.Spine) == 0 { + return nil, fmt.Errorf("pipeline: ingest epub: opf %s has an empty spine", opfPath) + } + byID := map[string]struct{ href, mediaType string }{} + for _, it := range opf.Manifest { + byID[it.ID] = struct{ href, mediaType string }{it.Href, it.MediaType} + } + opfDir := path.Dir(opfPath) + + // 3) Read each spine document in order (= one chapter). A dangling idref or a + // non-(x)html item is skipped, not fatal; a spine that yields zero readable + // content documents is a loud error. + doc := &Document{} + denseNo := 0 // matches SplitChunks: only a NON-empty chapter takes a number + read := 0 + for _, ref := range opf.Spine { + item, ok := byID[ref.IDRef] + if !ok || !isXHTML(item.mediaType, item.href) { + continue + } + entry := resolveHref(opfDir, item.href) + data, err := readZipEntry(files, entry) + if err != nil { + return nil, fmt.Errorf("pipeline: ingest epub: spine item %q → %s: %w", ref.IDRef, entry, err) + } + read++ + text, ruby, err := extractXHTML(data) + if err != nil { + return nil, fmt.Errorf("pipeline: ingest epub: extract %s: %w", entry, err) + } + norm := NormalizeSource(text) + doc.Chapters = append(doc.Chapters, norm) + // The chapter number ruby carries MUST equal the number SplitChunks assigns + // (dense — empty chapters are skipped), otherwise memory v2's since_ch is off + // by the count of empty spine items (cover/nav) before the reading. A + // ruby-bearing chapter is never empty (its base sits in the body), so denseNo + // is well-defined here; empty chapters carry no ruby. + if len(splitParagraphs(norm)) == 0 { + continue + } + denseNo++ + for i := range ruby { + ruby[i].Chapter = denseNo + } + doc.Ruby = append(doc.Ruby, ruby...) + } + if read == 0 { + return nil, fmt.Errorf("pipeline: ingest epub: spine resolved to no readable (x)html chapters") + } + return doc, nil +} + +// isXHTML reports whether a manifest item is an (x)html content document. It +// checks the media-type (parameters like "; charset=utf-8" stripped), then falls +// back to the href extension for ANY unrecognized/absent type — real epubs ship +// xhtml chapters typed application/xml or text/xml, so a content-document extension +// is the reliable signal. A genuine non-content asset keeps its own extension +// (.jpg/.css/.ncx/.svg), so the extension fallback does not misclassify it +// (self-review: a present-but-unrecognized type used to silently drop the chapter). +func isXHTML(mediaType, href string) bool { + mt := strings.ToLower(strings.TrimSpace(mediaType)) + if i := strings.IndexByte(mt, ';'); i >= 0 { + mt = strings.TrimSpace(mt[:i]) + } + switch mt { + case "application/xhtml+xml", "text/html", "application/html": + return true + } + switch extOf(href) { + case ".xhtml", ".html", ".htm": + return true + } + return false +} + +// resolveHref joins an OPF-relative href to the OPF directory (percent-decoded, +// slash-cleaned) into a zip entry path. +func resolveHref(opfDir, href string) string { + if h, err := url.PathUnescape(href); err == nil { + href = h + } + // Drop any fragment (chapter.xhtml#frag). + if i := strings.IndexByte(href, '#'); i >= 0 { + href = href[:i] + } + return path.Clean(path.Join(opfDir, href)) +} + +// readZipEntry reads a zip entry by cleaned path. +func readZipEntry(files map[string]*zip.File, name string) ([]byte, error) { + f, ok := files[path.Clean(name)] + if !ok { + return nil, fmt.Errorf("missing zip entry %q", name) + } + rc, err := f.Open() + if err != nil { + return nil, fmt.Errorf("open zip entry %q: %w", name, err) + } + defer rc.Close() + return io.ReadAll(rc) +} + +// --- xhtml text + ruby extraction ---------------------------------------------- + +// blockTags emit a paragraph break (blank line) so the chunker's paragraph packing +// survives extraction; without them a whole chapter collapses into one paragraph. +var blockTags = map[string]bool{ + "p": true, "div": true, "h1": true, "h2": true, "h3": true, "h4": true, + "h5": true, "h6": true, "li": true, "blockquote": true, "section": true, + "article": true, "tr": true, "table": true, "ul": true, "ol": true, "dl": true, + "dd": true, "dt": true, "pre": true, "figure": true, "figcaption": true, + "header": true, "footer": true, "aside": true, "nav": true, "hr": true, + "main": true, "td": true, "th": true, "caption": true, +} + +// skipRoots are subtrees whose text is not prose (CSS/JS/metadata). +var skipRoots = map[string]bool{"script": true, "style": true, "head": true} + +// extractXHTML strips tags to text and captures ruby. Rules: +// - block tag → blank line;
→ newline (paragraph structure for the chunker); +// - basereading: BASE goes to the body, READING is captured +// as a (base,reading) pair, parenthesis fallbacks are dropped; +// -