206 lines
10 KiB
Go
206 lines
10 KiB
Go
package chunk
|
||
|
||
import (
|
||
"fmt"
|
||
"strings"
|
||
"testing"
|
||
|
||
"textmachine/backend/internal/lang"
|
||
)
|
||
|
||
// zhRuChapterRule mirrors a live zh→ru book after the source/pair split: the CJK source grammar (第 ·
|
||
// 章节節回話, derived from the embedded cjk-section.txt) joined to the pair's «Глава {n}» template. The units
|
||
// are no longer hand-copied here — that copy is exactly what had drifted from ingest's five.
|
||
func zhRuChapterRule() *lang.ChapterRule {
|
||
return &lang.ChapterRule{Structure: lang.DefaultCJKStructure(), Template: "Глава {n}"}
|
||
}
|
||
|
||
// TestStripHeadingRerun2 pins the pack-13 title policy on the LIVE rerun2 header shapes (第N节：subtitle):
|
||
// the marker is stripped from the model input, the subtitle is kept, and the deterministic «Глава N» is
|
||
// rendered — the fix for the cross-arm chaos («Раздел 2» / «Первая глава» / orphaned « :»).
|
||
func TestStripHeadingRerun2(t *testing.T) {
|
||
hr := zhRuChapterRule()
|
||
cases := []struct {
|
||
chapter string
|
||
wantHeading string
|
||
wantStripped string
|
||
}{
|
||
{"第一节：纵身亡魔心仍不悔\n\n正文第一段。", "Глава 1", "纵身亡魔心仍不悔\n\n正文第一段。"},
|
||
{"第二节：逆光阴五百年觉悟\n\n身体。", "Глава 2", "逆光阴五百年觉悟\n\n身体。"},
|
||
{"第四节：古月方源！\n\n夜。", "Глава 4", "古月方源！\n\n夜。"},
|
||
{"第五节：人祖三蛊，希望开窍\n\n春。", "Глава 5", "人祖三蛊，希望开窍\n\n春。"},
|
||
{"第一章 図書館の秘密\n\n本文。", "Глава 1", "図書館の秘密\n\n本文。"}, // fullwidth-space separator (ja-shape)
|
||
{"第十二章：标题\n\n正文。", "Глава 12", "标题\n\n正文。"}, // multi-digit CJK numeral
|
||
{"第1章 Title\n\nbody.", "Глава 1", "Title\n\nbody."}, // Arabic numeral + space
|
||
{"第一节\n\n正文。", "Глава 1", "正文。"}, // header with NO subtitle → whole line drops
|
||
}
|
||
for _, c := range cases {
|
||
gotH, gotS := stripHeading(c.chapter, hr)
|
||
if gotH != c.wantHeading || gotS != c.wantStripped {
|
||
t.Errorf("stripHeading(%q):\n heading = %q, want %q\n stripped = %q, want %q",
|
||
c.chapter, gotH, c.wantHeading, gotS, c.wantStripped)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestStripHeadingGenericLatinMarker pins the П3 generic-header closure (D39.60 §5 E-2): a heading rule with
|
||
// a Latin marker and NO units — marker + space + digits, «Chapter 12» — is detected and rendered
|
||
// deterministically. That class was inexpressible before (matchHeaderLine required a unit rune and tolerated
|
||
// no space). The unit-bearing CJK path is unchanged (TestStripHeadingRerun2 is the regression).
|
||
func TestStripHeadingGenericLatinMarker(t *testing.T) {
|
||
hr := &lang.ChapterRule{Structure: lang.NewSourceStructure("Chapter", ""), Template: "Глава {n}"} // no units → unit optional
|
||
cases := []struct{ chapter, wantHeading, wantStripped string }{
|
||
{"Chapter 12\n\nThe body.", "Глава 12", "The body."},
|
||
{"Chapter 3: The Fall\n\nbody.", "Глава 3", "The Fall\n\nbody."},
|
||
{"Chapter 1 Beginnings\n\nbody.", "Глава 1", "Beginnings\n\nbody."},
|
||
}
|
||
for _, c := range cases {
|
||
gotH, gotS := stripHeading(c.chapter, hr)
|
||
if gotH != c.wantHeading || gotS != c.wantStripped {
|
||
t.Errorf("stripHeading(%q): heading=%q want %q; stripped=%q want %q", c.chapter, gotH, c.wantHeading, gotS, c.wantStripped)
|
||
}
|
||
}
|
||
// A glued ordinal («12th») is not a header; prose merely opening with «Chapter» + content is not either.
|
||
for _, neg := range []string{"Chapter 12th\n\nx.", "Chapters were long.\n\nx."} {
|
||
if h, _ := stripHeading(neg, hr); h != "" {
|
||
t.Errorf("stripHeading(%q) must be a no-op, got heading %q", neg, h)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestChapterUnitJaWebnovel pins the 話 data addition (D39.60 §3.1): a ja .txt whose chapters are 第N話 (the
|
||
// dominant Japanese-webnovel unit) auto-detects 話 and splits, instead of silently becoming ONE chapter.
|
||
func TestChapterUnitJaWebnovel(t *testing.T) {
|
||
lines := []string{"第1話 はじまり", "本文だ。", "第2話 つづき", "本文だ。"}
|
||
if u := detectChapterUnit(lines, lang.DefaultCJKStructure()); u != '話' {
|
||
t.Errorf("detectChapterUnit must pick 話 for a 第N話 webnovel, got %q", u)
|
||
}
|
||
}
|
||
|
||
// TestStripHeadingNegatives asserts the precision guards: a nil rule is inert; a glued measure word is not
|
||
// a header; prose that merely opens with 第 is untouched.
|
||
func TestStripHeadingNegatives(t *testing.T) {
|
||
hr := zhRuChapterRule()
|
||
negatives := []string{
|
||
"第一回见面时，他笑了。\n\n正文。", // 回 measure word glued to content 见 → NOT a header (ingest parity)
|
||
"这是第三节的内容。\n\n正文。", // 第 not at line start
|
||
"普通的一段话。\n\n第二段。", // no marker at all
|
||
"第节：无数字\n\n正文。", // no numeral between marker and unit
|
||
}
|
||
for _, chap := range negatives {
|
||
if gotH, gotS := stripHeading(chap, hr); gotH != "" || gotS != chap {
|
||
t.Errorf("stripHeading(%q) should be a no-op, got heading=%q stripped=%q", chap, gotH, gotS)
|
||
}
|
||
}
|
||
// A nil rule is always a no-op (a book with no heading.txt).
|
||
if gotH, gotS := stripHeading("第一节：标题\n\n正文。", nil); gotH != "" || gotS != "第一节：标题\n\n正文。" {
|
||
t.Errorf("nil rule must be inert, got heading=%q", gotH)
|
||
}
|
||
}
|
||
|
||
// headingFixtureNominals are the numbers the fixture's headers carry. They are NOT consecutive on purpose:
|
||
// a chapter's ORDINAL (its position in the cut, and the key chunk_status is addressed by) and the NUMBER in
|
||
// its title are two different quantities, and a fixture numbered 一·二 makes them equal — after which a
|
||
// title rendered from the counter and a title rendered from the header are indistinguishable, and the whole
|
||
// class of "the title came from the wrong number" is invisible.
|
||
var headingFixtureNominals = []int{1, 3}
|
||
|
||
// headingFixtureChapters returns two chapters whose cut is NON-DEGENERATE for the heading seam: each
|
||
// becomes several draft chunks AND more than one edit unit. Both counts are needed and they are different
|
||
// guarantees — chunks that all land in ONE unit make "the chapter's first chunk" and "an edit unit's first
|
||
// chunk" the same position, so a title moved from one to the other is invisible. Measured under testSeg():
|
||
// a ~900-rune paragraph fills a draft chunk on its own, and the fourth of them closes a second edit unit.
|
||
func headingFixtureChapters() []string {
|
||
body := strings.Repeat("\n\n"+strings.Repeat("古月方源站在洞口。", 100), 4)
|
||
return []string{"第一节：纵身亡魔心仍不悔" + body, "第三节：逆光阴" + body}
|
||
}
|
||
|
||
// headingCutShape reports, per chapter, how many draft chunks and how many distinct edit units a cut
|
||
// produced, and how many of its chunks carry a title.
|
||
func headingCutShape(chunks []Chunk) (chunksPer, unitsPer map[int]int, headings int) {
|
||
chunksPer, unitsPer = map[int]int{}, map[int]int{}
|
||
units := map[int]map[int]bool{}
|
||
for _, c := range chunks {
|
||
chunksPer[c.Chapter]++
|
||
if units[c.Chapter] == nil {
|
||
units[c.Chapter] = map[int]bool{}
|
||
}
|
||
units[c.Chapter][c.EditUnitID] = true
|
||
if c.Heading != "" {
|
||
headings++
|
||
}
|
||
}
|
||
for ch, u := range units {
|
||
unitsPer[ch] = len(u)
|
||
}
|
||
return chunksPer, unitsPer, headings
|
||
}
|
||
|
||
// TestSplitChunksHeadingCarried asserts the deterministic title lands on the chapter's FIRST chunk and on
|
||
// no other chunk, the source marker is stripped from ch.Text, and ApplyHeading prepends it correctly.
|
||
//
|
||
// ⛔ THE FIXTURE IS HALF THE ASSERTION. "A non-first chunk carries no title" says nothing at all about a
|
||
// chapter that produced a single chunk, and only half of what it should about a chapter whose chunks share
|
||
// one edit unit. Both counts are therefore asserted rather than assumed: a budget change that degenerates
|
||
// this cut fails here, instead of quietly emptying the pin underneath it.
|
||
func TestSplitChunksHeadingCarried(t *testing.T) {
|
||
hr := zhRuChapterRule()
|
||
chapters := headingFixtureChapters()
|
||
chunks := SplitChunks(chapters, testSeg(), hr, testAbbrevs())
|
||
|
||
chunksPer, unitsPer, headings := headingCutShape(chunks)
|
||
t.Logf("cut: %d chunks; chunks per chapter %v; edit units per chapter %v; chunks carrying a title %d",
|
||
len(chunks), chunksPer, unitsPer, headings)
|
||
for ch := 1; ch <= len(chapters); ch++ {
|
||
if chunksPer[ch] < 2 || unitsPer[ch] < 2 {
|
||
t.Fatalf("degenerate fixture: chapter %d is %d chunks in %d edit units, want ≥2 of each",
|
||
ch, chunksPer[ch], unitsPer[ch])
|
||
}
|
||
}
|
||
if headings != len(chapters) {
|
||
t.Fatalf("%d chunks carry a title, want %d — one per chapter", headings, len(chapters))
|
||
}
|
||
|
||
// The whole seam in one sweep: the title is on chunk 0 of its chapter and nowhere else. The edit unit
|
||
// is named in the message because the mutation this catches moves the title to a UNIT boundary.
|
||
for _, c := range chunks {
|
||
want := ""
|
||
if c.ChunkIdx == 0 {
|
||
if c.Chapter < 1 || c.Chapter > len(headingFixtureNominals) {
|
||
t.Fatalf("the cut produced chapter %d and the fixture declares %d headers — the chapter counter no longer indexes them", c.Chapter, len(headingFixtureNominals))
|
||
}
|
||
want = fmt.Sprintf("Глава %d", headingFixtureNominals[c.Chapter-1])
|
||
}
|
||
if c.Heading != want {
|
||
t.Errorf("chapter %d chunk %d (edit unit %d) heading = %q, want %q — only a chapter's FIRST chunk carries the title",
|
||
c.Chapter, c.ChunkIdx, c.EditUnitID, c.Heading, want)
|
||
}
|
||
}
|
||
if containsRune(chunks[0].Text, '第') {
|
||
t.Errorf("chunk0 text still carries the source marker: %q", chunks[0].Text)
|
||
}
|
||
// ApplyHeading prepends only to a non-empty final and is a no-op on an empty/no-heading unit.
|
||
if got := ApplyHeading(chunks[0].Heading, "перевод."); got != "Глава 1\n\nперевод." {
|
||
t.Errorf("ApplyHeading = %q", got)
|
||
}
|
||
if got := ApplyHeading(chunks[0].Heading, ""); got != "" {
|
||
t.Errorf("ApplyHeading on empty final must stay empty, got %q", got)
|
||
}
|
||
if got := ApplyHeading("", "перевод."); got != "перевод." {
|
||
t.Errorf("ApplyHeading with no heading must be a no-op, got %q", got)
|
||
}
|
||
// A book with NO heading rule keeps the source header verbatim (byte-identical to pre-pack).
|
||
plain := SplitChunks(chapters, testSeg(), nil, testAbbrevs())
|
||
if !containsRune(plain[0].Text, '第') || plain[0].Heading != "" {
|
||
t.Errorf("nil-rule chunk0 must keep the marker and carry no heading: text=%q heading=%q", plain[0].Text, plain[0].Heading)
|
||
}
|
||
}
|
||
|
||
func containsRune(s string, r rune) bool {
|
||
for _, c := range s {
|
||
if c == r {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|