textmachine/backend/internal/chunk/chunker_heading_test.go

package chunk

import (
	"testing"

	"textmachine/backend/internal/lang"
)

// zhRuHeadingRule mirrors configs/langpacks/zh-ru/heading.txt (第 · 章节節回 · «Глава {n}»).
func zhRuHeadingRule() *lang.HeadingRule {
	return &lang.HeadingRule{
		Marker:   "第",
		Units:    map[rune]bool{'章': true, '节': true, '節': true, '回': true},
		Template: "Глава {n}",
	}
}

// TestStripHeadingRerun2 pins the pack-13 title policy on the LIVE rerun2 header shapes (第N节：subtitle):
// the marker is stripped from the model input, the subtitle is kept, and the deterministic «Глава N» is
// rendered — the fix for the cross-arm chaos («Раздел 2» / «Первая глава» / orphaned « :»).
func TestStripHeadingRerun2(t *testing.T) {
	hr := zhRuHeadingRule()
	cases := []struct {
		chapter      string
		wantHeading  string
		wantStripped string
	}{
		{"第一节：纵身亡魔心仍不悔\n\n正文第一段。", "Глава 1", "纵身亡魔心仍不悔\n\n正文第一段。"},
		{"第二节：逆光阴五百年觉悟\n\n身体。", "Глава 2", "逆光阴五百年觉悟\n\n身体。"},
		{"第四节：古月方源！\n\n夜。", "Глава 4", "古月方源！\n\n夜。"},
		{"第五节：人祖三蛊，希望开窍\n\n春。", "Глава 5", "人祖三蛊，希望开窍\n\n春。"},
		{"第一章　図書館の秘密\n\n本文。", "Глава 1", "図書館の秘密\n\n本文。"},   // fullwidth-space separator (ja-shape)
		{"第十二章：标题\n\n正文。", "Глава 12", "标题\n\n正文。"},         // multi-digit CJK numeral
		{"第1章 Title\n\nbody.", "Глава 1", "Title\n\nbody."}, // Arabic numeral + space
		{"第一节\n\n正文。", "Глава 1", "正文。"},                    // header with NO subtitle → whole line drops
	}
	for _, c := range cases {
		gotH, gotS := stripHeading(c.chapter, hr)
		if gotH != c.wantHeading || gotS != c.wantStripped {
			t.Errorf("stripHeading(%q):\n  heading  = %q, want %q\n  stripped = %q, want %q",
				c.chapter, gotH, c.wantHeading, gotS, c.wantStripped)
		}
	}
}

// TestStripHeadingGenericLatinMarker pins the П3 generic-header closure (D39.60 §5 E-2): a heading rule with
// a Latin marker and NO units — marker + space + digits, «Chapter 12» — is detected and rendered
// deterministically. That class was inexpressible before (matchHeaderLine required a unit rune and tolerated
// no space). The unit-bearing CJK path is unchanged (TestStripHeadingRerun2 is the regression).
func TestStripHeadingGenericLatinMarker(t *testing.T) {
	hr := &lang.HeadingRule{Marker: "Chapter", Units: map[rune]bool{}, Template: "Глава {n}"} // no units → unit optional
	cases := []struct{ chapter, wantHeading, wantStripped string }{
		{"Chapter 12\n\nThe body.", "Глава 12", "The body."},
		{"Chapter 3: The Fall\n\nbody.", "Глава 3", "The Fall\n\nbody."},
		{"Chapter 1 Beginnings\n\nbody.", "Глава 1", "Beginnings\n\nbody."},
	}
	for _, c := range cases {
		gotH, gotS := stripHeading(c.chapter, hr)
		if gotH != c.wantHeading || gotS != c.wantStripped {
			t.Errorf("stripHeading(%q): heading=%q want %q; stripped=%q want %q", c.chapter, gotH, c.wantHeading, gotS, c.wantStripped)
		}
	}
	// A glued ordinal («12th») is not a header; prose merely opening with «Chapter» + content is not either.
	for _, neg := range []string{"Chapter 12th\n\nx.", "Chapters were long.\n\nx."} {
		if h, _ := stripHeading(neg, hr); h != "" {
			t.Errorf("stripHeading(%q) must be a no-op, got heading %q", neg, h)
		}
	}
}

// TestChapterUnitJaWebnovel pins the 話 data addition (D39.60 §3.1): a ja .txt whose chapters are 第N話 (the
// dominant Japanese-webnovel unit) auto-detects 話 and splits, instead of silently becoming ONE chapter.
func TestChapterUnitJaWebnovel(t *testing.T) {
	lines := []string{"第1話　はじまり", "本文だ。", "第2話　つづき", "本文だ。"}
	if u := detectChapterUnit(lines); u != '話' {
		t.Errorf("detectChapterUnit must pick 話 for a 第N話 webnovel, got %q", u)
	}
}

// TestStripHeadingNegatives asserts the precision guards: a nil rule is inert; a glued measure word is not
// a header; prose that merely opens with 第 is untouched.
func TestStripHeadingNegatives(t *testing.T) {
	hr := zhRuHeadingRule()
	negatives := []string{
		"第一回见面时，他笑了。\n\n正文。", // 回 measure word glued to content 见 → NOT a header (ingest parity)
		"这是第三节的内容。\n\n正文。",   // 第 not at line start
		"普通的一段话。\n\n第二段。",    // no marker at all
		"第节：无数字\n\n正文。",      // no numeral between marker and unit
	}
	for _, chap := range negatives {
		if gotH, gotS := stripHeading(chap, hr); gotH != "" || gotS != chap {
			t.Errorf("stripHeading(%q) should be a no-op, got heading=%q stripped=%q", chap, gotH, gotS)
		}
	}
	// A nil rule is always a no-op (a book with no heading.txt).
	if gotH, gotS := stripHeading("第一节：标题\n\n正文。", nil); gotH != "" || gotS != "第一节：标题\n\n正文。" {
		t.Errorf("nil rule must be inert, got heading=%q", gotH)
	}
}

// TestSplitChunksHeadingCarried asserts the deterministic title lands on the chapter's FIRST chunk only,
// the source marker is stripped from ch.Text, and ApplyHeading prepends it correctly.
func TestSplitChunksHeadingCarried(t *testing.T) {
	hr := zhRuHeadingRule()
	chapters := []string{
		"第一节：纵身亡魔心仍不悔\n\n古月方源站着。",
		"第二节：逆光阴\n\n夜色降临。",
	}
	chunks := SplitChunks(chapters, testSeg(), hr, testAbbrevs())
	if len(chunks) < 2 {
		t.Fatalf("want ≥2 chunks, got %d", len(chunks))
	}
	// Chapter 1, chunk 0 carries «Глава 1»; its text no longer contains the marker.
	if chunks[0].Heading != "Глава 1" {
		t.Errorf("chunk0 heading = %q, want «Глава 1»", chunks[0].Heading)
	}
	if containsRune(chunks[0].Text, '第') {
		t.Errorf("chunk0 text still carries the source marker: %q", chunks[0].Text)
	}
	// ApplyHeading prepends only to a non-empty final and is a no-op on an empty/no-heading unit.
	if got := ApplyHeading(chunks[0].Heading, "перевод."); got != "Глава 1\n\nперевод." {
		t.Errorf("ApplyHeading = %q", got)
	}
	if got := ApplyHeading(chunks[0].Heading, ""); got != "" {
		t.Errorf("ApplyHeading on empty final must stay empty, got %q", got)
	}
	if got := ApplyHeading("", "перевод."); got != "перевод." {
		t.Errorf("ApplyHeading with no heading must be a no-op, got %q", got)
	}
	// A book with NO heading rule keeps the source header verbatim (byte-identical to pre-pack).
	plain := SplitChunks(chapters, testSeg(), nil, testAbbrevs())
	if !containsRune(plain[0].Text, '第') || plain[0].Heading != "" {
		t.Errorf("nil-rule chunk0 must keep the marker and carry no heading: text=%q heading=%q", plain[0].Text, plain[0].Heading)
	}
}

func containsRune(s string, r rune) bool {
	for _, c := range s {
		if c == r {
			return true
		}
	}
	return false
}