textmachine/backend/internal/chunk/headingparity_test.go

package chunk

import (
	"fmt"
	"strings"
	"testing"

	"textmachine/backend/internal/lang"
)

// headingparity_test.go holds the two halves of a question nobody had asked the code: does the chapter
// heading rule mean the same thing on both sides of the ingest→chunker seam? The rule is spread over three
// predicates in two files — the ingest's length guard (chapterHeaderMaxRunes), the chunker's numeral guard
// (a zero section is not a section) and the shared separator class — and each half was pinned only by its
// own tests, which cannot see a disagreement between them.
//
// The first test is the GUARANTEE, the second its named exception. Neither works alone: a pin of a known
// defect is a second description of the bug, and it leaves nothing behind once the defect is fixed.

// headingParityBody is chapter prose, long enough to be real and short enough to keep the fixture cheap.
func headingParityBody() string { return "\n" + strings.Repeat("古月方源站在洞口。", 30) }

// TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders is the live guarantee: on the header shapes a book
// actually uses, the ingest's predicate and the chunker's parse answer the same, and a book cut by the
// ingest comes out of the chunker with a deterministic title on EVERY chapter and no source marker left in
// the text the model reads. Green today, and it must stay green after the divergence below is closed.
func TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders(t *testing.T) {
	st := lang.DefaultCJKStructure()
	rule := zhRuChapterRule()
	// ⛔ THE NUMBERS ARE NOT CONSECUTIVE, ON PURPOSE. A file numbered 一·二·三 makes the parsed header number
	// equal the chapter's ORDINAL, and every assertion below would then hold just as well for a title
	// rendered from the counter as for one rendered from the header — the two quantities coincide and the
	// whole class of «the title came from the wrong place» goes invisible. Here chapter 2 must be «Глава 3».
	headers := []struct {
		line string
		num  int
	}{
		{"第一章 起始", 1},
		{"第三章 继续", 3},
		{"第七章：正常标题", 7},
	}
	var src strings.Builder
	for _, h := range headers {
		fmt.Fprintf(&src, "%s%s\n\n", h.line, headingParityBody())
	}
	doc, err := IngestEncoded(writeTempTXT(t, src.String()), "", "zh", st)
	if err != nil {
		t.Fatal(err)
	}
	if len(doc.Chapters) != len(headers) {
		t.Fatalf("the ingest cut %d chapters out of %d headers", len(doc.Chapters), len(headers))
	}
	unit := detectChapterUnit(strings.Split(src.String(), "\n"), st)
	if unit == 0 {
		t.Fatal("test premise broken: the ingest detected no chapter unit, so its predicate was never asked")
	}

	// The two predicates on the same lines. The chunker's NUMBER is asserted too: agreeing that a line is a
	// header while reading a different number out of it is the same defect one layer down.
	for _, h := range headers {
		byIngest := isChapterHeader(h.line, unit, st)
		n, _, byChunker := matchHeaderLine(h.line, rule)
		if !byIngest || !byChunker || n != h.num {
			t.Fatalf("%q: the ingest says header=%v; the chunker says header=%v number=%d — want a header on both sides and number %d",
				h.line, byIngest, byChunker, n, h.num)
		}
	}

	chunks := SplitChunks(doc.Chapters, testSeg(), rule, testAbbrevs())
	titled := 0
	for _, c := range chunks {
		if c.ChunkIdx != 0 {
			continue
		}
		titled++
		// Guarded, not indexed straight into: the whole point of this fixture is that a chapter's NUMBER and
		// its position are different quantities, so an index built from one to look up the other has to say
		// so when it goes out of range instead of panicking.
		if c.Chapter < 1 || c.Chapter > len(headers) {
			t.Fatalf("the cut produced chapter %d and the fixture has %d headers — the chapter counter no longer indexes them",
				c.Chapter, len(headers))
		}
		// The title is the SOURCE HEADER's number, not the chapter's position in the file. On this fixture
		// the two differ from chapter 2 on, which is what makes the assertion say anything.
		if want := fmt.Sprintf("Глава %d", headers[c.Chapter-1].num); c.Heading != want {
			t.Fatalf("chapter %d (header %q) opens with heading %q, want %q — the title must come from the header's own number, not from the chapter counter",
				c.Chapter, headers[c.Chapter-1].line, c.Heading, want)
		}
		if _, opensWithMarker := st.MatchMarker(c.Text); opensWithMarker {
			t.Fatalf("chapter %d still opens with its source marker, so the raw header reaches the model: %q",
				c.Chapter, firstRunes(c.Text, 16))
		}
	}
	if titled != len(headers) {
		t.Fatalf("%d chapters opened, want %d", titled, len(headers))
	}
}

// TestIngestAndChunkerDivergeOnAZeroPrologueAndALongHeader pins a DIVERGENCE that exists today (backlog
// row 346), not a guarantee. Two header shapes are cut by the ingest and refused by the chunker, so the
// header line survives into the text the model translates and the chapter numbers shift under the reader:
//
//   - «第零章», a prologue: parseSectionNumeral refuses a zero section, the ingest's regex accepts it;
//   - a header past chapterHeaderMaxRunes: the length guard exists only in the ingest.
//
// ⛔ THE FIX IS NOT IN SCOPE HERE AND IT IS NOT $0: it changes the TEXT of a chunk, which bumps
// chunkerVersion, which moves both waves' snapshots and re-bills the book. It lands in the re-cut window.
// So what is asserted is TODAY's shape and the test is green — a red test cannot be left in main, and a
// skip would hide the whole class rather than name one case of it. When the fix lands this test fails on
// purpose, and every failure message says what to do about it.
func TestIngestAndChunkerDivergeOnAZeroPrologueAndALongHeader(t *testing.T) {
	const closed = "the divergence backlog row 346 pins has CLOSED — that is the fix landing, not a regression. Delete this test and fold the case into TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders"
	st := lang.DefaultCJKStructure()
	rule := zhRuChapterRule()
	src := "第零章：序幕" + headingParityBody() + "\n\n第一章 起始" + headingParityBody() + "\n\n第二章 继续" + headingParityBody() + "\n"

	doc, err := IngestEncoded(writeTempTXT(t, src), "", "zh", st)
	if err != nil {
		t.Fatal(err)
	}
	// (1) The ingest cuts the prologue as a chapter of its own.
	if len(doc.Chapters) != 3 {
		// NOT the `closed` message: this is the test's PREMISE, and it breaks when the INGEST stops cutting
		// the prologue as its own chapter — the opposite side of the seam from the fix this test waits for.
		// Telling a reader to delete the test would send them to fold a case nobody has closed.
		t.Fatalf("premise broken: the ingest cut %d chapters, want 3 (a prologue and two) — the ingest's own header predicate changed, and the divergence below has not been judged either way", len(doc.Chapters))
	}
	chunks := SplitChunks(doc.Chapters, testSeg(), rule, testAbbrevs())
	// Guarded rather than indexed straight into: this test is BUILT to fail one day, and whoever reads that
	// failure has to get the message below, not an index panic.
	if len(chunks) < 3 {
		t.Fatalf("the cut produced %d chunks for 3 chapters — %s", len(chunks), closed)
	}
	// (2) The chunker does not recognise the prologue's header, so the chapter opens untitled …
	if chunks[0].Heading != "" {
		t.Fatalf("the prologue now carries the title %q — %s", chunks[0].Heading, closed)
	}
	// (3) … and its raw CJK marker travels to the model inside the text it translates.
	if !strings.HasPrefix(chunks[0].Text, "第零章") {
		t.Fatalf("the prologue's source marker no longer reaches the model (%q) — %s", firstRunes(chunks[0].Text, 16), closed)
	}
	// (4) The half a reader sees, and it is NOT a divergence pin: a chapter's ORDINAL and the number in its
	// title are two different quantities, and the prologue makes them differ from chapter 2 on — the cut
	// numbers 第一章 as chapter 2 while the template renders its own «Глава 1». Correct on both sides, and it
	// survives the fix in (2): recognising 第零章 gives it a title, it does not renumber anything. Asserted
	// here because this is the file where the two numbers are measurably apart, and the re-cut design has to
	// say which of them addresses a chapter (chunk_status is keyed on the ordinal; a reader sees the title).
	var second *Chunk
	for i := range chunks {
		if chunks[i].Chapter == 2 && chunks[i].ChunkIdx == 0 {
			second = &chunks[i]
		}
	}
	if second == nil {
		t.Fatalf("the cut has no chapter 2 — %s", closed)
	}
	if second.Heading != "Глава 1" {
		t.Fatalf("chapter 2 (第一章) is titled %q, want «Глава 1»: the ordinal and the title's number came apart differently than measured — this is the addressing question, NOT the row-346 divergence, so read it before assuming a fix landed", second.Heading)
	}

	// The length half, with its own CONTROL. ⚠ The header must carry a separator after its unit rune
	// («第四章：…»): without one BOTH sides refuse it for an unrelated reason and the divergence does not
	// reproduce at any length at all.
	unit := detectChapterUnit(strings.Split(src, "\n"), st)
	atCeiling := "第四章：" + strings.Repeat("标", chapterHeaderMaxRunes-4)
	pastCeiling := "第四章：" + strings.Repeat("标", chapterHeaderMaxRunes-3)
	if len([]rune(atCeiling)) != chapterHeaderMaxRunes || len([]rune(pastCeiling)) != chapterHeaderMaxRunes+1 {
		t.Fatalf("the test built the wrong lengths: %d and %d runes around a ceiling of %d",
			len([]rune(atCeiling)), len([]rune(pastCeiling)), chapterHeaderMaxRunes)
	}
	// CONTROL: at exactly the ceiling both halves still say "header". It is what makes the disagreement
	// below a disagreement about LENGTH rather than about the shape of this fixture.
	_, _, chunkerAt := matchHeaderLine(atCeiling, rule)
	if ingestAt := isChapterHeader(atCeiling, unit, st); !ingestAt || !chunkerAt {
		t.Fatalf("control broken: at exactly %d runes the ingest says %v and the chunker says %v — they must still agree there",
			chapterHeaderMaxRunes, ingestAt, chunkerAt)
	}
	n, _, chunkerPast := matchHeaderLine(pastCeiling, rule)
	if ingestPast := isChapterHeader(pastCeiling, unit, st); ingestPast || !chunkerPast || n != 4 {
		t.Fatalf("at %d runes the ingest says header=%v and the chunker says header=%v number=%d; today the ingest refuses it and the chunker reads chapter 4. %s",
			chapterHeaderMaxRunes+1, ingestPast, chunkerPast, n, closed)
	}
}

// firstRunes keeps a failure message readable when the text it quotes is a whole chapter.
func firstRunes(s string, n int) string {
	r := []rune(s)
	if len(r) <= n {
		return s
	}
	return string(r[:n]) + "…"
}