package chunk import ( "fmt" "strings" "testing" "textmachine/backend/internal/lang" ) // headingparity_test.go holds the two halves of a question nobody had asked the code: does the chapter // heading rule mean the same thing on both sides of the ingest→chunker seam? The rule is spread over three // predicates in two files — the ingest's length guard (chapterHeaderMaxRunes), the chunker's numeral guard // (a zero section is not a section) and the shared separator class — and each half was pinned only by its // own tests, which cannot see a disagreement between them. // // The first test is the GUARANTEE, the second its named exception. Neither works alone: a pin of a known // defect is a second description of the bug, and it leaves nothing behind once the defect is fixed. // headingParityBody is chapter prose, long enough to be real and short enough to keep the fixture cheap. func headingParityBody() string { return "\n" + strings.Repeat("古月方源站在洞口。", 30) } // TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders is the live guarantee: on the header shapes a book // actually uses, the ingest's predicate and the chunker's parse answer the same, and a book cut by the // ingest comes out of the chunker with a deterministic title on EVERY chapter and no source marker left in // the text the model reads. Green today, and it must stay green after the divergence below is closed. func TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders(t *testing.T) { st := lang.DefaultCJKStructure() rule := zhRuChapterRule() // ⛔ THE NUMBERS ARE NOT CONSECUTIVE, ON PURPOSE. A file numbered 一·二·三 makes the parsed header number // equal the chapter's ORDINAL, and every assertion below would then hold just as well for a title // rendered from the counter as for one rendered from the header — the two quantities coincide and the // whole class of «the title came from the wrong place» goes invisible. Here chapter 2 must be «Глава 3». headers := []struct { line string num int }{ {"第一章 起始", 1}, {"第三章 继续", 3}, {"第七章:正常标题", 7}, } var src strings.Builder for _, h := range headers { fmt.Fprintf(&src, "%s%s\n\n", h.line, headingParityBody()) } doc, err := IngestEncoded(writeTempTXT(t, src.String()), "", "zh", st) if err != nil { t.Fatal(err) } if len(doc.Chapters) != len(headers) { t.Fatalf("the ingest cut %d chapters out of %d headers", len(doc.Chapters), len(headers)) } unit := detectChapterUnit(strings.Split(src.String(), "\n"), st) if unit == 0 { t.Fatal("test premise broken: the ingest detected no chapter unit, so its predicate was never asked") } // The two predicates on the same lines. The chunker's NUMBER is asserted too: agreeing that a line is a // header while reading a different number out of it is the same defect one layer down. for _, h := range headers { byIngest := isChapterHeader(h.line, unit, st) n, _, byChunker := matchHeaderLine(h.line, rule) if !byIngest || !byChunker || n != h.num { t.Fatalf("%q: the ingest says header=%v; the chunker says header=%v number=%d — want a header on both sides and number %d", h.line, byIngest, byChunker, n, h.num) } } chunks := SplitChunks(doc.Chapters, testSeg(), rule, testAbbrevs()) titled := 0 for _, c := range chunks { if c.ChunkIdx != 0 { continue } titled++ // Guarded, not indexed straight into: the whole point of this fixture is that a chapter's NUMBER and // its position are different quantities, so an index built from one to look up the other has to say // so when it goes out of range instead of panicking. if c.Chapter < 1 || c.Chapter > len(headers) { t.Fatalf("the cut produced chapter %d and the fixture has %d headers — the chapter counter no longer indexes them", c.Chapter, len(headers)) } // The title is the SOURCE HEADER's number, not the chapter's position in the file. On this fixture // the two differ from chapter 2 on, which is what makes the assertion say anything. if want := fmt.Sprintf("Глава %d", headers[c.Chapter-1].num); c.Heading != want { t.Fatalf("chapter %d (header %q) opens with heading %q, want %q — the title must come from the header's own number, not from the chapter counter", c.Chapter, headers[c.Chapter-1].line, c.Heading, want) } if _, opensWithMarker := st.MatchMarker(c.Text); opensWithMarker { t.Fatalf("chapter %d still opens with its source marker, so the raw header reaches the model: %q", c.Chapter, firstRunes(c.Text, 16)) } } if titled != len(headers) { t.Fatalf("%d chapters opened, want %d", titled, len(headers)) } } // TestIngestAndChunkerDivergeOnAZeroPrologueAndALongHeader pins a DIVERGENCE that exists today (backlog // row 346), not a guarantee. Two header shapes are cut by the ingest and refused by the chunker, so the // header line survives into the text the model translates and the chapter numbers shift under the reader: // // - «第零章», a prologue: parseSectionNumeral refuses a zero section, the ingest's regex accepts it; // - a header past chapterHeaderMaxRunes: the length guard exists only in the ingest. // // ⛔ THE FIX IS NOT IN SCOPE HERE AND IT IS NOT $0: it changes the TEXT of a chunk, which bumps // chunkerVersion, which moves both waves' snapshots and re-bills the book. It lands in the re-cut window. // So what is asserted is TODAY's shape and the test is green — a red test cannot be left in main, and a // skip would hide the whole class rather than name one case of it. When the fix lands this test fails on // purpose, and every failure message says what to do about it. func TestIngestAndChunkerDivergeOnAZeroPrologueAndALongHeader(t *testing.T) { const closed = "the divergence backlog row 346 pins has CLOSED — that is the fix landing, not a regression. Delete this test and fold the case into TestIngestAndChunkerAgreeOnOrdinaryChapterHeaders" st := lang.DefaultCJKStructure() rule := zhRuChapterRule() src := "第零章:序幕" + headingParityBody() + "\n\n第一章 起始" + headingParityBody() + "\n\n第二章 继续" + headingParityBody() + "\n" doc, err := IngestEncoded(writeTempTXT(t, src), "", "zh", st) if err != nil { t.Fatal(err) } // (1) The ingest cuts the prologue as a chapter of its own. if len(doc.Chapters) != 3 { // NOT the `closed` message: this is the test's PREMISE, and it breaks when the INGEST stops cutting // the prologue as its own chapter — the opposite side of the seam from the fix this test waits for. // Telling a reader to delete the test would send them to fold a case nobody has closed. t.Fatalf("premise broken: the ingest cut %d chapters, want 3 (a prologue and two) — the ingest's own header predicate changed, and the divergence below has not been judged either way", len(doc.Chapters)) } chunks := SplitChunks(doc.Chapters, testSeg(), rule, testAbbrevs()) // Guarded rather than indexed straight into: this test is BUILT to fail one day, and whoever reads that // failure has to get the message below, not an index panic. if len(chunks) < 3 { t.Fatalf("the cut produced %d chunks for 3 chapters — %s", len(chunks), closed) } // (2) The chunker does not recognise the prologue's header, so the chapter opens untitled … if chunks[0].Heading != "" { t.Fatalf("the prologue now carries the title %q — %s", chunks[0].Heading, closed) } // (3) … and its raw CJK marker travels to the model inside the text it translates. if !strings.HasPrefix(chunks[0].Text, "第零章") { t.Fatalf("the prologue's source marker no longer reaches the model (%q) — %s", firstRunes(chunks[0].Text, 16), closed) } // (4) The half a reader sees, and it is NOT a divergence pin: a chapter's ORDINAL and the number in its // title are two different quantities, and the prologue makes them differ from chapter 2 on — the cut // numbers 第一章 as chapter 2 while the template renders its own «Глава 1». Correct on both sides, and it // survives the fix in (2): recognising 第零章 gives it a title, it does not renumber anything. Asserted // here because this is the file where the two numbers are measurably apart, and the re-cut design has to // say which of them addresses a chapter (chunk_status is keyed on the ordinal; a reader sees the title). var second *Chunk for i := range chunks { if chunks[i].Chapter == 2 && chunks[i].ChunkIdx == 0 { second = &chunks[i] } } if second == nil { t.Fatalf("the cut has no chapter 2 — %s", closed) } if second.Heading != "Глава 1" { t.Fatalf("chapter 2 (第一章) is titled %q, want «Глава 1»: the ordinal and the title's number came apart differently than measured — this is the addressing question, NOT the row-346 divergence, so read it before assuming a fix landed", second.Heading) } // The length half, with its own CONTROL. ⚠ The header must carry a separator after its unit rune // («第四章:…»): without one BOTH sides refuse it for an unrelated reason and the divergence does not // reproduce at any length at all. unit := detectChapterUnit(strings.Split(src, "\n"), st) atCeiling := "第四章:" + strings.Repeat("标", chapterHeaderMaxRunes-4) pastCeiling := "第四章:" + strings.Repeat("标", chapterHeaderMaxRunes-3) if len([]rune(atCeiling)) != chapterHeaderMaxRunes || len([]rune(pastCeiling)) != chapterHeaderMaxRunes+1 { t.Fatalf("the test built the wrong lengths: %d and %d runes around a ceiling of %d", len([]rune(atCeiling)), len([]rune(pastCeiling)), chapterHeaderMaxRunes) } // CONTROL: at exactly the ceiling both halves still say "header". It is what makes the disagreement // below a disagreement about LENGTH rather than about the shape of this fixture. _, _, chunkerAt := matchHeaderLine(atCeiling, rule) if ingestAt := isChapterHeader(atCeiling, unit, st); !ingestAt || !chunkerAt { t.Fatalf("control broken: at exactly %d runes the ingest says %v and the chunker says %v — they must still agree there", chapterHeaderMaxRunes, ingestAt, chunkerAt) } n, _, chunkerPast := matchHeaderLine(pastCeiling, rule) if ingestPast := isChapterHeader(pastCeiling, unit, st); ingestPast || !chunkerPast || n != 4 { t.Fatalf("at %d runes the ingest says header=%v and the chunker says header=%v number=%d; today the ingest refuses it and the chunker reads chapter 4. %s", chapterHeaderMaxRunes+1, ingestPast, chunkerPast, n, closed) } } // firstRunes keeps a failure message readable when the text it quotes is a whole chapter. func firstRunes(s string, n int) string { r := []rune(s) if len(r) <= n { return s } return string(r[:n]) + "…" }