31 lines
1.5 KiB
Go
31 lines
1.5 KiB
Go
package text
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"golang.org/x/text/unicode/norm"
|
|
)
|
|
|
|
// source.go: the canonical form of a SOURCE text. Applied once at ingest and again by the
|
|
// renderer before hashing, so the same book text produces the same request_hash regardless of
|
|
// the editor, the OS line endings or the Unicode composition it was authored with.
|
|
|
|
// NormalizeSource canonicalizes the source chunk so the request-hash is stable
|
|
// across editors and operating systems (review finding): strip a UTF-8 BOM,
|
|
// CRLF/CR → LF, Unicode NFC, trim surrounding whitespace. Without this a file
|
|
// saved in a different editor (BOM) or on Windows (CRLF) would produce a different
|
|
// hash and silently re-translate a book already paid for.
|
|
func NormalizeSource(s string) string {
|
|
s = strings.TrimPrefix(s, "\uFEFF") // UTF-8 BOM
|
|
s = strings.ReplaceAll(s, "\r\n", "\n")
|
|
s = strings.ReplaceAll(s, "\r", "\n")
|
|
// Replace invalid UTF-8 with U+FFFD exactly ONCE, here, so all downstream
|
|
// segmentation sees byte-valid text. Without this the chunker diverges from itself: its
|
|
// []rune-path over sentences (chunker.go) would silently replace a stray byte with U+FFFD,
|
|
// while the string path over paragraphs would keep it raw — a size-dependent corruption
|
|
// leaking into {{text}} and request_hash (external review #3). This is an ingest rule →
|
|
// covered by chunkerVersion (a deliberate re-translation, not a silent one).
|
|
s = strings.ToValidUTF8(s, "\uFFFD")
|
|
s = norm.NFC.String(s)
|
|
return strings.TrimSpace(s)
|
|
}
|