package lang import ( "crypto/sha256" "encoding/hex" "fmt" "os" "path/filepath" "regexp" "strings" ) // structureFile is the source language's chapter-structure data: //structure.txt, the convention // reader.txt already uses for the target language. Keyed by ONE language, not by a pair: «第一章» opens a // chapter whatever it is translated into. const structureFile = "structure.txt" // structureAlgoVersion versions the parser, folded into the fingerprint ahead of the data (as // packAlgoVersion/embedAlgoVersion are). const structureAlgoVersion = "structure-v1" // SourceStructure is a source language's chapter-header grammar: the rune(s) opening a numbered header and // the unit runes that may close it. ONE value read by both the ingest splitter (to find boundaries) and the // chunker (to strip a header it found). // // ⚠ The two readers used to read different files — ingest the embedded cjk-section.txt (五 units), the // chunker the pair's heading.txt (four, 話 missing) — so a ja 第N話 chapter was cut by one and left unstripped // by the other. One resolved value makes that unreachable rather than merely fixed. type SourceStructure struct { // Markers are the alternative literal prefixes that open a numbered header — «第» for CJK, «Chapter» for // a Latin source. ALTERNATIVES, not a rune class: a class would read «Chapter» as "any of C,h,a,p,t,e,r", // which is how the two readers came to disagree the first time (ingest built a class, the chunker matched // a prefix, and they only agreed while every marker was one rune). Markers []string // Units are the section-unit runes accepted right after the numeral. Units map[rune]bool // UnitOrdered is the same set in authored order: the dominant-unit detector breaks a tie by first-seen, // so order is behaviour. UnitOrdered []rune fingerprint string headerRE *regexp.Regexp } // HeaderNumeralRE matches the marker + numeral run that opens a header. Built once per resolved structure — // it was a package singleton while the marker was a constant, and cannot be one now that a language may // bring its own. The numeral class stays the shared CJK one plus Arabic: pack 1 ships exactly the readers // that already existed. func (s *SourceStructure) HeaderNumeralRE() *regexp.Regexp { if s == nil { return nil } return s.headerRE } // NewSourceStructure builds a resolved grammar from a marker and an ordered unit string. The ONE place the // compiled regex and the fingerprint are established, so a caller cannot produce a half-built value. // Empty units are legal: a language whose headers carry no unit rune («Chapter 12») matches on marker + // numeral alone. func NewSourceStructure(marker, units string) *SourceStructure { s := &SourceStructure{Units: map[rune]bool{}} s.Markers = strings.Fields(marker) for _, r := range units { if isSpace(r) || s.Units[r] { continue } s.Units[r] = true s.UnitOrdered = append(s.UnitOrdered, r) } alt := make([]string, 0, len(s.Markers)) for _, m := range s.Markers { alt = append(alt, regexp.QuoteMeta(m)) } s.headerRE = regexp.MustCompile(`^\s*(?:` + strings.Join(alt, "|") + `)\s*[0-90-9` + DefaultCJKSection().HeadingNumeralClass() + `]+`) s.fingerprint = structureFingerprint("inline", []byte(s.canonical())) return s } // MatchMarker returns the marker that prefixes t and the rest of the line, or ok=false. Alternatives are // tried longest-first so a language declaring both «Chapter» and «Chap» cannot have the short one shadow the // long one. func (s *SourceStructure) MatchMarker(t string) (rest string, ok bool) { if s == nil { return "", false } best := -1 for _, m := range s.Markers { if len(m) > best && strings.HasPrefix(t, m) { best = len(m) } } if best < 0 { return "", false } return t[best:], true } // Fingerprint identifies the data that decided this book's boundaries. It rides cutInputs, so a structure // edit re-cuts the book and mints new unit ids instead of leaving stale ids on moved text. Computed // identically on the write and read-only paths, like pack.Version(). func (s *SourceStructure) Fingerprint() string { if s == nil { return structureAlgoVersion + "-none" } return s.fingerprint } // DefaultCJKStructure is the embedded default for a CJK-script source, DERIVED from cjk-section.txt rather // than copied: a second spelling would rebuild the drift above, and editing cjk-section.txt itself would move // EmbeddedVersion and re-cut every book alive for a refactor that changes no behaviour. func DefaultCJKStructure() *SourceStructure { sec := DefaultCJKSection() s := NewSourceStructure(strings.Join(strings.Split(sec.ChapterMarkerClass(), ""), " "), string(sec.ChapterUnitOrdered)) // No file of its own, so the fingerprint hashes what it resolved TO. The bytes of cjk-section.txt are // already guarded by EmbeddedVersion in the same cut tag. s.fingerprint = structureFingerprint("cjk", []byte(s.canonical())) return s } // canonical renders the resolved structure in the file's format in fixed order, so the fingerprint is stable // across Go's randomised map iteration. func (s *SourceStructure) canonical() string { return "marker\t" + strings.Join(s.Markers, " ") + "\nunits\t" + string(s.UnitOrdered) + "\n" } func structureFingerprint(origin string, b []byte) string { sum := sha256.Sum256(append([]byte(structureAlgoVersion+"\x00"+origin+"\x00"), b...)) return structureAlgoVersion + "-" + origin + "-" + hex.EncodeToString(sum[:])[:12] } // LoadSourceStructure resolves a book's structure data by ladder: // // 1. //structure.txt — a language shipping its own grammar wins, and shipping it is the whole // mechanism: a source absent from this repository needs a directory, not a Go change. // 2. a CJK-script language with no file gets DefaultCJKStructure(). This rung is the status quo preserved // deliberately, not a claim of generality — the cut has been CJK-only since it was written (row 283). // 3. anything else gets nil: no grammar was declared, so no header is matched and provenance says so. // // An empty root skips only rung 1, so the no-langpack ja golden keeps splitting on 第X章. A present but // malformed file fails loud: a half-read grammar would cut the book somewhere else, silently. func LoadSourceStructure(root, sourceLang string) (*SourceStructure, error) { lng := strings.ToLower(strings.TrimSpace(sourceLang)) if root != "" && lng != "" { b, err := os.ReadFile(filepath.Join(root, lng, structureFile)) switch { case err == nil: s, perr := parseSourceStructure(b) if perr != nil { return nil, fmt.Errorf("source structure %s/%s: %w", lng, structureFile, perr) } s.fingerprint = structureFingerprint("file", b) return s, nil case !os.IsNotExist(err): return nil, fmt.Errorf("source structure %s/%s: %w", lng, structureFile, err) } } if IsCJKScriptLang(lng) { return DefaultCJKStructure(), nil } return nil, nil } // parseSourceStructure reads the `keyvalue` lines of structure.txt, keys marker | units. `marker` is // required; `units` is OPTIONAL, and that is not laxity — a language whose headers are «Chapter 12» has no // unit rune to declare, and requiring one would make the file unable to express the very sources this data // exists to admit. A unit-bearing grammar still requires the unit right after the numeral (the guard against // a measure word «第一次»). func parseSourceStructure(b []byte) (*SourceStructure, error) { var marker, units string seen := map[string]bool{} for i, raw := range strings.Split(string(b), "\n") { t := strings.TrimRight(raw, "\r") if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") { continue } f := strings.SplitN(t, "\t", 2) if len(f) != 2 || strings.TrimSpace(f[1]) == "" { return nil, fmt.Errorf("line %d: want `keyvalue` with a non-empty value (%q)", i+1, t) } key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1]) if seen[key] { return nil, fmt.Errorf("line %d: key %q appears twice", i+1, key) } seen[key] = true switch key { case "marker": marker = val case "units": units = val default: return nil, fmt.Errorf("line %d: unknown key %q (want marker|units)", i+1, key) } } if !seen["marker"] { return nil, fmt.Errorf("structure needs a marker") } if len(strings.Fields(marker)) == 0 { return nil, fmt.Errorf("marker is empty after trimming") } return NewSourceStructure(marker, units), nil }