textmachine/backend/internal/lang/structure.go

package lang

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"regexp"

	"strings"
)

// structureFile is the source language's chapter-structure data: <root>/<lang>/structure.txt, the convention
// reader.txt already uses for the target language. Keyed by ONE language, not by a pair: «第一章» opens a
// chapter whatever it is translated into.
const structureFile = "structure.txt"

// structureAlgoVersion versions the parser, folded into the fingerprint ahead of the data (as
// packAlgoVersion/embedAlgoVersion are).
const structureAlgoVersion = "structure-v1"

// SourceStructure is a source language's chapter-header grammar: the rune(s) opening a numbered header and
// the unit runes that may close it. ONE value read by both the ingest splitter (to find boundaries) and the
// chunker (to strip a header it found).
//
// ⚠ The two readers used to read different files — ingest the embedded cjk-section.txt (五 units), the
// chunker the pair's heading.txt (four, 話 missing) — so a ja 第N話 chapter was cut by one and left unstripped
// by the other. One resolved value makes that unreachable rather than merely fixed.
type SourceStructure struct {
	// Markers are the alternative literal prefixes that open a numbered header — «第» for CJK, «Chapter» for
	// a Latin source. ALTERNATIVES, not a rune class: a class would read «Chapter» as "any of C,h,a,p,t,e,r",
	// which is how the two readers came to disagree the first time (ingest built a class, the chunker matched
	// a prefix, and they only agreed while every marker was one rune).
	Markers []string
	// Units are the section-unit runes accepted right after the numeral.
	Units map[rune]bool
	// UnitOrdered is the same set in authored order: the dominant-unit detector breaks a tie by first-seen,
	// so order is behaviour.
	UnitOrdered []rune

	fingerprint string
	headerRE    *regexp.Regexp
}

// HeaderNumeralRE matches the marker + numeral run that opens a header. Built once per resolved structure —
// it was a package singleton while the marker was a constant, and cannot be one now that a language may
// bring its own. The numeral class stays the shared CJK one plus Arabic: pack 1 ships exactly the readers
// that already existed.
func (s *SourceStructure) HeaderNumeralRE() *regexp.Regexp {
	if s == nil {
		return nil
	}
	return s.headerRE
}

// NewSourceStructure builds a resolved grammar from a marker and an ordered unit string. The ONE place the
// compiled regex and the fingerprint are established, so a caller cannot produce a half-built value.
// Empty units are legal: a language whose headers carry no unit rune («Chapter 12») matches on marker +
// numeral alone.
func NewSourceStructure(marker, units string) *SourceStructure {
	s := &SourceStructure{Units: map[rune]bool{}}
	s.Markers = strings.Fields(marker)
	for _, r := range units {
		if isSpace(r) || s.Units[r] {
			continue
		}
		s.Units[r] = true
		s.UnitOrdered = append(s.UnitOrdered, r)
	}
	alt := make([]string, 0, len(s.Markers))
	for _, m := range s.Markers {
		alt = append(alt, regexp.QuoteMeta(m))
	}
	s.headerRE = regexp.MustCompile(`^\s*(?:` + strings.Join(alt, "|") + `)\s*[0-9０-９` + DefaultCJKSection().HeadingNumeralClass() + `]+`)
	s.fingerprint = structureFingerprint("inline", []byte(s.canonical()))
	return s
}

// MatchMarker returns the marker that prefixes t and the rest of the line, or ok=false. Alternatives are
// tried longest-first so a language declaring both «Chapter» and «Chap» cannot have the short one shadow the
// long one.
func (s *SourceStructure) MatchMarker(t string) (rest string, ok bool) {
	if s == nil {
		return "", false
	}
	best := -1
	for _, m := range s.Markers {
		if len(m) > best && strings.HasPrefix(t, m) {
			best = len(m)
		}
	}
	if best < 0 {
		return "", false
	}
	return t[best:], true
}

// Fingerprint identifies the data that decided this book's boundaries. It rides cutInputs, so a structure
// edit re-cuts the book and mints new unit ids instead of leaving stale ids on moved text. Computed
// identically on the write and read-only paths, like pack.Version().
func (s *SourceStructure) Fingerprint() string {
	if s == nil {
		return structureAlgoVersion + "-none"
	}
	return s.fingerprint
}

// DefaultCJKStructure is the embedded default for a CJK-script source, DERIVED from cjk-section.txt rather
// than copied: a second spelling would rebuild the drift above, and editing cjk-section.txt itself would move
// EmbeddedVersion and re-cut every book alive for a refactor that changes no behaviour.
func DefaultCJKStructure() *SourceStructure {
	sec := DefaultCJKSection()
	s := NewSourceStructure(strings.Join(strings.Split(sec.ChapterMarkerClass(), ""), " "), string(sec.ChapterUnitOrdered))
	// No file of its own, so the fingerprint hashes what it resolved TO. The bytes of cjk-section.txt are
	// already guarded by EmbeddedVersion in the same cut tag.
	s.fingerprint = structureFingerprint("cjk", []byte(s.canonical()))
	return s
}

// canonical renders the resolved structure in the file's format in fixed order, so the fingerprint is stable
// across Go's randomised map iteration.
func (s *SourceStructure) canonical() string {
	return "marker\t" + strings.Join(s.Markers, " ") + "\nunits\t" + string(s.UnitOrdered) + "\n"
}

func structureFingerprint(origin string, b []byte) string {
	sum := sha256.Sum256(append([]byte(structureAlgoVersion+"\x00"+origin+"\x00"), b...))
	return structureAlgoVersion + "-" + origin + "-" + hex.EncodeToString(sum[:])[:12]
}

// LoadSourceStructure resolves a book's structure data by ladder:
//
//  1. <root>/<lang>/structure.txt — a language shipping its own grammar wins, and shipping it is the whole
//     mechanism: a source absent from this repository needs a directory, not a Go change.
//  2. a CJK-script language with no file gets DefaultCJKStructure(). This rung is the status quo preserved
//     deliberately, not a claim of generality — the cut has been CJK-only since it was written (row 283).
//  3. anything else gets nil: no grammar was declared, so no header is matched and provenance says so.
//
// An empty root skips only rung 1, so the no-langpack ja golden keeps splitting on 第X章. A present but
// malformed file fails loud: a half-read grammar would cut the book somewhere else, silently.
func LoadSourceStructure(root, sourceLang string) (*SourceStructure, error) {
	lng := strings.ToLower(strings.TrimSpace(sourceLang))
	if root != "" && lng != "" {
		b, err := os.ReadFile(filepath.Join(root, lng, structureFile))
		switch {
		case err == nil:
			s, perr := parseSourceStructure(b)
			if perr != nil {
				return nil, fmt.Errorf("source structure %s/%s: %w", lng, structureFile, perr)
			}
			s.fingerprint = structureFingerprint("file", b)
			return s, nil
		case !os.IsNotExist(err):
			return nil, fmt.Errorf("source structure %s/%s: %w", lng, structureFile, err)
		}
	}
	if IsCJKScriptLang(lng) {
		return DefaultCJKStructure(), nil
	}
	return nil, nil
}

// parseSourceStructure reads the `key<TAB>value` lines of structure.txt, keys marker | units. `marker` is
// required; `units` is OPTIONAL, and that is not laxity — a language whose headers are «Chapter 12» has no
// unit rune to declare, and requiring one would make the file unable to express the very sources this data
// exists to admit. A unit-bearing grammar still requires the unit right after the numeral (the guard against
// a measure word «第一次»).
func parseSourceStructure(b []byte) (*SourceStructure, error) {
	var marker, units string
	seen := map[string]bool{}
	for i, raw := range strings.Split(string(b), "\n") {
		t := strings.TrimRight(raw, "\r")
		if strings.TrimSpace(t) == "" || strings.HasPrefix(strings.TrimSpace(t), "#") {
			continue
		}
		f := strings.SplitN(t, "\t", 2)
		if len(f) != 2 || strings.TrimSpace(f[1]) == "" {
			return nil, fmt.Errorf("line %d: want `key<TAB>value` with a non-empty value (%q)", i+1, t)
		}
		key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
		if seen[key] {
			return nil, fmt.Errorf("line %d: key %q appears twice", i+1, key)
		}
		seen[key] = true
		switch key {
		case "marker":
			marker = val
		case "units":
			units = val
		default:
			return nil, fmt.Errorf("line %d: unknown key %q (want marker|units)", i+1, key)
		}
	}
	if !seen["marker"] {
		return nil, fmt.Errorf("structure needs a marker")
	}
	if len(strings.Fields(marker)) == 0 {
		return nil, fmt.Errorf("marker is empty after trimming")
	}
	return NewSourceStructure(marker, units), nil
}