package pipeline

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"unicode"
)

// sanitizer.go: the output-sanitizer (D30.3) — a deterministic verdict-axis gate on the
// FINAL chunk text that catches the "instant unreadability" defect classes NO existing
// gate detects (exp12 root-cause + flagman §5). It is OPT-IN (Gates.Sanitizer.Enabled),
// following the coverage gate's discipline: when enabled a defect flips the chunk to
// flagged (D2 flag+skip — the contaminated output never commits to TM/export), else it is
// a no-op. Pure and deterministic (no time/rand, sorted detail) so a resume re-derives the
// identical verdict — its rule version folds into the snapshot only when enabled
// (sanitizerVersion, mirroring coverageSnapshot), and moves to verdictSnapshotID once
// content-addressed resume lands (D15.2).
//
// Six classes, each tuned PRECISION over recall (fire only on a high-confidence signal;
// the ambiguous middle stays silent — a false flag turns a good chunk into an export
// placeholder, worse than a missed defect for an opt-in readability gate):
//
//	1. leading service preamble  — «Вот отредактированный перевод: …» (gemini 6/6, exp12 §4.1)
//	2. trailing note/edit block  — appended «Примечание:/Внесённые правки:/Что изменено:» meta
//	3. markdown ### header        — «### Глава 1» (8/54 finals of stage A — D29 finding)
//	4. Latin-script insertion    — an untranslated Latin phrase/heavy Latin in the ru output
//	5. broken word form           — homoglyph-mixed token or an invalid Cyrillic sign bigram
//	6. CJK-leak in the final      — a stray Han ideograph / kana / fullwidth glyph in the ru output
//	                                («…охотники,却有 деньги!» arms/C/17.0 — D38.1; below the ≥15%
//	                                echo threshold classify() already catches, so a sparse leak)
//
// DISPOSITION-BY-CLASS (D38 infra-pack): the classes split into two tiers so a chunk is not
// silently lost to an export placeholder when the leak is a REMOVABLE cosmetic span (D35.4a —
// ch5/ch20 chapter openers dropped whole for a leading «### Глава N»):
//   - COSMETIC (strippable, exported + flagged for a human, cosmeticOnly()==true): the markdown
//     header (3) and the CJK-leak (6). The leak sits in a bounded span stripCosmetic() removes
//     deterministically, leaving readable prose; the chunk is flagged sanitizer_stripped (a
//     "auto-cleaned, verify" signal), NOT dropped.
//   - SUBSTANTIVE (dropped to a placeholder, current behaviour): the preamble (1), trailing
//     note (2), Latin insertion (4) and broken word (5). Their contamination has no reliable
//     removable boundary (a preamble may swallow real narration; a Latin clause / broken token is
//     lost content), so the output is not trusted — flagged sanitizer_defect + skipped.
//
// HONEST recall gap (documented, precision over recall): the broken-word class catches only
// the two ZERO-false-positive signatures (invalid Cyrillic sign bigrams, mixed Cyrillic/Latin
// token). The junction-duplication split heuristic («Первопред предок») was REMOVED (D33.1б): it
// false-flagged ordinary prose sharing a morphemic boundary. A word broken WITHOUT an invalid
// character («Первок предок» split into two individually-valid tokens) or a grammatical-but-wrong
// form («валуне») is NOT offline-detectable without a morphology/dictionary pass (Ф2) — left silent.

// sanitizerVersion versions the rules below (folded into the snapshot when the gate is
// enabled, like cheapGateVersion/coverageGateVersion): editing any pattern shifts the
// resolved disposition of flagged chunks, so a bump is a loud --resnapshot. [D15.2 note:
// this is a VERDICT version — once content-addressed resume lands it moves to
// verdictSnapshotID and a rule edit re-classifies free instead of re-paying the book.]
// v2 (D33.1/33.2): tail edit-summary «Основные правки для справки:» now flags; the
// junction-duplication split detector was removed (false-flagged prose); markdown/heavy-Latin
// narrowed. A verdict-axis change → loud --resnapshot + golden re-capture (invariant №8).
// v3 (D38 infra-pack): added the CJK-leak class (Han/kana/fullwidth in the ru final) and the
// cosmetic-vs-substantive disposition split (the markdown header and CJK-leak are now STRIPPED
// and exported flagged, not dropped to an empty placeholder — D35.4a). Both shift the resolved
// disposition/export of a flagged chunk → a loud --resnapshot + golden re-capture (invariant №8).
// v4 (D38 infra-pack, adversarial review): stripCosmetic now FOLDS content-bearing fullwidth ASCII
// («１２３»→«123», not deleted) and ideographic comma/period, space-replaces dropped ideographs (no
// word-merge), removes an emptied bracket pair, and no longer reformats a legit «2 : 1» — all shift
// a stripped chunk's EXPORT text → a loud --resnapshot + golden re-capture (invariant №8).
const sanitizerVersion = "sanitizer-v4"

// sanitizer tuning constants (versioned by sanitizerVersion).
const (
	// trailingRegionLines: a note/edit header counts only when it sits in the LAST N
	// non-empty lines (a meta block the model appended), so a legitimate mid-narrative
	// «Примечание» in dialogue does not fire.
	trailingRegionLines = 6
	// latinPhraseWords: a run of this many SPACE-adjacent Latin word-tokens is a leaked
	// untranslated phrase/sentence (a couple of proper names never reaches it). Set to 5
	// (адверсариальное ревью): a ≤4-word quoted foreign motto («sic transit gloria mundi»)
	// is legit source-preserved content and must NOT be dropped. ⚠ ACCEPTED recall/precision
	// edge: a deliberately-quoted foreign phrase of ≥5 words («to be or not to be») still
	// trips it — rare, and the gate is opt-in (a redrive surfaces it to a human).
	latinPhraseWords = 5
	// latinHeavyShare / latinHeavyTokens: heavy Latin contamination — Latin letters are a
	// large share of all letters AND there are several Latin tokens (both required, so a
	// short chunk with one long Latin brand name does not trip the share alone).
	latinHeavyShare  = 0.25
	latinHeavyTokens = 5
)

// sanitizerResult is the per-chunk outcome: a count per class plus deterministic detail.
type sanitizerResult struct {
	Preamble       int      `json:"preamble,omitempty"`
	TrailingNote   int      `json:"trailing_note,omitempty"`
	MarkdownHeader int      `json:"markdown_header,omitempty"`
	LatinInsert    int      `json:"latin_insert,omitempty"`
	BrokenWord     int      `json:"broken_word,omitempty"`
	CJKLeak        int      `json:"cjk_leak,omitempty"`
	Detail         []string `json:"detail,omitempty"`
}

func (s sanitizerResult) total() int {
	return s.Preamble + s.TrailingNote + s.MarkdownHeader + s.LatinInsert + s.BrokenWord + s.CJKLeak
}

// cosmeticOnly reports whether the ONLY classes that fired are the STRIPPABLE cosmetic ones
// (markdown header and/or CJK-leak) — the strip-and-export tier (D35.4a). A single substantive
// class (preamble/trailing-note/latin/broken-word) pulls the whole chunk into the skip tier: its
// contamination has no reliable removable boundary, so the output is not trusted even if a
// cosmetic leak also happens to be present. False when nothing fired.
func (s sanitizerResult) cosmeticOnly() bool {
	return s.total() > 0 && s.Preamble == 0 && s.TrailingNote == 0 && s.LatinInsert == 0 && s.BrokenWord == 0
}

// summary is the human-readable disposition detail for a sanitizer flag (deterministic —
// Detail is sorted in sanitizeOutput). Kept short (the first two findings) for the log/row.
func (s sanitizerResult) summary() string {
	if len(s.Detail) == 0 {
		return "output-sanitizer defect"
	}
	if len(s.Detail) > 2 {
		return strings.Join(s.Detail[:2], "; ") + fmt.Sprintf("; …(+%d)", len(s.Detail)-2)
	}
	return strings.Join(s.Detail, "; ")
}

// sanitizeOutput runs all six classes over one chunk's FINAL translated text.
func sanitizeOutput(text string) sanitizerResult {
	var r sanitizerResult
	if det := detectLeadingPreamble(text); det != "" {
		r.Preamble++
		r.Detail = append(r.Detail, det)
	}
	if det := detectTrailingNote(text); det != "" {
		r.TrailingNote++
		r.Detail = append(r.Detail, det)
	}
	if n, det := detectMarkdownHeaders(text); n > 0 {
		r.MarkdownHeader = n
		r.Detail = append(r.Detail, det...)
	}
	if det := detectLatinInsertion(text); det != "" {
		r.LatinInsert++
		r.Detail = append(r.Detail, det)
	}
	if n, det := detectBrokenWords(text); n > 0 {
		r.BrokenWord = n
		r.Detail = append(r.Detail, det...)
	}
	if n, det := detectCJKLeak(text); n > 0 {
		r.CJKLeak = n
		r.Detail = append(r.Detail, det...)
	}
	sort.Strings(r.Detail)
	return r
}

// --- 1. leading service preamble ------------------------------------------------

// leadingPreamblePatterns match a service preamble at the very START of the output — the
// «Вот отредактированный перевод: …» leak (gemini 6/6, exp12 §4.1). PRECISION over recall
// (адверсариальное ревью: a lax rule dropped clean chapter openings like «Вот такой вариант
// развития — самый вероятный.», «Вот его вариант: он молча ушёл.»). Two guards make it tight:
//  1. terminator is a COLON only — the em-dash «—» is ubiquitous in narration, so it is NOT
//     a terminator (a service preamble is a labeled prefix ending in «:»);
//  2. the phrase must be a genuine TRANSLATION/EDIT service label, not merely «lead + common
//     noun»: either an edit-adjective (отредактированн/исправленн/улучшенн) + a перевод/текст/
//     вариант noun, or «перевод фрагмента/текста/…», or «ниже приведён … перевод/текст».
//
// Go RE2 \w/\b are ASCII-only, so Cyrillic uses [а-яё].
var leadingPreamblePatterns = []*regexp.Regexp{
	// «Вот/Представляю/Привожу … отредактированный/исправленный перевод/текст/вариант …:»
	// (the edit-adjective front-gate holds precision, so the tail-to-colon may be long — the
	// gemini leak carries «… с соблюдением всех терминов из глоссария:»).
	regexp.MustCompile(`(?is)^\s*(?:вот|представляю|привожу|держите)\s+[^\n:]{0,40}?(?:отредактированн|исправленн|улучшенн)[а-яё]+\s+(?:перевод|текст|вариант)[а-яё]*[^\n:]{0,120}?:`),
	// «Отредактированный/Исправленный перевод/текст/вариант …:» (label at the very start)
	regexp.MustCompile(`(?is)^\s*(?:отредактированн|исправленн|улучшенн)[а-яё]+\s+(?:перевод|текст|вариант)[а-яё]*[^\n:]{0,120}?:`),
	// «(Вот) перевод фрагмента/текста/отрывка/главы/черновика:»
	regexp.MustCompile(`(?is)^\s*(?:вот\s+)?перевод\s+(?:фрагмента|текста|отрывка|главы|черновика)\s*:`),
	// «Ниже приведён/представлен … перевод/отредактированный текст:»
	regexp.MustCompile(`(?is)^\s*ниже\s+(?:привед[её]н|представлен|дан|след)[а-яё]*\s+[^\n:]{0,40}?(?:перевод|отредактированн|текст)[а-яё]*[^\n:]{0,20}?:`),
}

func detectLeadingPreamble(text string) string {
	t := strings.TrimSpace(text)
	for _, re := range leadingPreamblePatterns {
		if loc := re.FindStringIndex(t); loc != nil {
			return "ведущая служебная преамбула: " + preview(strings.TrimSpace(t[:loc[1]]))
		}
	}
	return ""
}

// --- 2. trailing note / edit block ----------------------------------------------

// trailingNoteRE matches a note/edit meta-HEADER at line start: «Примечание:», «Заметки
// переводчика», «Комментарий:», «(прим. перев.)». PRECISION over recall (адверсариальное
// ревью: «Комментатор захлёбывался…», «Заметка белела на столе.», «Пояснения не требовалось.»,
// «Сноска объясняла обычай.» are ordinary narrative nouns/gerunds — NOT headers). So the header
// word MUST be followed by a COLON, or be «примечание/заметки/комментарий + переводчика/редактора»,
// or be the «(прим. перев./ред.)» marker. Only counted inside the trailing region (below).
var trailingNoteRE = regexp.MustCompile(`(?i)^[*_>\s-]{0,4}(?:(?:примечани[ея]|заметк[аи]|комментари[йяю]|пояснени[ея]|сноск[аи])\s*:|(?:примечани[ея]|заметк[аи]|комментари[йяю])\s+(?:переводчик|редактор)[а-яё]*|прим\.\s*(?:перев|ред)\.?)`)

// editMetaAnywhereRE matches UNAMBIGUOUS editor meta-commentary that is a defect wherever it
// appears (not only trailing): «Внесённые правки:», «Что изменено:», «Список правок:», and the
// gemini tail leak «Основные правки для справки: …» (D33.1а — the A5/5_1 defect the gate exists
// to catch, exp12:229: the leading-preamble rule caught the head form «Вот отредактированный
// перевод:» but NOT this trailing edit-summary). The generic «Изменения:/Правки:» alone were
// REMOVED (адверсариальное ревью — «Изменения — вот что пугало его.» / «Правка — дело тонкое.»
// are common narrative openers), keeping only editor-specific compounds that never occur in prose.
// «основн…правк» is gated to «… для справки» OR a colon so a bare narrative «Основные правки внёс
// редактор» (no summary label) does not fire.
var editMetaAnywhereRE = regexp.MustCompile(`(?im)^[*_>\s-]{0,4}(?:(?:внесённ|внесен)[а-яё]+\s+правк|основн[а-яё]+\s+правк[а-яё]*(?:\s+для\s+справк[а-яё]+|\s*:)|что\s+(?:было\s+)?(?:изменен|исправлен)|список\s+(?:правок|изменений)|список\s+внесённых)`)

func detectTrailingNote(text string) string {
	if loc := editMetaAnywhereRE.FindStringIndex(text); loc != nil {
		return "утёкший блок правок редактора: " + preview(strings.TrimSpace(firstLineAt(text, loc[0])))
	}
	lines := nonEmptyLines(text)
	start := len(lines) - trailingRegionLines
	if start < 0 {
		start = 0
	}
	for _, ln := range lines[start:] {
		if trailingNoteRE.MatchString(ln) {
			return "хвостовой блок заметок/примечаний: " + preview(strings.TrimSpace(ln))
		}
	}
	return ""
}

// --- 3. markdown ### header -----------------------------------------------------

// markdownHeaderRE matches a markdown ATX header line («### Глава», «# Title»): 1–6 hashes,
// a space, then a LETTER or DIGIT. «#1» / «№5» / a lone «#» do not match (no space+text); and
// (D33.2) a decorative scene-break rendered with hashes — «# # #», «## ***», «### ---» — no
// longer matches, since the char after the hashes is punctuation, not a header word/number.
var markdownHeaderRE = regexp.MustCompile(`(?m)^\s{0,3}#{1,6}\s+[\p{L}\p{Nd}]`)

func detectMarkdownHeaders(text string) (int, []string) {
	var det []string
	n := 0
	for _, ln := range strings.Split(text, "\n") {
		if markdownHeaderRE.MatchString(ln) {
			n++
			det = append(det, "markdown-заголовок в выходе: "+preview(strings.TrimSpace(ln)))
		}
	}
	return n, det
}

// --- 4. Latin-script insertion --------------------------------------------------

// detectLatinInsertion flags an untranslated Latin PHRASE (≥ latinPhraseWords Latin words
// separated only by SINGLE SPACES — a real «the quick brown fox», not an alphanumeric id) OR
// heavy Latin contamination (Latin ≥ latinHeavyShare of all letters AND ≥ latinHeavyTokens
// Latin tokens). Requiring space-adjacency is load-bearing: a hex/id run like «138fd5c384e3»
// or «a1b2c3d4» yields several single-letter Latin tokens with NO space between them, which
// must NOT read as a Latin phrase (a false positive the golden fixture's tags exposed). A
// handful of Latin proper names stays silent.
func detectLatinInsertion(text string) string {
	toks := scriptTokens(text)
	maxRun, run, heavyLatTokens := 0, 0, 0
	for _, tk := range toks {
		switch {
		case tk.lat:
			if tk.adjacent && run > 0 {
				run++ // a Latin word separated from the previous Latin word by one space
			} else {
				run = 1 // phrase start (digit/punct separator or a non-Latin predecessor)
			}
			// Heavy-Latin count EXCLUDES capitalized/brand tokens (D33.2): «iPhone», «iPad»,
			// «MacBook», «Google» are legit proper nouns, so a list of 5+ brands must not trip the
			// heavy-share gate. A genuinely leaked English clause is lowercase past its first word,
			// so it still clears latinHeavyTokens (and a real multi-word Latin PHRASE is caught by
			// maxRun below, which still counts every Latin token).
			if !hasUpperLatin(tk.text) {
				heavyLatTokens++
			}
			if run > maxRun {
				maxRun = run
			}
		case tk.cyr || tk.mixed:
			run = 0
		}
	}
	if maxRun >= latinPhraseWords {
		return "латинская вставка (фраза из подряд идущих латинских слов)"
	}
	latinLetters, cyrLetters := 0, 0
	for _, ru := range text {
		if unicode.Is(unicode.Latin, ru) {
			latinLetters++
		} else if unicode.Is(unicode.Cyrillic, ru) {
			cyrLetters++
		}
	}
	total := latinLetters + cyrLetters
	if total > 0 && heavyLatTokens >= latinHeavyTokens && float64(latinLetters)/float64(total) >= latinHeavyShare {
		return "латиница-врезки: тяжёлая доля латиницы в ru-выходе"
	}
	return ""
}

// hasUpperLatin reports whether s contains an uppercase Latin letter — the brand/proper-noun
// signature («iPhone», «MacBook», «Google») excluded from the heavy-Latin token count (D33.2).
func hasUpperLatin(s string) bool {
	for _, ru := range s {
		if unicode.Is(unicode.Latin, ru) && unicode.IsUpper(ru) {
			return true
		}
	}
	return false
}

// --- 5. broken word forms -------------------------------------------------------

// invalidSignRE matches Cyrillic sign bigrams that never occur in valid Russian: a hard/soft
// sign at a word START that is FOLLOWED BY A LETTER (a broken token like «ьзал»), or two signs
// adjacent (ъь/ьъ/ьь/ъъ). Requiring the following letter excludes a STANDALONE «ъ»/«ь» — a
// metalinguistic mention of the letter itself, «Буква ъ называлась ером.» (адверсариальное
// ревью nit). High precision otherwise: word-initial-sign-then-letter and doubled signs are
// impossible in well-formed text. \P{L} (non-letter), not RE2's ASCII-only \b.
var invalidSignRE = regexp.MustCompile(`(?i)(?:^|\P{L})[ъь][а-яё]|[ъь][ъь]`)

func detectBrokenWords(text string) (int, []string) {
	var det []string
	seen := map[string]bool{}
	add := func(s string) {
		if !seen[s] {
			seen[s] = true
			det = append(det, s)
		}
	}
	// (a) invalid Cyrillic sign bigrams.
	if invalidSignRE.MatchString(text) {
		add("невалидная кириллическая биграмма (мягкий/твёрдый знак в начале слова или подряд)")
	}
	// (b) mixed Cyrillic+Latin within one letter-run token (homoglyph artifact).
	// NOTE: the junction-duplication split detector («Первопред предок») was REMOVED (D33.1б).
	// It fired identically on ordinary prose whose adjacent tokens merely share a ≥4-rune
	// morphemic boundary («около колонны/колодца», «стало талой» — «около» is a frequent
	// preposition → flag-storm on 25 legit chapters), while its only synthetic target had an
	// overlap fraction (4/9) INDISTINGUISHABLE from those false positives (4/5): no threshold on
	// overlap length or share separates them, and the canonical «Первок предок» was never caught
	// anyway (recall gap — TestSanitizerBrokenWordRecallGap). Deterministic split-duplication with
	// no invalid sign / homoglyph needs a morphology pass (Ф2), so it stays silent here.
	for _, tk := range scriptTokens(text) {
		if tk.mixed {
			add("смешение кириллицы и латиницы в одном слове (гомоглиф): " + preview(tk.text))
		}
	}
	return len(det), det
}

// --- 6. CJK-leak in the final --------------------------------------------------

// isCJKLeakRune reports whether r is a stray CJK glyph that has NO place in a Russian final: a
// Han ideograph, kana, a fullwidth/halfwidth form (U+FF00–FFEF — «，（）！» etc.), or a CJK
// symbol/punctuation (U+3001–303F — «、。「」»). The ideographic SPACE U+3000 is deliberately
// EXCLUDED (it is an indent artifact, not a content leak — research/18 §C#4 "draft-21 includes a
// U+3000 indent → 9 real leaks"): stripCosmetic normalises it to a plain space, but it does not
// alone FIRE the class, so a legitimately-indented chunk is not flag-stormed. In clean Russian
// prose every one of these is count-0, so the class is high precision. The intrinsic classify()
// already flags a ≥15%-CJK output as an ECHO (cjk_artifact, substantive) BEFORE the sanitizer
// runs, so this class only ever sees a SPARSE leak in otherwise-Russian text.
func isCJKLeakRune(r rune) bool {
	switch {
	case unicode.Is(unicode.Han, r):
		return true
	case unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r):
		return true
	case r >= 0xFF00 && r <= 0xFFEF: // fullwidth & halfwidth forms
		return true
	case r >= 0x3001 && r <= 0x303F: // CJK symbols & punctuation, EXCLUDING U+3000 (ideographic space)
		return true
	}
	return false
}

// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail
// per distinct run. A single run («却有») is enough to fire — the class is COSMETIC, so the whole
// chunk is not lost: stripCosmetic removes the run and the remainder exports (flagged for review).
func detectCJKLeak(text string) (int, []string) {
	var det []string
	seen := map[string]bool{}
	n := 0
	rs := []rune(text)
	for i := 0; i < len(rs); {
		if !isCJKLeakRune(rs[i]) {
			i++
			continue
		}
		j := i
		for j < len(rs) && isCJKLeakRune(rs[j]) {
			j++
		}
		n++
		run := string(rs[i:j])
		if !seen[run] {
			seen[run] = true
			det = append(det, "CJK-утечка в ru-выходе: "+preview(run))
		}
		i = j
	}
	return n, det
}

// --- deterministic strip of the COSMETIC classes (D35.4a export fix) -------------

// leadingMarkdownStripRE matches the leading markdown ATX prefix on a line (optional indent, 1–6
// hashes, blank(s)) directly before a header WORD/number — the same shape markdownHeaderRE fires
// on. It captures the first header rune so ReplaceAll keeps the heading text, dropping only the
// «### » artifact («### Глава 5» → «Глава 5»). A decorative «### ---» scene break (punctuation
// after the hashes) does NOT match, so it is left intact (and it never fired the class either).
var leadingMarkdownStripRE = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+([\p{L}\p{Nd}])`)

// stripCosmetic deterministically removes the two STRIPPABLE cosmetic leak classes — leading
// markdown headers and CJK-leak runs — and tidies the whitespace the removal leaves, so the
// remainder is readable. It is a PURE function (resume re-derives the identical export), invoked
// ONLY on output the sanitizer classified as cosmeticOnly (classifyOutput guarantees the result
// is non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
// sanitizer_stripped). It never crosses newlines when tidying, so paragraph structure is kept.
func stripCosmetic(text string) string {
	// 1) markdown header prefixes → keep the heading text, drop the «#### » artifact.
	out := leadingMarkdownStripRE.ReplaceAllString(text, "$1")
	// 2) neutralise CJK-leak runes. CONTENT-BEARING glyphs are FOLDED, not dropped (adversarial
	// review MAJOR): a fullwidth ASCII form is a wrong-glyph rendering of a plain char, so «１２３»
	// folds to «123» (the number is kept, not deleted) and «，」→«,» — the same NFKC fold the
	// memory normaliser uses. Only genuinely contentless runes (Han/kana/CJK brackets/symbols) are
	// removed, and a removed run leaves a SINGLE SPACE so adjacent Russian words do not merge
	// («нашёл特产древний» → «нашёл древний», not «нашёлдревний»). U+3000 → a plain space (indent).
	var b strings.Builder
	b.Grow(len(out))
	for _, r := range out {
		switch {
		case r == '　': // U+3000 ideographic space → plain space
			b.WriteByte(' ')
		case r >= 0xFF01 && r <= 0xFF5E: // fullwidth ASCII form → fold to plain ASCII (１→1, Ａ→A, ，→,)
			b.WriteRune(r - 0xFEE0)
		case r == '、': // ideographic comma → plain comma (keep the clause boundary)
			b.WriteByte(',')
		case r == '。': // ideographic full stop → plain period
			b.WriteByte('.')
		case isCJKLeakRune(r): // Han / kana / CJK brackets / other symbol → drop, leaving a space
			b.WriteByte(' ')
		default:
			b.WriteRune(r)
		}
	}
	out = b.String()
	// 3) tidy the seams the removal created, per line (horizontal whitespace only — never a
	// newline): collapse doubled spaces, drop a bracket/quote pair the strip emptied («(羅漢拳)» →
	// «()» → removed), drop a space now sitting before closing punctuation or after an opening
	// bracket/quote (never-correct Russian spacing), and trim trailing blanks. ':' is deliberately
	// NOT in the closing-punct set: a space-before-colon is legitimate in a Russian ratio/score/time
	// («счёт 2 : 1») and must not be reformatted (adversarial review — false positive).
	out = multiSpaceRE.ReplaceAllString(out, " ")
	out = emptyBracketRE.ReplaceAllString(out, "")
	out = spaceBeforePunctRE.ReplaceAllString(out, "$1")
	out = spaceAfterOpenRE.ReplaceAllString(out, "$1")
	out = multiSpaceRE.ReplaceAllString(out, " ") // an emptied bracket may leave a doubled space
	out = trailingHSpaceRE.ReplaceAllString(out, "")
	return out
}

var (
	multiSpaceRE       = regexp.MustCompile(`[ \t]{2,}`)
	emptyBracketRE     = regexp.MustCompile(`\([ \t]*\)|«[ \t]*»|\[[ \t]*\]`)
	spaceBeforePunctRE = regexp.MustCompile(`[ \t]+([,.!?;…»)\]])`)
	spaceAfterOpenRE   = regexp.MustCompile(`([«(\[])[ \t]+`)
	trailingHSpaceRE   = regexp.MustCompile(`(?m)[ \t]+$`)
)

type scriptToken struct {
	text     string
	cyr      bool // Cyrillic-only
	lat      bool // Latin-only
	mixed    bool // both scripts in one letter run (homoglyph artifact)
	adjacent bool // separated from the previous token by exactly a single space (junction check)
}

// scriptTokens splits text into maximal letter-run tokens (letters only; digits/hyphens/
// spaces/punctuation are separators), tagging Cyrillic-only / Latin-only / mixed-script and
// whether the token is separated from the previous token by exactly a single space
// (adjacency — a duplication artifact sits inside a phrase, not across punctuation/newline).
func scriptTokens(text string) []scriptToken {
	var out []scriptToken
	var b strings.Builder
	hasLat, hasCyr := false, false
	gap := "" // separator run since the previous token: "" none yet, " " one space, "x" other
	flush := func() {
		if b.Len() > 0 {
			out = append(out, scriptToken{
				text:     b.String(),
				cyr:      hasCyr && !hasLat,
				lat:      hasLat && !hasCyr,
				mixed:    hasLat && hasCyr,
				adjacent: gap == " ",
			})
			b.Reset()
			gap = ""
		}
		hasLat, hasCyr = false, false
	}
	for _, ru := range text {
		switch {
		case unicode.Is(unicode.Latin, ru):
			b.WriteRune(ru)
			hasLat = true
		case unicode.Is(unicode.Cyrillic, ru):
			b.WriteRune(ru)
			hasCyr = true
		default:
			flush()
			if ru == ' ' && gap == "" {
				gap = " "
			} else {
				gap = "x" // newline / punctuation / tab / a second space breaks adjacency
			}
		}
	}
	flush()
	return out
}

// --- shared helpers -------------------------------------------------------------

func nonEmptyLines(text string) []string {
	var out []string
	for _, ln := range strings.Split(text, "\n") {
		if strings.TrimSpace(ln) != "" {
			out = append(out, ln)
		}
	}
	return out
}

// firstLineAt returns the line containing byte offset off.
func firstLineAt(text string, off int) string {
	start := strings.LastIndexByte(text[:off], '\n') + 1
	end := strings.IndexByte(text[off:], '\n')
	if end < 0 {
		return text[start:]
	}
	return text[start : off+end]
}