833 lines
45 KiB
Go
833 lines
45 KiB
Go
package pipeline
|
||
|
||
import (
|
||
"fmt"
|
||
"regexp"
|
||
"sort"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"golang.org/x/text/unicode/norm"
|
||
"golang.org/x/text/width"
|
||
)
|
||
|
||
// sanitizer.go: the output-sanitizer (D30.3) — a deterministic verdict-axis gate on the
|
||
// FINAL chunk text that catches the "instant unreadability" defect classes NO existing
|
||
// gate detects (exp12 root-cause + flagman §5). It is OPT-IN (Gates.Sanitizer.Enabled),
|
||
// following the coverage gate's discipline: when enabled a defect flips the chunk to
|
||
// flagged (D2 flag+skip — the contaminated output never commits to TM/export), else it is
|
||
// a no-op. Pure and deterministic (no time/rand, sorted detail) so a resume re-derives the
|
||
// identical verdict — its rule version folds into the snapshot only when enabled
|
||
// (sanitizerVersion, mirroring coverageSnapshot), and moves to verdictSnapshotID once
|
||
// content-addressed resume lands (D15.2).
|
||
//
|
||
// Six classes, each tuned PRECISION over recall (fire only on a high-confidence signal;
|
||
// the ambiguous middle stays silent — a false flag turns a good chunk into an export
|
||
// placeholder, worse than a missed defect for an opt-in readability gate):
|
||
//
|
||
// 1. leading service preamble — «Вот отредактированный перевод: …» (gemini 6/6, exp12 §4.1)
|
||
// 2. trailing note/edit block — appended «Примечание:/Внесённые правки:/Что изменено:» meta
|
||
// 3. markdown ### header — «### Глава 1» (8/54 finals of stage A — D29 finding)
|
||
// 4. Latin-script insertion — an untranslated Latin phrase/heavy Latin in the ru output
|
||
// 5. broken word form — homoglyph-mixed token or an invalid Cyrillic sign bigram
|
||
// 6. CJK-leak in the final — a stray Han ideograph / kana / fullwidth glyph in the ru output
|
||
// («…охотники,却有 деньги!» arms/C/17.0 — D38.1; below the ≥15%
|
||
// echo threshold classify() already catches, so a sparse leak)
|
||
//
|
||
// DISPOSITION-BY-CLASS (D38 infra-pack): the classes split into two tiers so a chunk is not
|
||
// silently lost to an export placeholder when the leak is a REMOVABLE cosmetic span (D35.4a —
|
||
// ch5/ch20 chapter openers dropped whole for a leading «### Глава N»):
|
||
// - COSMETIC (strippable, exported + flagged for a human, cosmeticOnly()==true): the markdown
|
||
// header (3) and the CJK-leak (6). The leak sits in a bounded span stripCosmetic() removes
|
||
// deterministically, leaving readable prose; the chunk is flagged sanitizer_stripped (a
|
||
// "auto-cleaned, verify" signal), NOT dropped.
|
||
// - SUBSTANTIVE (dropped to a placeholder, current behaviour): the preamble (1), trailing
|
||
// note (2), Latin insertion (4) and broken word (5). Their contamination has no reliable
|
||
// removable boundary (a preamble may swallow real narration; a Latin clause / broken token is
|
||
// lost content), so the output is not trusted — flagged sanitizer_defect + skipped.
|
||
//
|
||
// HONEST recall gap (documented, precision over recall): the broken-word class catches only
|
||
// the two ZERO-false-positive signatures (invalid Cyrillic sign bigrams, mixed Cyrillic/Latin
|
||
// token). The junction-duplication split heuristic («Первопред предок») was REMOVED (D33.1б): it
|
||
// false-flagged ordinary prose sharing a morphemic boundary. A word broken WITHOUT an invalid
|
||
// character («Первок предок» split into two individually-valid tokens) or a grammatical-but-wrong
|
||
// form («валуне») is NOT offline-detectable without a morphology/dictionary pass (Ф2) — left silent.
|
||
|
||
// sanitizerVersion versions the rules below (folded into the snapshot when the gate is
|
||
// enabled, like cheapGateVersion/coverageGateVersion): editing any pattern shifts the
|
||
// resolved disposition of flagged chunks, so a bump is a loud --resnapshot. [D15.2 note:
|
||
// this is a VERDICT version — once content-addressed resume lands it moves to
|
||
// verdictSnapshotID and a rule edit re-classifies free instead of re-paying the book.]
|
||
// v2 (D33.1/33.2): tail edit-summary «Основные правки для справки:» now flags; the
|
||
// junction-duplication split detector was removed (false-flagged prose); markdown/heavy-Latin
|
||
// narrowed. A verdict-axis change → loud --resnapshot + golden re-capture (invariant №8).
|
||
// v3 (D38 infra-pack): added the CJK-leak class (Han/kana/fullwidth in the ru final) and the
|
||
// cosmetic-vs-substantive disposition split (the markdown header and CJK-leak are now STRIPPED
|
||
// and exported flagged, not dropped to an empty placeholder — D35.4a). Both shift the resolved
|
||
// disposition/export of a flagged chunk → a loud --resnapshot + golden re-capture (invariant №8).
|
||
// v4 (D38 infra-pack, adversarial review): stripCosmetic now FOLDS content-bearing fullwidth ASCII
|
||
// («１２３»→«123», not deleted) and ideographic comma/period, space-replaces dropped ideographs (no
|
||
// word-merge), removes an emptied bracket pair, and no longer reformats a legit «2 : 1» — all shift
|
||
// a stripped chunk's EXPORT text → a loud --resnapshot + golden re-capture (invariant №8).
|
||
// v5 (D39 слой 6, export-contract): the detector is now a PURE detector over the export-normalised
|
||
// text — the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «１２３»→«123», ideographic space
|
||
// U+3000, «、»→«,» «。»→«.») are folded by the shared recoverableFold (reusing the x/text width.Fold
|
||
// primitive instead of the hand-rolled «r-0xFEE0» arithmetic, closing L5-stripcosmetic-duplicates-nfkc;
|
||
// deliberately NOT full NFKC, which would clobber «…»/«№») and therefore NO LONGER FIRE the CJK-leak
|
||
// class — only genuinely contentless CJK (Han/kana/CJK brackets & symbols) does. stripCosmetic
|
||
// delegates that fold too. This shifts which chunks flag and a stripped chunk's export → a loud
|
||
// --resnapshot + golden re-capture (invariant №8). [exportNormalize itself is an EPHEMERAL export
|
||
// projection, not a persisted input; only stripCosmetic's output rides a checkpoint, versioned here.]
|
||
// v6 (D39.2 T2, owner decision 16.07: KEEP glosses; + D39.4 adversarial fix-list F1/F2): (a) a
|
||
// whitelisted CJK term gloss — a balanced Han/kana run inside parens DIRECTLY after a Cyrillic/Latin
|
||
// head word, «Кулак Ло Ханя (羅漢拳)» — is NOT a leak: detectCJKLeak no longer fires on it and
|
||
// stripCosmetic/exportNormalize preserve its bytes; a BARE Han run stays a leak. (b) F1: ALL six classes
|
||
// now DETECT over the export-normalised (folded) text, closing the fullwidth-signature gate bypass
|
||
// (a «＃＃＃»/«：» variant used to read clean while export manufactured the ASCII defect). (c) F2: a gloss
|
||
// bounds its non-CJK content (inner length, proportional pinyin-token count, per-token length,
|
||
// lowercase), so a leaked English clause after a token Han char is not whitelisted. All shift which
|
||
// chunks flag and a stripped chunk's export → a loud --resnapshot + golden re-capture (invariant №8).
|
||
const sanitizerVersion = "sanitizer-v6"
|
||
|
||
// sanitizer tuning constants (versioned by sanitizerVersion).
|
||
const (
|
||
// trailingRegionLines: a note/edit header counts only when it sits in the LAST N
|
||
// non-empty lines (a meta block the model appended), so a legitimate mid-narrative
|
||
// «Примечание» in dialogue does not fire.
|
||
trailingRegionLines = 6
|
||
// latinPhraseWords: a run of this many SPACE-adjacent Latin word-tokens is a leaked
|
||
// untranslated phrase/sentence (a couple of proper names never reaches it). Set to 5
|
||
// (адверсариальное ревью): a ≤4-word quoted foreign motto («sic transit gloria mundi»)
|
||
// is legit source-preserved content and must NOT be dropped. ⚠ ACCEPTED recall/precision
|
||
// edge: a deliberately-quoted foreign phrase of ≥5 words («to be or not to be») still
|
||
// trips it — rare, and the gate is opt-in (a redrive surfaces it to a human).
|
||
latinPhraseWords = 5
|
||
// latinHeavyShare / latinHeavyTokens: heavy Latin contamination — Latin letters are a
|
||
// large share of all letters AND there are several Latin tokens (both required, so a
|
||
// short chunk with one long Latin brand name does not trip the share alone).
|
||
latinHeavyShare = 0.25
|
||
latinHeavyTokens = 5
|
||
)
|
||
|
||
// sanitizerResult is the per-chunk outcome: a count per class plus deterministic detail.
|
||
type sanitizerResult struct {
|
||
Preamble int `json:"preamble,omitempty"`
|
||
TrailingNote int `json:"trailing_note,omitempty"`
|
||
MarkdownHeader int `json:"markdown_header,omitempty"`
|
||
LatinInsert int `json:"latin_insert,omitempty"`
|
||
BrokenWord int `json:"broken_word,omitempty"`
|
||
CJKLeak int `json:"cjk_leak,omitempty"`
|
||
Detail []string `json:"detail,omitempty"`
|
||
}
|
||
|
||
func (s sanitizerResult) total() int {
|
||
return s.Preamble + s.TrailingNote + s.MarkdownHeader + s.LatinInsert + s.BrokenWord + s.CJKLeak
|
||
}
|
||
|
||
// cosmeticOnly reports whether the ONLY classes that fired are the STRIPPABLE cosmetic ones
|
||
// (markdown header and/or CJK-leak) — the strip-and-export tier (D35.4a). A single substantive
|
||
// class (preamble/trailing-note/latin/broken-word) pulls the whole chunk into the skip tier: its
|
||
// contamination has no reliable removable boundary, so the output is not trusted even if a
|
||
// cosmetic leak also happens to be present. False when nothing fired.
|
||
func (s sanitizerResult) cosmeticOnly() bool {
|
||
return s.total() > 0 && s.Preamble == 0 && s.TrailingNote == 0 && s.LatinInsert == 0 && s.BrokenWord == 0
|
||
}
|
||
|
||
// summary is the human-readable disposition detail for a sanitizer flag (deterministic —
|
||
// Detail is sorted in sanitizeOutput). Kept short (the first two findings) for the log/row.
|
||
func (s sanitizerResult) summary() string {
|
||
if len(s.Detail) == 0 {
|
||
return "output-sanitizer defect"
|
||
}
|
||
if len(s.Detail) > 2 {
|
||
return strings.Join(s.Detail[:2], "; ") + fmt.Sprintf("; …(+%d)", len(s.Detail)-2)
|
||
}
|
||
return strings.Join(s.Detail, "; ")
|
||
}
|
||
|
||
// sanitizeOutput runs all six classes over one chunk's FINAL translated text.
|
||
//
|
||
// FOLD-FIRST (F1, D39.4 — adversarial-found gate bypass): ALL six classes DETECT over the
|
||
// export-normalised (recoverableFold-ed) text, not the raw text. Otherwise a FULLWIDTH variation of a
|
||
// signature slips past the ASCII regexes while exportNormalize still MANUFACTURES the ASCII defect in
|
||
// the shipped bytes: «Вот отредактированный перевод：»(U+FF1A fullwidth colon), «＃＃＃ Глава 5»
|
||
// (fullwidth hash), a trailing «Примечание：…», fullwidth-Latin — all read total=0 on the raw text yet
|
||
// export a real ASCII preamble/header/note/Latin leak (their ASCII twins ARE flagged, so it was a pure
|
||
// bypass; the pre-D39 v4 folded the whole fullwidth block and its re-check caught this — a regression).
|
||
// Only DETECTION folds; the strip/export paths (stripCosmetic/exportNormalize) still operate on the
|
||
// ORIGINAL text so they preserve the original/gloss bytes. Gloss spans are re-anchored consistently:
|
||
// the mask and detectCJKLeak both re-derive glossSegments over the SAME folded text they scan.
|
||
// recoverableFold is idempotent, so detectCJKLeak's own internal fold is a no-op on the folded input.
|
||
func sanitizeOutput(text string) sanitizerResult {
|
||
var r sanitizerResult
|
||
folded := recoverableFold(text)
|
||
// The two SCRIPT-based substantive classes (Latin insertion, broken word) additionally run with
|
||
// whitelisted CJK glosses masked out (D39.2 T2): a gloss's own pinyin/Han must not read as a leaked
|
||
// Latin phrase or a homoglyph token and drop the chunk. Masking is over the FOLDED text too.
|
||
glossMasked := maskGlossSpans(folded)
|
||
if det := detectLeadingPreamble(folded); det != "" {
|
||
r.Preamble++
|
||
r.Detail = append(r.Detail, det)
|
||
}
|
||
if det := detectTrailingNote(folded); det != "" {
|
||
r.TrailingNote++
|
||
r.Detail = append(r.Detail, det)
|
||
}
|
||
if n, det := detectMarkdownHeaders(folded); n > 0 {
|
||
r.MarkdownHeader = n
|
||
r.Detail = append(r.Detail, det...)
|
||
}
|
||
if det := detectLatinInsertion(glossMasked); det != "" {
|
||
r.LatinInsert++
|
||
r.Detail = append(r.Detail, det)
|
||
}
|
||
if n, det := detectBrokenWords(glossMasked); n > 0 {
|
||
r.BrokenWord = n
|
||
r.Detail = append(r.Detail, det...)
|
||
}
|
||
if n, det := detectCJKLeak(folded); n > 0 {
|
||
r.CJKLeak = n
|
||
r.Detail = append(r.Detail, det...)
|
||
}
|
||
sort.Strings(r.Detail)
|
||
return r
|
||
}
|
||
|
||
// --- 1. leading service preamble ------------------------------------------------
|
||
|
||
// leadingPreamblePatterns match a service preamble at the very START of the output — the
|
||
// «Вот отредактированный перевод: …» leak (gemini 6/6, exp12 §4.1). PRECISION over recall
|
||
// (адверсариальное ревью: a lax rule dropped clean chapter openings like «Вот такой вариант
|
||
// развития — самый вероятный.», «Вот его вариант: он молча ушёл.»). Two guards make it tight:
|
||
// 1. terminator is a COLON only — the em-dash «—» is ubiquitous in narration, so it is NOT
|
||
// a terminator (a service preamble is a labeled prefix ending in «:»);
|
||
// 2. the phrase must be a genuine TRANSLATION/EDIT service label, not merely «lead + common
|
||
// noun»: either an edit-adjective (отредактированн/исправленн/улучшенн) + a перевод/текст/
|
||
// вариант noun, or «перевод фрагмента/текста/…», or «ниже приведён … перевод/текст».
|
||
//
|
||
// Go RE2 \w/\b are ASCII-only, so Cyrillic uses [а-яё].
|
||
var leadingPreamblePatterns = []*regexp.Regexp{
|
||
// «Вот/Представляю/Привожу … отредактированный/исправленный перевод/текст/вариант …:»
|
||
// (the edit-adjective front-gate holds precision, so the tail-to-colon may be long — the
|
||
// gemini leak carries «… с соблюдением всех терминов из глоссария:»).
|
||
regexp.MustCompile(`(?is)^\s*(?:вот|представляю|привожу|держите)\s+[^\n:]{0,40}?(?:отредактированн|исправленн|улучшенн)[а-яё]+\s+(?:перевод|текст|вариант)[а-яё]*[^\n:]{0,120}?:`),
|
||
// «Отредактированный/Исправленный перевод/текст/вариант …:» (label at the very start)
|
||
regexp.MustCompile(`(?is)^\s*(?:отредактированн|исправленн|улучшенн)[а-яё]+\s+(?:перевод|текст|вариант)[а-яё]*[^\n:]{0,120}?:`),
|
||
// «(Вот) перевод фрагмента/текста/отрывка/главы/черновика:»
|
||
regexp.MustCompile(`(?is)^\s*(?:вот\s+)?перевод\s+(?:фрагмента|текста|отрывка|главы|черновика)\s*:`),
|
||
// «Ниже приведён/представлен … перевод/отредактированный текст:»
|
||
regexp.MustCompile(`(?is)^\s*ниже\s+(?:привед[её]н|представлен|дан|след)[а-яё]*\s+[^\n:]{0,40}?(?:перевод|отредактированн|текст)[а-яё]*[^\n:]{0,20}?:`),
|
||
}
|
||
|
||
func detectLeadingPreamble(text string) string {
|
||
t := strings.TrimSpace(text)
|
||
for _, re := range leadingPreamblePatterns {
|
||
if loc := re.FindStringIndex(t); loc != nil {
|
||
return "ведущая служебная преамбула: " + preview(strings.TrimSpace(t[:loc[1]]))
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// --- 2. trailing note / edit block ----------------------------------------------
|
||
|
||
// trailingNoteRE matches a note/edit meta-HEADER at line start: «Примечание:», «Заметки
|
||
// переводчика», «Комментарий:», «(прим. перев.)». PRECISION over recall (адверсариальное
|
||
// ревью: «Комментатор захлёбывался…», «Заметка белела на столе.», «Пояснения не требовалось.»,
|
||
// «Сноска объясняла обычай.» are ordinary narrative nouns/gerunds — NOT headers). So the header
|
||
// word MUST be followed by a COLON, or be «примечание/заметки/комментарий + переводчика/редактора»,
|
||
// or be the «(прим. перев./ред.)» marker. Only counted inside the trailing region (below).
|
||
var trailingNoteRE = regexp.MustCompile(`(?i)^[*_>\s-]{0,4}(?:(?:примечани[ея]|заметк[аи]|комментари[йяю]|пояснени[ея]|сноск[аи])\s*:|(?:примечани[ея]|заметк[аи]|комментари[йяю])\s+(?:переводчик|редактор)[а-яё]*|прим\.\s*(?:перев|ред)\.?)`)
|
||
|
||
// editMetaAnywhereRE matches UNAMBIGUOUS editor meta-commentary that is a defect wherever it
|
||
// appears (not only trailing): «Внесённые правки:», «Что изменено:», «Список правок:», and the
|
||
// gemini tail leak «Основные правки для справки: …» (D33.1а — the A5/5_1 defect the gate exists
|
||
// to catch, exp12:229: the leading-preamble rule caught the head form «Вот отредактированный
|
||
// перевод:» but NOT this trailing edit-summary). The generic «Изменения:/Правки:» alone were
|
||
// REMOVED (адверсариальное ревью — «Изменения — вот что пугало его.» / «Правка — дело тонкое.»
|
||
// are common narrative openers), keeping only editor-specific compounds that never occur in prose.
|
||
// «основн…правк» is gated to «… для справки» OR a colon so a bare narrative «Основные правки внёс
|
||
// редактор» (no summary label) does not fire.
|
||
var editMetaAnywhereRE = regexp.MustCompile(`(?im)^[*_>\s-]{0,4}(?:(?:внесённ|внесен)[а-яё]+\s+правк|основн[а-яё]+\s+правк[а-яё]*(?:\s+для\s+справк[а-яё]+|\s*:)|что\s+(?:было\s+)?(?:изменен|исправлен)|список\s+(?:правок|изменений)|список\s+внесённых)`)
|
||
|
||
func detectTrailingNote(text string) string {
|
||
if loc := editMetaAnywhereRE.FindStringIndex(text); loc != nil {
|
||
return "утёкший блок правок редактора: " + preview(strings.TrimSpace(firstLineAt(text, loc[0])))
|
||
}
|
||
lines := nonEmptyLines(text)
|
||
start := len(lines) - trailingRegionLines
|
||
if start < 0 {
|
||
start = 0
|
||
}
|
||
for _, ln := range lines[start:] {
|
||
if trailingNoteRE.MatchString(ln) {
|
||
return "хвостовой блок заметок/примечаний: " + preview(strings.TrimSpace(ln))
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// --- 3. markdown ### header -----------------------------------------------------
|
||
|
||
// markdownHeaderRE matches a markdown ATX header line («### Глава», «# Title»): 1–6 hashes,
|
||
// a space, then a LETTER or DIGIT. «#1» / «№5» / a lone «#» do not match (no space+text); and
|
||
// (D33.2) a decorative scene-break rendered with hashes — «# # #», «## ***», «### ---» — no
|
||
// longer matches, since the char after the hashes is punctuation, not a header word/number.
|
||
var markdownHeaderRE = regexp.MustCompile(`(?m)^\s{0,3}#{1,6}\s+[\p{L}\p{Nd}]`)
|
||
|
||
func detectMarkdownHeaders(text string) (int, []string) {
|
||
var det []string
|
||
n := 0
|
||
for _, ln := range strings.Split(text, "\n") {
|
||
if markdownHeaderRE.MatchString(ln) {
|
||
n++
|
||
det = append(det, "markdown-заголовок в выходе: "+preview(strings.TrimSpace(ln)))
|
||
}
|
||
}
|
||
return n, det
|
||
}
|
||
|
||
// --- 4. Latin-script insertion --------------------------------------------------
|
||
|
||
// detectLatinInsertion flags an untranslated Latin PHRASE (≥ latinPhraseWords Latin words
|
||
// separated only by SINGLE SPACES — a real «the quick brown fox», not an alphanumeric id) OR
|
||
// heavy Latin contamination (Latin ≥ latinHeavyShare of all letters AND ≥ latinHeavyTokens
|
||
// Latin tokens). Requiring space-adjacency is load-bearing: a hex/id run like «138fd5c384e3»
|
||
// or «a1b2c3d4» yields several single-letter Latin tokens with NO space between them, which
|
||
// must NOT read as a Latin phrase (a false positive the golden fixture's tags exposed). A
|
||
// handful of Latin proper names stays silent.
|
||
func detectLatinInsertion(text string) string {
|
||
toks := scriptTokens(text)
|
||
maxRun, run, heavyLatTokens := 0, 0, 0
|
||
for _, tk := range toks {
|
||
switch {
|
||
case tk.lat:
|
||
if tk.adjacent && run > 0 {
|
||
run++ // a Latin word separated from the previous Latin word by one space
|
||
} else {
|
||
run = 1 // phrase start (digit/punct separator or a non-Latin predecessor)
|
||
}
|
||
// Heavy-Latin count EXCLUDES capitalized/brand tokens (D33.2): «iPhone», «iPad»,
|
||
// «MacBook», «Google» are legit proper nouns, so a list of 5+ brands must not trip the
|
||
// heavy-share gate. A genuinely leaked English clause is lowercase past its first word,
|
||
// so it still clears latinHeavyTokens (and a real multi-word Latin PHRASE is caught by
|
||
// maxRun below, which still counts every Latin token).
|
||
if !hasUpperLatin(tk.text) {
|
||
heavyLatTokens++
|
||
}
|
||
if run > maxRun {
|
||
maxRun = run
|
||
}
|
||
case tk.cyr || tk.mixed:
|
||
run = 0
|
||
}
|
||
}
|
||
if maxRun >= latinPhraseWords {
|
||
return "латинская вставка (фраза из подряд идущих латинских слов)"
|
||
}
|
||
latinLetters, cyrLetters := 0, 0
|
||
for _, ru := range text {
|
||
if unicode.Is(unicode.Latin, ru) {
|
||
latinLetters++
|
||
} else if unicode.Is(unicode.Cyrillic, ru) {
|
||
cyrLetters++
|
||
}
|
||
}
|
||
total := latinLetters + cyrLetters
|
||
if total > 0 && heavyLatTokens >= latinHeavyTokens && float64(latinLetters)/float64(total) >= latinHeavyShare {
|
||
return "латиница-врезки: тяжёлая доля латиницы в ru-выходе"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// hasUpperLatin reports whether s contains an uppercase Latin letter — the brand/proper-noun
|
||
// signature («iPhone», «MacBook», «Google») excluded from the heavy-Latin token count (D33.2).
|
||
func hasUpperLatin(s string) bool {
|
||
for _, ru := range s {
|
||
if unicode.Is(unicode.Latin, ru) && unicode.IsUpper(ru) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// --- 5. broken word forms -------------------------------------------------------
|
||
|
||
// invalidSignRE matches Cyrillic sign bigrams that never occur in valid Russian: a hard/soft
|
||
// sign at a word START that is FOLLOWED BY A LETTER (a broken token like «ьзал»), or two signs
|
||
// adjacent (ъь/ьъ/ьь/ъъ). Requiring the following letter excludes a STANDALONE «ъ»/«ь» — a
|
||
// metalinguistic mention of the letter itself, «Буква ъ называлась ером.» (адверсариальное
|
||
// ревью nit). High precision otherwise: word-initial-sign-then-letter and doubled signs are
|
||
// impossible in well-formed text. \P{L} (non-letter), not RE2's ASCII-only \b.
|
||
var invalidSignRE = regexp.MustCompile(`(?i)(?:^|\P{L})[ъь][а-яё]|[ъь][ъь]`)
|
||
|
||
func detectBrokenWords(text string) (int, []string) {
|
||
var det []string
|
||
seen := map[string]bool{}
|
||
add := func(s string) {
|
||
if !seen[s] {
|
||
seen[s] = true
|
||
det = append(det, s)
|
||
}
|
||
}
|
||
// (a) invalid Cyrillic sign bigrams.
|
||
if invalidSignRE.MatchString(text) {
|
||
add("невалидная кириллическая биграмма (мягкий/твёрдый знак в начале слова или подряд)")
|
||
}
|
||
// (b) mixed Cyrillic+Latin within one letter-run token (homoglyph artifact).
|
||
// NOTE: the junction-duplication split detector («Первопред предок») was REMOVED (D33.1б).
|
||
// It fired identically on ordinary prose whose adjacent tokens merely share a ≥4-rune
|
||
// morphemic boundary («около колонны/колодца», «стало талой» — «около» is a frequent
|
||
// preposition → flag-storm on 25 legit chapters), while its only synthetic target had an
|
||
// overlap fraction (4/9) INDISTINGUISHABLE from those false positives (4/5): no threshold on
|
||
// overlap length or share separates them, and the canonical «Первок предок» was never caught
|
||
// anyway (recall gap — TestSanitizerBrokenWordRecallGap). Deterministic split-duplication with
|
||
// no invalid sign / homoglyph needs a morphology pass (Ф2), so it stays silent here.
|
||
for _, tk := range scriptTokens(text) {
|
||
if tk.mixed {
|
||
add("смешение кириллицы и латиницы в одном слове (гомоглиф): " + preview(tk.text))
|
||
}
|
||
}
|
||
return len(det), det
|
||
}
|
||
|
||
// --- 6. CJK-leak in the final --------------------------------------------------
|
||
|
||
// isCJKLeakRune reports whether r is a genuinely CONTENTLESS CJK glyph that has NO place in a Russian
|
||
// final AND cannot be recovered by the export normaliser: a Han ideograph, kana, or a CJK
|
||
// symbol/bracket/punctuation (U+3001–303F — «「」『』【】〜※»). It is ALWAYS consulted on
|
||
// recoverableFold-ed text (detectCJKLeak/stripCosmetic fold first), so the RECOVERABLE wrong-glyph
|
||
// forms are already gone before this runs: fullwidth ASCII «！１» folded to «!1», the ideographic
|
||
// space U+3000 folded to a plain space, and «、»/«。» mapped to «,»/«.». Those must NOT fire — the
|
||
// export contract silently fixes them (D39 слой 6); only the contentless class remains here. ACCEPTED
|
||
// NARROWING (precision over recall): the old code fired on the WHOLE U+FF00–FFEF block; width.Fold now
|
||
// recovers the common fullwidth-ASCII/symbol forms to real glyphs (so they no longer flag), but a few
|
||
// RARE legacy forms in that block (halfwidth hangul jamo, halfwidth arrows/symbols, the katakana
|
||
// middle dot) fold to non-CJK glyphs this predicate does not catch and ship un-flagged — an
|
||
// extremely improbable artifact in zh/ja→ru output, traded for not over-flagging the common case. In
|
||
// clean Russian prose every one of these is count-0, so the class is high precision. The intrinsic
|
||
// classify() already flags a ≥15%-CJK output as an ECHO (cjk_artifact, substantive) BEFORE the
|
||
// sanitizer runs, so this class only ever sees a SPARSE leak in otherwise-Russian text.
|
||
func isCJKLeakRune(r rune) bool {
|
||
switch {
|
||
case unicode.Is(unicode.Han, r):
|
||
return true
|
||
case unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r):
|
||
return true
|
||
case r >= 0x3001 && r <= 0x303F: // CJK symbols & punctuation, EXCLUDING U+3000 (ideographic space)
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// --- CJK-gloss whitelist (D39.2 T2, owner decision 16.07: keep glosses) ---------
|
||
|
||
// glossParenRE finds a candidate parenthesised CJK gloss: a Cyrillic/Latin HEAD letter, an optional
|
||
// short whitespace gap, then a single-level (NO nested paren, NO newline) «(...)» in ASCII OR fullwidth
|
||
// parens. The gap class covers a plain space, a NON-BREAKING space, and the CJK-native IDEOGRAPHIC space
|
||
// U+3000 (all bind a term to its gloss). U+3000 is load-bearing for detection↔strip CONSISTENCY (D39.4
|
||
// review finding 1): the FOLD-FIRST detector sees U+3000 folded to a plain space and recognises the
|
||
// gloss, but stripCosmetic/exportNormalize detect glosses on the ORIGINAL bytes — so without U+3000 in
|
||
// the gap class a U+3000-bound gloss would read CLEAN in detection yet be DESTROYED by the strip. The
|
||
// head requirement is load-bearing — a paren block with no letter head (line start, after punctuation, a
|
||
// bare «(羅漢拳)») does NOT match, so its Han stays a leak. Group 1 is the whole paren block (the
|
||
// protected span); isValidGloss then validates its inner content. RE2 \p classes are Unicode.
|
||
var glossParenRE = regexp.MustCompile(`[\p{Cyrillic}\p{Latin}][ \x{00A0}\x{3000}]{0,2}(\([^()\n]*\)|（[^（）\n]*）)`)
|
||
|
||
// Gloss bounds (D39.2 T2 + D39.4 F2): a whitelisted gloss is a short TERM with at most a proportional
|
||
// pinyin romanisation, NOT a leaked clause smuggled in parens. maxGlossCJKRunes bounds the CJK term
|
||
// (羅漢拳, 五行八卦掌); the F2 bounds keep the non-CJK content pinyin-plausible so «拳 the iron fist that
|
||
// breaks the sky» is NOT whitelisted. [Tunable: flagged to the orchestrator; raise if a real term is
|
||
// dropped in practice.]
|
||
const (
|
||
maxGlossCJKRunes = 6 // CJK chars in the term (九阳真经=4, 五行八卦掌=5)
|
||
maxGlossInnerRunes = 40 // whole inner-content backstop (a clause is long)
|
||
maxGlossLatinTokenRunes = 24 // one romanisation word ≈ 4×maxGlossCJKRunes: the JOINED pinyin of a ≤6-char term («jiǔyángzhēnjīng»=15, «wǔxíngbāguàzhǎng»=16) must fit, while a long English word («antidisestablishmentarianism»=28) exceeds it (D39.4 review finding 2)
|
||
)
|
||
|
||
// isValidGloss reports whether a matched paren block «(inner)» is a whitelisted CJK TERM gloss: inner
|
||
// carries 1..maxGlossCJKRunes CJK letters (Han/kana — the original term) plus at most a PROPORTIONAL
|
||
// pinyin romanisation (Latin tokens ≤ cjk+1, each lowercase and ≤maxGlossLatinTokenRunes), and NOTHING
|
||
// outside the gloss charset (CJK letters + Latin/digits/combining tone marks + the separators space «·»
|
||
// «'» «-»), within maxGlossInnerRunes total. A parenthetical with Cyrillic prose, CJK PUNCTUATION
|
||
// («、」»), a nested fullwidth paren, a too-long CJK run (a clause, not a term), or a non-proportional /
|
||
// uppercase / over-long Latin run (a leaked English phrase, F2) fails — so any CJK inside stays a leak
|
||
// (precision over recall: a false whitelist would smuggle a real leak past the detector to export).
|
||
func isValidGloss(block string) bool {
|
||
rs := []rune(block)
|
||
if len(rs) < 3 { // «(x)» is the minimum
|
||
return false
|
||
}
|
||
inner := rs[1 : len(rs)-1] // inner content, without the opening/closing paren
|
||
if len(inner) > maxGlossInnerRunes {
|
||
return false // a term gloss is short; a long run is a clause (F2)
|
||
}
|
||
cjk, latinToks, tokLen := 0, 0, 0
|
||
for _, r := range inner {
|
||
switch {
|
||
case unicode.Is(unicode.Han, r), unicode.Is(unicode.Hiragana, r), unicode.Is(unicode.Katakana, r):
|
||
cjk++
|
||
if tokLen > 0 { // a CJK char ends the current Latin token
|
||
latinToks++
|
||
tokLen = 0
|
||
}
|
||
case unicode.Is(unicode.Latin, r):
|
||
if unicode.IsUpper(r) {
|
||
return false // pinyin romanisation is lowercase; an uppercase Latin run reads as prose (F2)
|
||
}
|
||
tokLen++
|
||
if tokLen > maxGlossLatinTokenRunes {
|
||
return false // a token longer than any romanisation word → not pinyin (F2)
|
||
}
|
||
case unicode.IsDigit(r), unicode.Is(unicode.Mn, r):
|
||
// a pinyin tone digit / combining tone mark stays within the current Latin token
|
||
if tokLen > 0 {
|
||
tokLen++
|
||
}
|
||
case r == ' ' || r == '·' || r == '\'' || r == '-':
|
||
if tokLen > 0 { // an intra-gloss separator ends the current Latin token
|
||
latinToks++
|
||
tokLen = 0
|
||
}
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
if tokLen > 0 {
|
||
latinToks++
|
||
}
|
||
// The Latin tokens must be PROPORTIONAL to the term — pinyin transliterates the Han roughly
|
||
// one token per character (+1 slack) — so a 1-CJK head with a 7-word English clause is rejected (F2).
|
||
return cjk >= 1 && cjk <= maxGlossCJKRunes && latinToks <= cjk+1
|
||
}
|
||
|
||
// maskGlossSpans replaces every whitelisted gloss block with a single space, so the SUBSTANTIVE
|
||
// whole-text sanitizer classes that would otherwise mis-fire on a gloss's own bytes run gloss-blind: the
|
||
// Latin-insertion class must not read a space-separated PINYIN romanisation «(五行八卦掌 wu xing ba gua
|
||
// zhang)» as a leaked Latin phrase, and the broken-word class must not read a Han-glued pinyin token as a
|
||
// homoglyph — both would drop the whole chunk to a placeholder, defeating the "keep glosses" intent. The
|
||
// CJK-leak/strip/normalize paths use glossSegments directly (they must preserve the gloss bytes); these
|
||
// substantive classes only need the gloss GONE from their view, so a plain mask suffices. Pure.
|
||
func maskGlossSpans(text string) string {
|
||
spans := glossSpans(text)
|
||
if len(spans) == 0 {
|
||
return text
|
||
}
|
||
var b strings.Builder
|
||
b.Grow(len(text))
|
||
pos := 0
|
||
for _, sp := range spans {
|
||
b.WriteString(text[pos:sp[0]])
|
||
b.WriteByte(' ') // a single space breaks token adjacency and drops the gloss's pinyin/Han from view
|
||
pos = sp[1]
|
||
}
|
||
b.WriteString(text[pos:])
|
||
return b.String()
|
||
}
|
||
|
||
// glossSpans returns the byte ranges of the whitelisted CJK glosses in text (the paren blocks,
|
||
// including their parens), left-to-right and non-overlapping (FindAll semantics). Detected on the
|
||
// ORIGINAL text so the span bytes can be preserved verbatim (a fullwidth char that is PART of a gloss
|
||
// is not folded — owner decision 16.07). Two adjacent glosses «(拳) и (腿)» yield two spans.
|
||
func glossSpans(text string) [][2]int {
|
||
m := glossParenRE.FindAllStringSubmatchIndex(text, -1)
|
||
if m == nil {
|
||
return nil
|
||
}
|
||
var spans [][2]int
|
||
for _, idx := range m {
|
||
s, e := idx[2], idx[3] // group 1 = the paren block
|
||
if s < 0 {
|
||
continue
|
||
}
|
||
if isValidGloss(text[s:e]) {
|
||
spans = append(spans, [2]int{s, e})
|
||
}
|
||
}
|
||
return spans
|
||
}
|
||
|
||
// textSegment is one slice of a chunk split by glossSegments: a normal segment (folded/stripped/
|
||
// detected) or a gloss segment (preserved verbatim).
|
||
type textSegment struct {
|
||
text string
|
||
gloss bool
|
||
}
|
||
|
||
// glossSegments splits text into alternating normal / gloss segments on the whitelisted gloss spans, so
|
||
// the three export-contract functions treat a gloss the SAME way: normal segments are folded/detected/
|
||
// stripped, gloss segments are passed through untouched. The whole text is one normal segment when no
|
||
// gloss is present.
|
||
func glossSegments(text string) []textSegment {
|
||
spans := glossSpans(text)
|
||
if len(spans) == 0 {
|
||
return []textSegment{{text: text}}
|
||
}
|
||
var segs []textSegment
|
||
pos := 0
|
||
for _, sp := range spans {
|
||
if sp[0] > pos {
|
||
segs = append(segs, textSegment{text: text[pos:sp[0]]})
|
||
}
|
||
segs = append(segs, textSegment{text: text[sp[0]:sp[1]], gloss: true})
|
||
pos = sp[1]
|
||
}
|
||
if pos < len(text) {
|
||
segs = append(segs, textSegment{text: text[pos:]})
|
||
}
|
||
return segs
|
||
}
|
||
|
||
// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail per
|
||
// distinct run. It runs over the RECOVERABLE-FOLD of each NON-gloss segment (D39 слой 6): a fullwidth
|
||
// «１２３» or a «、» is a wrong-glyph rendering the export contract silently fixes, so it is NOT a leak —
|
||
// only a genuinely contentless Han/kana/CJK-bracket run is. A whitelisted CJK term gloss «(羅漢拳)» is
|
||
// skipped entirely (D39.2 T2 — owner keeps glosses). A single run («却有») is enough to fire — the class
|
||
// is COSMETIC, so the whole chunk is not lost: stripCosmetic removes the run and the remainder exports.
|
||
func detectCJKLeak(text string) (int, []string) {
|
||
var det []string
|
||
seen := map[string]bool{}
|
||
n := 0
|
||
for _, seg := range glossSegments(text) {
|
||
if seg.gloss {
|
||
continue // a whitelisted gloss is legitimate original-term content, not a leak
|
||
}
|
||
rs := []rune(recoverableFold(seg.text))
|
||
for i := 0; i < len(rs); {
|
||
if !isCJKLeakRune(rs[i]) {
|
||
i++
|
||
continue
|
||
}
|
||
j := i
|
||
for j < len(rs) && isCJKLeakRune(rs[j]) {
|
||
j++
|
||
}
|
||
n++
|
||
run := string(rs[i:j])
|
||
if !seen[run] {
|
||
seen[run] = true
|
||
det = append(det, "CJK-утечка в ru-выходе: "+preview(run))
|
||
}
|
||
i = j
|
||
}
|
||
}
|
||
return n, det
|
||
}
|
||
|
||
// --- deterministic strip of the COSMETIC classes (D35.4a export fix) -------------
|
||
|
||
// leadingMarkdownStripRE matches the leading markdown ATX prefix on a line (optional indent, 1–6
|
||
// hashes, blank(s)) directly before a header WORD/number — the same shape markdownHeaderRE fires
|
||
// on. It captures the first header rune so ReplaceAll keeps the heading text, dropping only the
|
||
// «### » artifact («### Глава 5» → «Глава 5»). A decorative «### ---» scene break (punctuation
|
||
// after the hashes) does NOT match, so it is left intact (and it never fired the class either).
|
||
var leadingMarkdownStripRE = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+([\p{L}\p{Nd}])`)
|
||
|
||
// --- export-contract normalization (D39 слой 6, D29.1a) --------------------------
|
||
|
||
// recoverableFold folds the RECOVERABLE wrong-glyph Unicode renderings a Russian final can carry into
|
||
// their intended characters — the shared primitive of the export contract. It is a WIDTH fold
|
||
// (width.Fold: fullwidth ASCII «！１２» → «!12», the ideographic space U+3000 → a plain space,
|
||
// halfwidth kana → fullwidth kana) plus the two CJK punctuation glyphs the width fold leaves alone
|
||
// («、» → «,», «。» → «.»). It reuses the x/text width.Fold primitive instead of the old hand-rolled
|
||
// «r-0xFEE0» arithmetic (L5-stripcosmetic-duplicates-nfkc) — but deliberately NOT full norm.NFKC,
|
||
// which would over-fold legitimate Russian typography (NFKC decomposes «…»→«...», «№²½»→…), a
|
||
// regression the golden strip contract forbids. Used by exportNormalize (every final), stripCosmetic
|
||
// (before dropping the contentless leak) and detectCJKLeak (which flags only what SURVIVES the fold —
|
||
// a genuinely contentless Han/kana/CJK-bracket, never a foldable glyph). Pure.
|
||
func recoverableFold(text string) string {
|
||
return ideographicPunctFold.Replace(width.Fold.String(text))
|
||
}
|
||
|
||
// ideographicPunctFold maps the two CJK punctuation glyphs the width fold does NOT touch to their
|
||
// ASCII equivalents (a «、»/«。» in a Russian final is an unambiguous wrong-glyph rendering of «,»/«.»).
|
||
var ideographicPunctFold = strings.NewReplacer("、", ",", "。", ".")
|
||
|
||
// exportNormalize is the export-contract normalization (D29.1a / target arch слой 6) applied to
|
||
// EVERY final text before it ships (chunkrun.ChunkOutcome.FinalText), clean OR flagged-stripped — the
|
||
// fix for a cosmetic defect being normalised on a flagged chunk but shipped RAW on a clean one
|
||
// (L5-normalization-fused-to-flag-path). It folds the recoverable wrong glyphs (recoverableFold),
|
||
// strips a stray combining stress mark on a Cyrillic base («источа́ло»→«источало»,
|
||
// L5-diacritics-class-lost), and trims trailing horizontal whitespace per line (never a newline, so
|
||
// paragraph structure is kept). It is a PURE, IDEMPOTENT projection over the final text: it never
|
||
// touches the wire, the stored checkpoint or the disposition — only the exported bytes — so it is
|
||
// NOT a snapshot/verdict input (recomputed each run; a change re-projects existing checkpoints for
|
||
// free, no re-bill). Deliberately conservative on whitespace: it does NOT collapse internal doubled
|
||
// spaces (a clean chunk's spacing is left as the model wrote it).
|
||
func exportNormalize(text string) string {
|
||
var b strings.Builder
|
||
b.Grow(len(text))
|
||
for _, seg := range glossSegments(text) {
|
||
if seg.gloss {
|
||
// A whitelisted CJK term gloss is preserved VERBATIM — including any fullwidth char that is
|
||
// part of the gloss, which is NOT folded (owner decision 16.07, D39.2 T2).
|
||
b.WriteString(seg.text)
|
||
continue
|
||
}
|
||
out := recoverableFold(seg.text)
|
||
// NFC composes a canonically-decomposed base+mark (a «й» stored as и+U+0306, a «ё» as е+U+0308)
|
||
// BEFORE the combining-strip below, so the strip only removes the NON-composable stress accents
|
||
// (U+0301 on a vowel), never corrupting «й»/«ё» into «и»/«е». NFC is canonical-only, so it leaves
|
||
// «…» and other compatibility characters intact (unlike NFKC).
|
||
out = norm.NFC.String(out)
|
||
out = stripCombiningOnCyrillic(out)
|
||
b.WriteString(out)
|
||
}
|
||
return trailingHSpaceRE.ReplaceAllString(b.String(), "")
|
||
}
|
||
|
||
// stripCombiningOnCyrillic drops a combining mark (unicode.Mn) that sits directly on a CYRILLIC base
|
||
// — the leaked stress-accent defect the exp12 flagman named («источа́ло» → «источало»,
|
||
// L5-diacritics-class-lost). Narrow and zero-FP for Russian prose (which never carries combining
|
||
// stress marks); a combining mark on a Latin base (a foreign name «é») is preserved. Runs AFTER NFKC
|
||
// (recoverableFold), which has already composed any canonically-composable base+mark.
|
||
func stripCombiningOnCyrillic(s string) string {
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
prevCyrillic := false
|
||
for _, r := range s {
|
||
if unicode.Is(unicode.Mn, r) {
|
||
if prevCyrillic {
|
||
continue // drop the stress accent on a Cyrillic vowel; leave prevCyrillic set
|
||
}
|
||
b.WriteRune(r)
|
||
continue
|
||
}
|
||
b.WriteRune(r)
|
||
prevCyrillic = unicode.Is(unicode.Cyrillic, r)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// stripCosmetic deterministically removes the two STRIPPABLE cosmetic leak classes — leading
|
||
// markdown headers and CONTENTLESS CJK-leak runs — and tidies the whitespace the removal leaves, so
|
||
// the remainder is readable. It is a PURE function (resume re-derives the identical export), invoked
|
||
// ONLY on output the sanitizer classified as cosmeticOnly (classifyOutput guarantees the result is
|
||
// non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
|
||
// sanitizer_stripped). It never crosses newlines when tidying, so paragraph structure is kept.
|
||
func stripCosmetic(text string) string {
|
||
// 1) markdown header prefixes → keep the heading text, drop the «#### » artifact. Line-leading, so
|
||
// it never touches a mid-line gloss span computed below.
|
||
out := leadingMarkdownStripRE.ReplaceAllString(text, "$1")
|
||
// 2+3) per segment: fold the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «１２３»→«123»,
|
||
// U+3000→space, «、»→«,» «。»→«.») via the shared export-contract fold (L5-stripcosmetic-duplicates-nfkc)
|
||
// and drop the contentless CJK-leak runes the fold cannot recover, each leaving a SINGLE SPACE so
|
||
// adjacent Russian words do not merge («нашёл特产древний»→«нашёл древний»). A whitelisted CJK term
|
||
// gloss «(羅漢拳)» has its TERM BYTES preserved (D39.2 T2 — owner keeps glosses), so its parens no
|
||
// longer empty out and its term is not dropped. [The step-4 whitespace tidy below runs over the
|
||
// reassembled string, so it may normalise STRAY whitespace INSIDE a malformed gloss («(羅漢拳 )» →
|
||
// «(羅漢拳)»); this is benign and does NOT diverge from what `tmctl translate`/`tmctl export` ship,
|
||
// since translate's stripped-export path is exportNormalize(stripCosmetic(x)) and Export reads that
|
||
// same derived checkpoint — both go through THIS function, so the shipped bytes agree.]
|
||
var b strings.Builder
|
||
b.Grow(len(out))
|
||
for _, seg := range glossSegments(out) {
|
||
if seg.gloss {
|
||
b.WriteString(seg.text)
|
||
continue
|
||
}
|
||
for _, r := range recoverableFold(seg.text) {
|
||
if isCJKLeakRune(r) {
|
||
b.WriteByte(' ')
|
||
} else {
|
||
b.WriteRune(r)
|
||
}
|
||
}
|
||
}
|
||
out = b.String()
|
||
// 4) tidy the seams the removal created, per line (horizontal whitespace only — never a
|
||
// newline): collapse doubled spaces, drop a bracket/quote pair the strip emptied («(羅漢拳)» →
|
||
// «()» → removed), drop a space now sitting before closing punctuation or after an opening
|
||
// bracket/quote (never-correct Russian spacing), and trim trailing blanks. ':' is deliberately
|
||
// NOT in the closing-punct set: a space-before-colon is legitimate in a Russian ratio/score/time
|
||
// («счёт 2 : 1») and must not be reformatted (adversarial review — false positive).
|
||
out = multiSpaceRE.ReplaceAllString(out, " ")
|
||
out = emptyBracketRE.ReplaceAllString(out, "")
|
||
out = spaceBeforePunctRE.ReplaceAllString(out, "$1")
|
||
out = spaceAfterOpenRE.ReplaceAllString(out, "$1")
|
||
out = multiSpaceRE.ReplaceAllString(out, " ") // an emptied bracket may leave a doubled space
|
||
out = trailingHSpaceRE.ReplaceAllString(out, "")
|
||
return out
|
||
}
|
||
|
||
var (
|
||
multiSpaceRE = regexp.MustCompile(`[ \t]{2,}`)
|
||
emptyBracketRE = regexp.MustCompile(`\([ \t]*\)|«[ \t]*»|\[[ \t]*\]`)
|
||
spaceBeforePunctRE = regexp.MustCompile(`[ \t]+([,.!?;…»)\]])`)
|
||
spaceAfterOpenRE = regexp.MustCompile(`([«(\[])[ \t]+`)
|
||
trailingHSpaceRE = regexp.MustCompile(`(?m)[ \t]+$`)
|
||
)
|
||
|
||
type scriptToken struct {
|
||
text string
|
||
cyr bool // Cyrillic-only
|
||
lat bool // Latin-only
|
||
mixed bool // both scripts in one letter run (homoglyph artifact)
|
||
adjacent bool // separated from the previous token by exactly a single space (junction check)
|
||
}
|
||
|
||
// scriptTokens splits text into maximal letter-run tokens (letters only; digits/hyphens/
|
||
// spaces/punctuation are separators), tagging Cyrillic-only / Latin-only / mixed-script and
|
||
// whether the token is separated from the previous token by exactly a single space
|
||
// (adjacency — a duplication artifact sits inside a phrase, not across punctuation/newline).
|
||
func scriptTokens(text string) []scriptToken {
|
||
var out []scriptToken
|
||
var b strings.Builder
|
||
hasLat, hasCyr := false, false
|
||
gap := "" // separator run since the previous token: "" none yet, " " one space, "x" other
|
||
flush := func() {
|
||
if b.Len() > 0 {
|
||
out = append(out, scriptToken{
|
||
text: b.String(),
|
||
cyr: hasCyr && !hasLat,
|
||
lat: hasLat && !hasCyr,
|
||
mixed: hasLat && hasCyr,
|
||
adjacent: gap == " ",
|
||
})
|
||
b.Reset()
|
||
gap = ""
|
||
}
|
||
hasLat, hasCyr = false, false
|
||
}
|
||
for _, ru := range text {
|
||
switch {
|
||
case unicode.Is(unicode.Latin, ru):
|
||
b.WriteRune(ru)
|
||
hasLat = true
|
||
case unicode.Is(unicode.Cyrillic, ru):
|
||
b.WriteRune(ru)
|
||
hasCyr = true
|
||
default:
|
||
flush()
|
||
if ru == ' ' && gap == "" {
|
||
gap = " "
|
||
} else {
|
||
gap = "x" // newline / punctuation / tab / a second space breaks adjacency
|
||
}
|
||
}
|
||
}
|
||
flush()
|
||
return out
|
||
}
|
||
|
||
// --- shared helpers -------------------------------------------------------------
|
||
|
||
func nonEmptyLines(text string) []string {
|
||
var out []string
|
||
for _, ln := range strings.Split(text, "\n") {
|
||
if strings.TrimSpace(ln) != "" {
|
||
out = append(out, ln)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// firstLineAt returns the line containing byte offset off.
|
||
func firstLineAt(text string, off int) string {
|
||
start := strings.LastIndexByte(text[:off], '\n') + 1
|
||
end := strings.IndexByte(text[off:], '\n')
|
||
if end < 0 {
|
||
return text[start:]
|
||
}
|
||
return text[start : off+end]
|
||
}
|