782 lines
37 KiB
Go
782 lines
37 KiB
Go
// Package checks holds the deterministic, $0 verdicts over one chunk's text: the output sanitizer
|
||
// (contamination classes + the export-contract normalization), the cheap style flaggers and the
|
||
// defect-class checkers, the excision coverage gate and the post-reflow regression guard.
|
||
//
|
||
// Every check here is a PURE function of its inputs — no store, no clock, no randomness, no LLM —
|
||
// so a resumed run re-derives an identical verdict without re-billing. The package reports FACTS
|
||
// (this class fired, this metric was breached); mapping a fact onto a chunk disposition is the
|
||
// driver's job, so no disposition/flag vocabulary is imported here. All pair- and target-specific
|
||
// patterns and word lists arrive as DATA (a *lang.Pack for the pair, lang.TargetChecks for the
|
||
// target); a book with no pack runs the language-specific checkers inert rather than skipping them.
|
||
package checks
|
||
|
||
import (
|
||
"fmt"
|
||
"sort"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"textmachine/backend/internal/lang"
|
||
"textmachine/backend/internal/text"
|
||
)
|
||
|
||
// cheapgates.go: four cheap, deterministic post-check flaggers on the FINAL chunk text (04-unhappy
|
||
// §6/§9, plan v3). They are OBSERVABILITY, never hard gates (like the glossary post-check's default
|
||
// flagger mode): a hit is recorded in the retrieval-state and surfaced in the report / chapter
|
||
// passport, but it never changes a chunk disposition or costs an LLM call. All four are pure and
|
||
// deterministic (no time/rand, sorted detail) so a resume re-derives identical counts.
|
||
//
|
||
// 1. dialogue-dash linter (Rosenthal §47–52): direct speech marked with straight quotes or a
|
||
// hyphen/en-dash instead of the em-dash on a new line, or a chunk mixing the two styles.
|
||
// 2. yofikator: inconsistent ё — the SAME word spelled both with ё and with е (Пётр/Петр), or a
|
||
// brief ё-policy (all-yo / all-e) violated.
|
||
// 3. translit-interjection blocklist: untranslated JP/EN fillers («ара-ара», «маа», «хмф»,
|
||
// «нани», «ауч», «упс») left in the Russian output (04-unhappy §9); per-project Allowlist.
|
||
// 4. 万/億 magnitude gate: a CJK myriad/hundred-million magnitude in the source whose order is not
|
||
// represented on the Russian side (三万 → «три миллиона» is a 100× error, 04-unhappy §9).
|
||
//
|
||
// FALSE POSITIVES are the main risk (quotations, stylization, homographs), so each rule is tuned
|
||
// for PRECISION over recall: it fires only on a high-confidence signal and stays silent on the
|
||
// ambiguous middle. The unit tests assert both firing AND non-firing.
|
||
|
||
// CheapGateVersion versions the rules above. It is folded into the job snapshot (like
|
||
// classifierVersion): editing any rule is a loud --resnapshot, so the reported style-flag counts
|
||
// never shift silently between runs under the same snapshot. [D15.2 note: this is a VERDICT
|
||
// version — once content-addressed resume lands it moves to verdictSnapshotID and a rule edit
|
||
// re-classifies free instead of re-paying the book.]
|
||
//
|
||
// v2 (D20.4, package №3) closes three adversarial-review false positives, so the bump is LOUD by
|
||
// design (counts shift): (1) a chevron CITATION at line start no longer counts as dialogue-style
|
||
// mixing — only the «…», — attribution shape does; (2) a source 万/億 magnitude is no longer
|
||
// false-flagged against a STRAY output integer (a year, a count) — only a mismatching magnitude
|
||
// WORD triggers, bare integers can merely confirm; (3) same mechanism covers a 万-in-a-name +
|
||
// unrelated output number. No live book has run under v1 (D18 acceptance pending), so the re-pin is free.
|
||
//
|
||
// v3 (WS5, R4) adds the defect-class checkers DC1 (时辰 double-hour units), DC2 (千万/数十万 magnitude
|
||
// scale) and DC6 (register negative-list — the zh-ru pack, checkers.go). They are observability
|
||
// (never a disposition), tuned precision-over-recall; the bump is a loud --resnapshot as the discipline
|
||
// requires (a rule/pack edit shifts recorded counts). They fire 0 on a non-zh source / non-register text.
|
||
// Ш-2 (see text/norm.go): the cheap style/DC checkers classify via unicode predicates + NFC folding, so a
|
||
// toolchain Unicode bump that shifts a class is a loud --resnapshot rather than a silent count change.
|
||
//
|
||
// v4 (pack-13) adds three GENERAL observability checkers (never a disposition, tuned precision over
|
||
// recall): a 成-percent scale checker (六成六=66% mis-rendered as a decimal fraction), a Latin-residue
|
||
// checker (a whole Latin word left in the Russian output), and a broken-word checker (a Russian word
|
||
// ending in the impossible «-йть»). All are language-general — no book-specific word lists. Editing a rule
|
||
// shifts the recorded counts, so the version bump is a loud --resnapshot; they fire 0 on a non-zh source /
|
||
// clean Russian output (the golden fixture stays style_flags=0).
|
||
// v5 (mini-run + live verification, 25–26.07) closes ONE measured false-positive class and deliberately
|
||
// leaves two others alone. The dialogue-dash MIXING rule now counts a chevron line ONLY when its
|
||
// attribution names a SPEECH verb and is not qualified into a thought («пробормотал ПРО СЕБЯ»): Russian
|
||
// sets inner speech in «…» beside spoken «—» on purpose, so a chunk carrying both is correctly typeset
|
||
// prose — the rule was flagging the convention it exists to protect (8 of 8 hits on the 10-chapter run).
|
||
//
|
||
// The POLARITY is the load-bearing choice and it was picked by measurement, not taste. The first attempt
|
||
// asked "is this a THOUGHT?" against a list of thought verbs; a live 2-chapter run produced «понимал» and
|
||
// «размышлял» within minutes, and that list has no natural boundary — every miss becomes a false flag on
|
||
// good prose. Asking "is this SPOKEN?" needs a small closed list, and every miss becomes silence on a real
|
||
// clash: the failure mode the gate's precision-over-recall bias actually wants. Both lists are TARGET data
|
||
// (`speech_verb`, `inner_marker`), so a target shipping neither runs the mixing rule inert.
|
||
// Measured on the real exports, per chunk: 10-chapter mini-run 8 → 0, 2-chapter verification 1 → 0, while
|
||
// genuine spoken-chevron mixing still fires (pinned by test).
|
||
//
|
||
// NOT changed, on purpose, after the same run flagged them (both examined, both written, both reverted):
|
||
// - 成-percent: the two live hits («три десятых» for 三成, «сорок четыре сотых» for 四成四) carry the
|
||
// RIGHT value in the wrong FORM, but the sub-check's own ratified fixtures treat the form as the
|
||
// defect, and the one blunt discriminator available (fire on digits only) would silence the genuine
|
||
// 100× shape «шесть и шесть десятых» = 6.6, which is spelled out too. A correct fix compares the
|
||
// rendered VALUE with the source percentage and therefore needs target numeral-word data — a design,
|
||
// not a patch. Left firing rather than narrowed by guesswork.
|
||
// - DC2 千万: the live hit (杀了千万人 → «тысячи и тысячи людей») is shape-identical to the ratified TRUE
|
||
// positive (千万生灵 → «тысячи жизней»). Hyperbole and magnitude are not separable offline here — the
|
||
// rule's header says so (§5-A4) — so the hit is the class's accepted ambiguity, not an implementation
|
||
// bug. Any coefficient-based veto kills the true positive with the false one.
|
||
//
|
||
// v7 (checker package, D39.39) lands three measured checker-pack changes whose recorded counts shift, so the
|
||
// bump is a loud --resnapshot as the discipline requires: chevron attribution join extended to «X!»/«X?»/«X…»
|
||
// (#3), Latin-residue now flags Title/ALL-CAPS leaks with an allowlist (#6, «Cultivation»/«BANK»), and the
|
||
// unit/scale detectors expose the Р2 hard(value)/soft(form) split (UnitScaleHard/Soft) without changing Total.
|
||
const CheapGateVersion = "cheapgate-v7-checkerpack-attribution+latincaps+p2split+combmark+u" + unicode.Version
|
||
|
||
// CheapGateConfig carries the brief-derived knobs: the ё-policy and the per-project Allowlist of
|
||
// surfaces that look like a blocklisted interjection but are legitimate here (e.g. a character
|
||
// named «Ара»). Both come from book.yaml.
|
||
type CheapGateConfig struct {
|
||
YoPolicy string // "auto" (inconsistency only) | "all-yo" | "all-e"
|
||
Allowlist map[string]bool // lower-cased surfaces exempt from the interjection blocklist
|
||
// RegressionEnabled turns on the post-reflow regression guard (D38, regressionguard.go): an
|
||
// OPT-IN observability flagger (draft→final length collapse + number drift) folded into this
|
||
// result. Off → the two regression fields stay 0 and the output is byte-identical to before.
|
||
RegressionEnabled bool
|
||
// checkers is the compiled WS5/pack-13 checker spec (pair-14 data-out): the pair's DETECTION patterns +
|
||
// lookup tables (from the pair langpack) plus the target-general lists (from embedded target data),
|
||
// resolved once per run. nil for a bare config or a book with no data → the language-specific checkers
|
||
// run inert (fire 0, the no-pack golden path); the general Latin-residue check needs no spec.
|
||
Checkers *Checkers
|
||
// RegisterBlocklist is the book's DC6 out-of-register target lexis (book.yaml register_blocklist, D39.79
|
||
// Q4): genre-wrong fairy-tale/chancery words for THIS book. It lives with the book, not the pair pack, so
|
||
// a second zh→ru book of another genre ships its own (or none, running DC6 inert). Lower-cased on load.
|
||
RegisterBlocklist []string
|
||
}
|
||
|
||
// CheapGateResult is the per-chunk outcome: a count per flagger plus human-readable detail lines
|
||
// (deterministic order) for the report. Total() is what the passport surfaces.
|
||
type CheapGateResult struct {
|
||
DialogueDash int `json:"dialogue_dash,omitempty"`
|
||
YoInconsistent int `json:"yo,omitempty"`
|
||
TranslitInterj int `json:"translit_interj,omitempty"`
|
||
NumberMagnitude int `json:"number_magnitude,omitempty"`
|
||
// LengthCollapse / NumberDrift are the opt-in post-reflow regression guard (D38,
|
||
// regressionguard.go), folded into this observability result. They stay 0 unless
|
||
// cfg.RegressionEnabled, so a book that does not enable the guard serialises identically.
|
||
LengthCollapse int `json:"length_collapse,omitempty"`
|
||
NumberDrift int `json:"number_drift,omitempty"`
|
||
// DC1TimeUnits / DC2Magnitude / DC6Register are the WS5 defect-class checkers (checkers.go),
|
||
// observability like the others. Zero on a non-zh source / non-register final (the golden fixture).
|
||
DC1TimeUnits int `json:"dc1_time_units,omitempty"`
|
||
DC2Magnitude int `json:"dc2_magnitude,omitempty"`
|
||
DC6Register int `json:"dc6_register,omitempty"`
|
||
// PercentScale / LatinResidue / BrokenWord are the pack-13 general checkers (checkers.go),
|
||
// observability like the others. Zero on a clean Russian output / non-zh source.
|
||
PercentScale int `json:"percent_scale,omitempty"`
|
||
LatinResidue int `json:"latin_residue,omitempty"`
|
||
BrokenWord int `json:"broken_word,omitempty"`
|
||
Detail []string `json:"detail,omitempty"`
|
||
}
|
||
|
||
func (c CheapGateResult) Total() int {
|
||
return c.DialogueDash + c.YoInconsistent + c.TranslitInterj + c.NumberMagnitude + c.LengthCollapse + c.NumberDrift +
|
||
c.DC1TimeUnits + c.DC2Magnitude + c.DC6Register + c.PercentScale + c.LatinResidue + c.BrokenWord
|
||
}
|
||
|
||
// UnitScaleHard / UnitScaleSoft split the unit/scale detectors by the Р2 contract (D39.63): HARD = a wrong
|
||
// VALUE (DC1 double-hour count-as-hours, exact; NumberMagnitude, a magnitude word of the wrong ORDER, e.g.
|
||
// 三万→«три миллиона»); SOFT = an ambiguous/unconverted FORM (DC2 千万→«тысячи», hyperbole-ambiguous;
|
||
// PercentScale, correct value in fraction form). Total() is unchanged — this is classification for the report
|
||
// and a future hard gate (backlog 12), not new detection. Rationale/metrics: package report.
|
||
func (c CheapGateResult) UnitScaleHard() int { return c.DC1TimeUnits + c.NumberMagnitude }
|
||
func (c CheapGateResult) UnitScaleSoft() int { return c.DC2Magnitude + c.PercentScale }
|
||
|
||
// RunCheapGates runs the four always-on style flaggers over one chunk's source and FINAL text, plus
|
||
// (opt-in) the draft→final regression guard. `draft` is the first-stage translator output (== final
|
||
// when there is no distinct reflow stage, so the guard then trivially never fires).
|
||
func RunCheapGates(source, draft, final string, cfg CheapGateConfig) CheapGateResult {
|
||
var r CheapGateResult
|
||
n, det := lintDialogueDash(final, cfg.Checkers)
|
||
r.DialogueDash, r.Detail = n, append(r.Detail, det...)
|
||
n, det = cfg.Checkers.lintYofikation(final, cfg.YoPolicy)
|
||
r.YoInconsistent = n
|
||
r.Detail = append(r.Detail, det...)
|
||
n, det = cfg.Checkers.lintTranslitInterjections(final, cfg.Allowlist)
|
||
r.TranslitInterj = n
|
||
r.Detail = append(r.Detail, det...)
|
||
n, det = cfg.Checkers.lintNumberMagnitude(source, final)
|
||
r.NumberMagnitude = n
|
||
r.Detail = append(r.Detail, det...)
|
||
// WS5 defect-class checkers (DC1/DC2/DC6, checkers.go) — src↔target observability flaggers. All patterns
|
||
// + tables are pair langpack DATA (pair-14 data-out); the spec is nil/inert for a no-pack book → fire 0.
|
||
n, det = cfg.Checkers.lintTimeUnits(source, final)
|
||
r.DC1TimeUnits = n
|
||
r.Detail = append(r.Detail, det...)
|
||
n, det = cfg.Checkers.lintMagnitudeScale(source, final)
|
||
r.DC2Magnitude = n
|
||
r.Detail = append(r.Detail, det...)
|
||
n, det = cfg.Checkers.lintRegisterLexicon(final, cfg.RegisterBlocklist)
|
||
r.DC6Register = n
|
||
r.Detail = append(r.Detail, det...)
|
||
// pack-13 general checkers (checkers.go): percent scale (pair data), Latin residue (language-general),
|
||
// broken word (target data).
|
||
n, det = cfg.Checkers.lintPercentScale(source, final)
|
||
r.PercentScale = n
|
||
r.Detail = append(r.Detail, det...)
|
||
n, det = lintLatinResidue(final, cfg.Allowlist)
|
||
r.LatinResidue = n
|
||
r.Detail = append(r.Detail, det...)
|
||
n, det = cfg.Checkers.lintBrokenWord(final)
|
||
r.BrokenWord = n
|
||
r.Detail = append(r.Detail, det...)
|
||
if cfg.RegressionEnabled {
|
||
rg := runRegressionGuard(draft, final)
|
||
r.LengthCollapse = rg.LengthCollapse
|
||
r.NumberDrift = rg.NumberDrift
|
||
r.Detail = append(r.Detail, rg.Detail...)
|
||
}
|
||
return r
|
||
}
|
||
|
||
// --- 1. dialogue-dash linter ----------------------------------------------------
|
||
|
||
// lintDialogueDash flags direct-speech lines opened with the wrong marker. Russian direct speech
|
||
// takes an EM-dash «—» at the line start; MTL output leaves a straight ASCII quote or a plain
|
||
// hyphen/en-dash. It fires per offending line, and additionally when a chunk MIXES em-dash speech
|
||
// with chevron-quote «…» speech (a within-chapter style clash). Precision guards: a hyphen only
|
||
// counts as a mis-set dash when followed by a space and a letter (so a hyphenated word wrap or a
|
||
// «- 1» list item does not fire); chevron lines count only under mixing (a chunk that uses «…» for
|
||
// speech throughout may be a deliberate style, but mixing it with dashes is an inconsistency).
|
||
func lintDialogueDash(text string, c *Checkers) (int, []string) {
|
||
var emDash, hyphenLike, chevronSpeech, inverseMarker int
|
||
var detail []string
|
||
for _, line := range strings.Split(text, "\n") {
|
||
t := strings.TrimLeft(line, " \t ")
|
||
rs := []rune(t)
|
||
if len(rs) == 0 {
|
||
continue
|
||
}
|
||
switch rs[0] {
|
||
case '—': // em-dash: the correct Russian dialogue marker — count only true speech shape
|
||
if dialogueShape(rs[1:]) { // «— 15 минут спустя» (dash+space+digit, a scene break) is NOT dialogue (self-review)
|
||
emDash++
|
||
// k4_inverse (D39.64 §5.5): a thought typeset as spoken — a dash line whose ATTRIBUTION (after
|
||
// the «, —» join) carries an inner-speech marker («вздохнул ПРО СЕБЯ»). Scoped to the attribution,
|
||
// NOT the whole line, so a spoken line that merely SAYS «про себя» inside the reply is not flagged
|
||
// (adversarial-review false positive). Mirror of the chevron inner-marker veto; inert without data.
|
||
if attr := speechAttribution(t); attr != "" && c.lineHasInnerMarker(attr) {
|
||
detail = append(detail, "inner speech typeset as spoken dialogue (inner-speech marker in a dash line's attribution): "+preview(t))
|
||
inverseMarker++
|
||
}
|
||
}
|
||
case '"': // straight ASCII quote leading a line → Russian typography never uses these → MTL artifact
|
||
if quoteShape(rs[1:]) {
|
||
detail = append(detail, "direct speech opened with a straight quote \" instead of a dash: "+preview(t))
|
||
hyphenLike++ // counted in the offending total
|
||
}
|
||
case '-', '–': // hyphen / en-dash where an em-dash belongs
|
||
if dialogueShape(rs[1:]) {
|
||
detail = append(detail, "a line using a hyphen/short dash instead of the long «—»: "+preview(t))
|
||
hyphenLike++
|
||
}
|
||
case '«': // chevron-led line counts toward mixing ONLY as SPOKEN chevron dialogue
|
||
// D20.4 FP: a «-citation at line start does not count. Mini-run FP (25.07): neither does a
|
||
// THOUGHT. Russian typography sets inner speech in chevrons and speech aloud in dashes ON
|
||
// PURPOSE, so a chunk carrying both is correctly typeset prose, not a style clash — this
|
||
// rule was flagging the convention it exists to protect (8 of 8 hits on the 10-chapter
|
||
// mini-run were thoughts). The line therefore counts only when its attribution names a
|
||
// SPEECH verb (target data), i.e. an unrecognised attribution is silence, not a flag.
|
||
if chevronSpeechShape(rs) && c.isSpokenChevronLine(t) {
|
||
chevronSpeech++
|
||
}
|
||
}
|
||
}
|
||
n := hyphenLike + inverseMarker
|
||
if emDash > 0 && chevronSpeech > 0 {
|
||
n += chevronSpeech
|
||
detail = append(detail, fmt.Sprintf("mixed direct-speech styles in the chunk: %d lines with «—» and %d with «…»", emDash, chevronSpeech))
|
||
}
|
||
return n, detail
|
||
}
|
||
|
||
// lineHasInnerMarker reports whether the line carries one of the target's inner-speech markers (target
|
||
// data — «про себя», «мысленно», «себе под нос») as a WHOLE word/phrase, so «в уме» does NOT match inside
|
||
// «в умении» (a substring false positive the mixing rule must not inherit). Nil-safe / inert without data.
|
||
func (c *Checkers) lineHasInnerMarker(line string) bool {
|
||
if c == nil {
|
||
return false
|
||
}
|
||
low := []rune(strings.ToLower(line))
|
||
for _, m := range c.innerMarker {
|
||
if containsWholeWordPhrase(low, []rune(m)) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// containsWholeWordPhrase reports whether phrase occurs in low bounded by a non-word rune (or an edge) on
|
||
// both OUTER ends; a multi-word marker's internal spaces match literally. Mirrors the interjection check's
|
||
// whole-word rule (isWordRune boundary; the register check uses the target's isTargetWordLetter, same effect
|
||
// for Cyrillic), so «в уме» is not found inside «в умении».
|
||
func containsWholeWordPhrase(low, phrase []rune) bool {
|
||
n := len(phrase)
|
||
if n == 0 || n > len(low) {
|
||
return false
|
||
}
|
||
for i := 0; i+n <= len(low); i++ {
|
||
if !text.RunesEqual(low[i:i+n], phrase) {
|
||
continue
|
||
}
|
||
if (i == 0 || !isWordRune(low[i-1])) && (i+n == len(low) || !isWordRune(low[i+n])) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// speechAttribution returns the IMMEDIATE attribution clause of a dash-dialogue line: the text between the
|
||
// FIRST speech→attribution join (a mark , . ! ? … then a dash) and the NEXT such mark (or end of line). Both
|
||
// bounds are load-bearing. Taking the FIRST join excludes a marker in the leading SPOKEN clause; cutting at
|
||
// the next mark excludes a marker in a FOLLOWING clause — a continued reply after «. —» OR a bare narration
|
||
// sentence «. Про себя же он…» that has no second dash at all. So «…гу, — вздохнул про себя Фан Юань. — В
|
||
// своё время…» yields «вздохнул про себя Фан Юань», and the inner-marker test sees only the attribution
|
||
// itself — closing the k4_inverse false positives (a spoken line that merely SAYS «про себя»; the homograph
|
||
// «про себя»=«о себе» in a trailing clause; a narration tail). "" when the line carries no inline attribution.
|
||
//
|
||
// RECALL CEILING of the rule, in the contract (mirror of #3's Р4 note, not a data gap): cutting at the NEXT
|
||
// mark also hides a marker that sits AFTER an internal comma/ellipsis WITHIN one attribution — «— …гу, —
|
||
// сказал он, про себя ругаясь» yields «сказал он», so the inner «про себя» is invisible and the inverse
|
||
// thought stays unflagged. That FN is a ceiling of the two-sided cut itself, not of the corpus; widening the
|
||
// segment to recover it would re-open the trailing-clause false positives the cut exists to close.
|
||
func speechAttribution(line string) string {
|
||
rs := []rune(line)
|
||
for k := 0; k+1 < len(rs); k++ {
|
||
if !isSentencePunct(rs[k]) {
|
||
continue
|
||
}
|
||
j := k + 1
|
||
for j < len(rs) && isInlineSpace(rs[j]) {
|
||
j++
|
||
}
|
||
if j >= len(rs) || !(rs[j] == '—' || rs[j] == '–' || rs[j] == '-') {
|
||
continue
|
||
}
|
||
start := j + 1
|
||
end := start
|
||
for end < len(rs) && !isSentencePunct(rs[end]) {
|
||
end++
|
||
}
|
||
return strings.TrimSpace(string(rs[start:end]))
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// isSentencePunct reports whether r bounds an attribution segment: a comma (the «, —» join), a full stop, or
|
||
// a terminal ! ? … . The same set opens a join (before a dash) and closes the attribution (the next such mark).
|
||
func isSentencePunct(r rune) bool {
|
||
switch r {
|
||
case ',', '.', '!', '?', '…':
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// chevronSpeechShape reports whether a chevron line is a spoken-dialogue turn, not a citation/title. Signal:
|
||
// a closing » with an attribution dash — «Реплика», — (comma join) OR «Реплика!»/«?»/«…» — (exclamatory reply,
|
||
// comma omitted, #3/D39.39). A flat «X» — (a letter before », no comma) is a citation copula and is rejected —
|
||
// the D20.4 false positive the comma once guarded. isSpokenChevronLine (attribution verb) is the second gate.
|
||
// Р4 recall CEILING ~0.22 by construction: 24 of the corpus replies carry NO attribution dash this shape can
|
||
// latch onto (structurally invisible), capping recall — measured K4b 0.109→0.217. Label metrics: package report.
|
||
func chevronSpeechShape(rs []rune) bool {
|
||
i := 1 // rs[0] is '«'
|
||
for i < len(rs) && isInlineSpace(rs[i]) {
|
||
i++
|
||
}
|
||
if i >= len(rs) || !unicode.IsLetter(rs[i]) {
|
||
return false // «» empty or «123…» — not spoken content
|
||
}
|
||
for ; i < len(rs); i++ {
|
||
if rs[i] != '»' {
|
||
continue
|
||
}
|
||
j := i + 1
|
||
hasComma := j < len(rs) && rs[j] == ','
|
||
if hasComma {
|
||
j++
|
||
}
|
||
for j < len(rs) && isInlineSpace(rs[j]) {
|
||
j++
|
||
}
|
||
if j >= len(rs) || !(rs[j] == '\u2014' || rs[j] == '\u2013' || rs[j] == '-') {
|
||
continue
|
||
}
|
||
// comma join = attributed reply; else require a sentence-final mark inside the quote (exclamatory
|
||
// reply). A bare «X» — is a citation copula → keep scanning, never accept.
|
||
if hasComma || sentenceFinalBefore(rs, i) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// isInlineSpace reports whether r is an inline space (ASCII space or NBSP) — the whitespace the dialogue
|
||
// shapes skip around a marker. Thin space U+2009 is deliberately NOT included (an inherited HEAD convention,
|
||
// unchanged per row 93): a marker set off by a thin space would not be skipped — a known, unmeasured limitation.
|
||
func isInlineSpace(r rune) bool { return r == ' ' || r == '\u00a0' }
|
||
|
||
// sentenceFinalBefore reports whether the rune just before the closing » at rs[i] is a sentence-final mark:
|
||
// ! ? … or an ASCII ellipsis of TWO+ dots — only the last two are inspected, so «..» and «...» both pass (the
|
||
// code and this comment reconciled to two, row 93; the «..»-behaviour is frozen by the label baseline). It is
|
||
// the signature of an exclamatory/interrogative/trailing-off reply, as opposed to a flat citation.
|
||
func sentenceFinalBefore(rs []rune, i int) bool {
|
||
if i == 0 {
|
||
return false
|
||
}
|
||
switch rs[i-1] {
|
||
case '!', '?', '\u2026':
|
||
return true
|
||
case '.':
|
||
return i >= 2 && rs[i-2] == '.'
|
||
}
|
||
return false
|
||
}
|
||
|
||
// dialogueShape reports whether the runes after a leading marker look like spoken text: an optional
|
||
// space then a letter. It filters out non-dialogue leading dashes (word wraps, «-1», bare marks).
|
||
func dialogueShape(after []rune) bool {
|
||
i := 0
|
||
for i < len(after) && (after[i] == ' ' || after[i] == ' ') {
|
||
i++
|
||
}
|
||
if i == 0 {
|
||
return false // a marker glued to the next glyph (a hyphenated fragment), not «— реплика»
|
||
}
|
||
return i < len(after) && unicode.IsLetter(after[i])
|
||
}
|
||
|
||
// quoteShape reports whether the runes after a leading quote look like spoken text: an OPTIONAL
|
||
// space then a letter («"Привет»). Unlike a dash, a quote sits directly on the first word, so no
|
||
// space is required.
|
||
func quoteShape(after []rune) bool {
|
||
i := 0
|
||
for i < len(after) && (after[i] == ' ' || after[i] == ' ') {
|
||
i++
|
||
}
|
||
return i < len(after) && unicode.IsLetter(after[i])
|
||
}
|
||
|
||
// --- 2. yofikator ---------------------------------------------------------------
|
||
|
||
// lintYofikation flags inconsistent ё. Default ("auto"): the same word appears BOTH with ё and, as
|
||
// a separate token, with its exact ё→е form (Пётр/Петр) — excluding the target's homograph whitelist
|
||
// (е-spellings that are DISTINCT words from their ё-counterpart, все≠всё …; TARGET data, lang.TargetChecks
|
||
// "yo_homograph" — pair-14 data-out, so the ё↔е FOLD stays as generic orthography and only the wordlist is
|
||
// data). Policy "all-e": any ё present is a violation. Policy "all-yo": same signal as auto. Full "every
|
||
// word that SHOULD have ё" enforcement needs a ё-dictionary (deferred, B-tier). Inert if no target data.
|
||
func (c *Checkers) lintYofikation(out, policy string) (int, []string) {
|
||
if c == nil {
|
||
return 0, nil
|
||
}
|
||
words := c.tokenizeWords(out)
|
||
if policy == "all-e" {
|
||
seen := map[string]bool{}
|
||
var det []string
|
||
n := 0
|
||
for _, w := range words {
|
||
if strings.ContainsRune(w, 'ё') && !seen[w] {
|
||
seen[w] = true
|
||
n++
|
||
det = append(det, "ё under the all-e policy: "+w)
|
||
}
|
||
}
|
||
sort.Strings(det)
|
||
return n, det
|
||
}
|
||
// auto / all-yo: detect a word present in BOTH its ё-form and its е-form.
|
||
present := map[string]bool{}
|
||
for _, w := range words {
|
||
present[w] = true
|
||
}
|
||
seenPair := map[string]bool{}
|
||
var det []string
|
||
n := 0
|
||
for _, w := range words {
|
||
if !strings.ContainsRune(w, 'ё') {
|
||
continue
|
||
}
|
||
eForm := strings.ReplaceAll(w, "ё", "е")
|
||
if eForm == w || c.yoHomograph[eForm] {
|
||
continue // no ё, or a distinct-word homograph (все/всё) — not an inconsistency
|
||
}
|
||
if present[eForm] && !seenPair[w] {
|
||
seenPair[w] = true
|
||
n++
|
||
det = append(det, fmt.Sprintf("inconsistent ё: %q and %q", w, eForm))
|
||
}
|
||
}
|
||
sort.Strings(det)
|
||
return n, det
|
||
}
|
||
|
||
// --- 3. translit-interjection blocklist -----------------------------------------
|
||
|
||
// lintTranslitInterjections counts whole-word (letter-bounded, case-insensitive) occurrences of a
|
||
// blocklisted interjection (c.translitInterjection — TARGET data, was a Go literal) in the output, minus any
|
||
// surface on the per-project Allowlist. Whole-word matching keeps «ара» from firing inside «характер»; the
|
||
// Allowlist exempts legitimate uses. A target that ships no blocklist runs this inert.
|
||
//
|
||
// NOTE (D20.4): only the EXACT hyphenated reduplications in the data fire. Other reduplicated fillers
|
||
// («уху-уху», «ня-ня») are NOT caught — a generic «X-X» rule was rejected as too false-positive-prone
|
||
// (legit «еле-еле», «чуть-чуть»). Extend the DATA per corpus finding, not by heuristic.
|
||
func (c *Checkers) lintTranslitInterjections(out string, Allowlist map[string]bool) (int, []string) {
|
||
if c == nil {
|
||
return 0, nil
|
||
}
|
||
low := []rune(strings.ToLower(out))
|
||
counts := map[string]int{}
|
||
for _, interj := range c.translitInterjection {
|
||
if Allowlist[interj] {
|
||
continue
|
||
}
|
||
f := []rune(interj)
|
||
for i := 0; i+len(f) <= len(low); i++ {
|
||
if !text.RunesEqual(low[i:i+len(f)], f) {
|
||
continue
|
||
}
|
||
if (i == 0 || !isWordRune(low[i-1])) && (i+len(f) == len(low) || !isWordRune(low[i+len(f)])) {
|
||
counts[interj]++
|
||
}
|
||
}
|
||
}
|
||
n := 0
|
||
surfaces := make([]string, 0, len(counts))
|
||
for k, c := range counts {
|
||
n += c
|
||
surfaces = append(surfaces, k)
|
||
}
|
||
sort.Strings(surfaces)
|
||
var det []string
|
||
for _, s := range surfaces {
|
||
det = append(det, fmt.Sprintf("untranslated interjection %q ×%d", s, counts[s]))
|
||
}
|
||
return n, det
|
||
}
|
||
|
||
// isWordRune reports whether r is part of a word for boundary checks (letter or a hyphen inside a
|
||
// compound interjection like «ара-ара»). The hyphen inclusion means «ара-ара» matches as one unit
|
||
// and its inner «ара» does not double-count against a hyphen boundary.
|
||
func isWordRune(r rune) bool {
|
||
return unicode.IsLetter(r) || r == '-'
|
||
}
|
||
|
||
// --- 4. 万/億 magnitude gate -----------------------------------------------------
|
||
|
||
// lintNumberMagnitude flags a source CJK myriad/hundred-million magnitude whose ORDER of magnitude
|
||
// is not represented on the Russian side (04-unhappy §9: 三万 → «три миллиона» is a 100× error). It
|
||
// parses each CJK numeral RUN that carries a big marker (万/萬/億/亿/兆) into an order of magnitude,
|
||
// then checks the output's magnitude coverage (Russian magnitude words expanded to a [base,base+2]
|
||
// range for their possible multiplier, plus Arabic-number orders). It fires only when the source's
|
||
// TOP order is outside every output range — a conservative, multiplier-tolerant signal that leaves
|
||
// 三億→«триста миллионов» (8 within миллион's [6,8]) silent while catching 三万→«три миллиона».
|
||
func (c *Checkers) lintNumberMagnitude(source, final string) (int, []string) {
|
||
if c == nil || len(c.magnitudeStem) == 0 {
|
||
return 0, nil // no target magnitude-word data → the gate can never confirm coverage; stay inert
|
||
}
|
||
// SOURCE-script gate (строка 79, D39.39): the 万/億/兆 markers are Han, so a DECLARED non-dense source
|
||
// cannot carry a magnitude — stay inert rather than firing on a stray Han rune quoted inside e.g. an
|
||
// English source. An UNDECLARED source (nil scripts) keeps the prior content-only behaviour.
|
||
if len(c.sourceScripts) > 0 && !c.sourceHasDenseScript() {
|
||
return 0, nil
|
||
}
|
||
srcOrders := cjkMagnitudeOrders(source)
|
||
if len(srcOrders) == 0 {
|
||
return 0, nil
|
||
}
|
||
maxSrc := 0
|
||
for _, o := range srcOrders {
|
||
if o > maxSrc {
|
||
maxSrc = o
|
||
}
|
||
}
|
||
if maxSrc < 4 { // only gate on 万+ magnitudes (the 万/億 concern)
|
||
return 0, nil
|
||
}
|
||
// A bare output integer (a year, a count, a page number) may CONFIRM the magnitude but must never
|
||
// TRIGGER a flag: an unrelated number of a different order is not evidence the 万/億 was
|
||
// mistranslated (D20.4 FPs: «万 as part of a name + an unrelated number in the output», «a paraphrased
|
||
// magnitude + a year»). So an Arabic figure of the RIGHT order suppresses; a mismatching one is ignored.
|
||
for _, o := range arabicNumberOrders(final) {
|
||
if o == maxSrc {
|
||
return 0, nil // an Arabic figure of the source order confirms coverage (30000 for 三万)
|
||
}
|
||
}
|
||
wordRanges := c.magnitudeWordRanges(final)
|
||
for _, rg := range wordRanges {
|
||
if maxSrc >= rg[0] && maxSrc <= rg[1] {
|
||
return 0, nil // a magnitude WORD covers the source order (三万 → «тридцать тысяч»)
|
||
}
|
||
}
|
||
// Only a mismatching magnitude WORD is evidence of an order-of-magnitude error (三万 → «три миллиона»). Absent
|
||
// any magnitude word, stay silent: the magnitude was rephrased as prose, or the number in the
|
||
// output is unrelated (the two D20.4 FPs above). Accepted recall cost: a dropped-magnitude error
|
||
// rendered as a BARE integer of a smaller order (三万 → «300») is now also silent — indistinguishable
|
||
// offline from an unrelated stray integer without number alignment (Ф2). Precision over recall.
|
||
if len(wordRanges) == 0 {
|
||
return 0, nil
|
||
}
|
||
return 1, []string{fmt.Sprintf("the source magnitude 10^%d is not reflected in the translation's orders of magnitude — a possible magnitude error", maxSrc)}
|
||
}
|
||
|
||
// isCJKNumeralRune reports whether r can form a CJK numeral expression: an ASCII digit or a shared
|
||
// lang.CJKSection numeral (Zero∪Digit∪Unit∪Magnitude, pair-14 data-out — the gate no longer keeps its own
|
||
// rune set). A maximal run of these is one candidate number.
|
||
func isCJKNumeralRune(r rune) bool {
|
||
return (r >= '0' && r <= '9') || lang.DefaultCJKSection().IsNumeralRune(r)
|
||
}
|
||
|
||
// cjkMagnitudeOrders returns the base-10 order of every CJK numeral run in text that contains a big
|
||
// marker (万/億/兆). Runs without a big marker are ignored (the gate is about myriad-scale orders of magnitude).
|
||
func cjkMagnitudeOrders(text string) []int {
|
||
var orders []int
|
||
rs := []rune(text)
|
||
for i := 0; i < len(rs); {
|
||
if !isCJKNumeralRune(rs[i]) {
|
||
i++
|
||
continue
|
||
}
|
||
j := i
|
||
for j < len(rs) && isCJKNumeralRune(rs[j]) {
|
||
j++
|
||
}
|
||
run := string(rs[i:j])
|
||
// A run counts only when it carries a big marker AND has an explicit DIGIT coefficient before
|
||
// it (self-review major): this excludes the common web-novel IDIOMS that are not magnitudes —
|
||
// 万一 (in case), 万分 (extremely), 万物 (all things), 千万 (by all means), 亿万 (myriads) — where 万/億
|
||
// is not preceded by a digit. It also drops bare-unit magnitudes (十万/百万) — an accepted recall
|
||
// trade for not false-flagging the far more frequent idioms.
|
||
if containsBigMarker(run) && hasDigitBeforeBigMarker(run) {
|
||
if v, ok := parseCJKNumber(run); ok && v > 0 {
|
||
orders = append(orders, orderOf(v))
|
||
}
|
||
}
|
||
i = j
|
||
}
|
||
return orders
|
||
}
|
||
|
||
// isBigMarker / containsBigMarker read the shared lang.CJKSection magnitude set (万/萬/億/亿/兆, pair-14
|
||
// data-out) instead of a hard-coded rune string.
|
||
func isBigMarker(r rune) bool { _, ok := lang.DefaultCJKSection().BigUnit(r); return ok }
|
||
func containsBigMarker(run string) bool {
|
||
for _, r := range run {
|
||
if isBigMarker(r) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// hasDigitBeforeBigMarker reports whether a digit (一-九 / 两 / 0-9) appears before the FIRST big
|
||
// marker (万/億/兆) in the run — the signature of a real magnitude expression (三万) vs an idiom (万一).
|
||
func hasDigitBeforeBigMarker(run string) bool {
|
||
sec := lang.DefaultCJKSection()
|
||
for _, r := range run {
|
||
if isBigMarker(r) {
|
||
return false // hit a big marker with no digit before it → idiom / bare unit
|
||
}
|
||
if _, isDigit := sec.Digit[r]; (r >= '0' && r <= '9') || isDigit {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// parseCJKNumber parses a CJK numeral expression (mixed with Arabic digits) into its integer value.
|
||
// Standard section algorithm: small units (十百千) scale the pending coefficient into the current
|
||
// <10^4 section; big units (万億兆) flush the section times the big unit into the total. Returns
|
||
// ok=false on a shape it cannot parse (conservative — an unparseable run does not flag).
|
||
func parseCJKNumber(s string) (int64, bool) {
|
||
sec := lang.DefaultCJKSection()
|
||
var total, section, cur int64
|
||
sawBig := false
|
||
for _, r := range s {
|
||
switch {
|
||
case r >= '0' && r <= '9':
|
||
cur = cur*10 + int64(r-'0')
|
||
case sec.Zero[r]:
|
||
cur = cur * 10
|
||
default:
|
||
if d, ok := sec.Digit[r]; ok {
|
||
cur = cur*10 + int64(d) // positional accumulation (一二→12), matching the Arabic-digit branch (self-review)
|
||
continue
|
||
}
|
||
if u, ok := sec.Unit[r]; ok {
|
||
if cur == 0 {
|
||
cur = 1
|
||
}
|
||
section += cur * int64(u)
|
||
cur = 0
|
||
continue
|
||
}
|
||
if u, ok := sec.BigUnit(r); ok {
|
||
sawBig = true
|
||
section += cur
|
||
if section == 0 {
|
||
section = 1
|
||
}
|
||
total += section * u
|
||
section = 0
|
||
cur = 0
|
||
continue
|
||
}
|
||
return 0, false // an unexpected rune
|
||
}
|
||
}
|
||
if !sawBig {
|
||
return 0, false // no 万/億/兆 → not a magnitude expression this gate cares about
|
||
}
|
||
return total + section + cur, true
|
||
}
|
||
|
||
func orderOf(v int64) int {
|
||
o := 0
|
||
for v >= 10 {
|
||
v /= 10
|
||
o++
|
||
}
|
||
return o
|
||
}
|
||
|
||
// magnitudeWordRanges returns the covered [minOrder,maxOrder] ranges implied by Russian magnitude
|
||
// WORDS in the output. A word covers [base, base+2] because an unseen multiplier can lift it up to
|
||
// two orders (триста миллионов = 3·10^8, base 6 → order 8). Arabic integers are handled separately
|
||
// by the caller (they may only CONFIRM coverage, never trigger a mismatch — D20.4), so they are NOT
|
||
// folded in here.
|
||
func (c *Checkers) magnitudeWordRanges(text string) [][2]int {
|
||
low := strings.ToLower(text)
|
||
var ranges [][2]int
|
||
for stem, base := range c.magnitudeStem { // target-general data (pair-14 data-out)
|
||
if strings.Contains(low, stem) {
|
||
ranges = append(ranges, [2]int{base, base + 2})
|
||
}
|
||
}
|
||
return ranges
|
||
}
|
||
|
||
// arabicNumberOrders returns the order of each Arabic integer in text, stitching grouping
|
||
// separators (space / NBSP / comma) between runs of exactly three digits so "30 000" reads as one
|
||
// 5-digit number, not "30" and "000".
|
||
func arabicNumberOrders(text string) []int {
|
||
rs := []rune(text)
|
||
var orders []int
|
||
for i := 0; i < len(rs); {
|
||
if !isASCIIDigit(rs[i]) {
|
||
i++
|
||
continue
|
||
}
|
||
// Leading group.
|
||
j := i
|
||
for j < len(rs) && isASCIIDigit(rs[j]) {
|
||
j++
|
||
}
|
||
digits := j - i
|
||
// Stitch " ddd" / ",ddd" groups.
|
||
for j < len(rs) {
|
||
if (rs[j] == ' ' || rs[j] == ' ' || rs[j] == ',') && j+3 < len(rs)+1 {
|
||
k := j + 1
|
||
g := 0
|
||
for k < len(rs) && isASCIIDigit(rs[k]) {
|
||
k++
|
||
g++
|
||
}
|
||
if g == 3 {
|
||
digits += 3
|
||
j = k
|
||
continue
|
||
}
|
||
}
|
||
break
|
||
}
|
||
orders = append(orders, digits-1)
|
||
i = j
|
||
}
|
||
return orders
|
||
}
|
||
|
||
func isASCIIDigit(r rune) bool { return r >= '0' && r <= '9' }
|
||
|
||
// preview trims a long line for the human detail (deterministic).
|
||
func preview(s string) string {
|
||
rs := []rune(s)
|
||
if len(rs) > 48 {
|
||
return string(rs[:48]) + "…"
|
||
}
|
||
return s
|
||
}
|