textmachine/backend/internal/checks/checkers.go

package checks

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"unicode"

	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/text"
)

// checkers.go: the WS5 defect-class checkers (DC1 时辰 double-hour units, DC2 千万/数十万 magnitude scale, DC6
// register negative-list) + the pack-13 general checkers (percent scale, Latin residue, broken word) —
// deterministic, $0 OBSERVABILITY flaggers on the source↔FINAL text, ported from ws5_checkers_verify.py.
// Like the four cheap style gates they are NEVER a disposition (a hit is recorded, never drops a chunk),
// tuned PRECISION over recall.
//
// PAIR-AGNOSTIC (pair-14 data-out): this file no longer holds any language literal. Every DETECTION pattern,
// lookup table and wordlist is DATA — the SOURCE-gated pair checkers (DC1/DC2/percent: they need a zh source
// token and compare src↔tgt) read the pair pack configs/langpacks/<pair>/dc-checkers.txt (lang.DCCheckerData);
// the TARGET-general ones (register, broken word: they run on ANY →target output) read the embedded target
// data (lang.TargetChecks). The ALGORITHM (compare counts, ×2 hours, suppress-if-ok, whole-word match) stays
// here. A pair/target that ships no data runs the relevant sub-checker inert (empty → 0, the no-pack golden
// path). Version rides the langpack Version() (data) + CheapGateVersion (algorithm) — a data or rule edit is
// a loud --resnapshot.

// Checkers is the compiled, per-run checker spec: the pair's DETECTION patterns compiled ONCE + its lookup
// tables + the target-general lists, resolved from the langpack. A nil receiver, a nil pattern or an empty
// table leaves that sub-checker inert. Built once per run (CompileCheckers), carried in CheapGateConfig.
type Checkers struct {
	numeral      map[rune]int
	ruHours      map[string]int
	registerNeg  []string
	brokenSuffix []string // target-general (any →ru output)
	// speechVerb is the target's SPOKEN-attribution vocabulary («сказал», «ответил», «крикнул»): the
	// verbs that make a chevron-quoted line real dialogue rather than a thought. It is target data
	// because the convention it encodes — thought in «…», speech aloud with «—» — is a fact about the
	// target's typography, not about the source pair or the algorithm.
	speechVerb []string
	// innerMarker vetoes speechVerb: a qualifier («про себя», «мысленно») that makes the line inner
	// speech even under a speech verb — «пробормотал про себя» is a thought said with a speaking verb.
	innerMarker   []string
	yoHomograph   map[string]bool // target-general: ё↔е homograph whitelist (cheapgates yofikator)
	magnitudeStem map[string]int  // target-general: ru magnitude word stem → base-10 exponent (cheapgates)

	shichenRE, ruHoursRE, chengRE, decimalFractionRE *regexp.Regexp
	qianwanOKRE, shushiwanOKRE, shushiwanFireRE      *regexp.Regexp
	qianwanSrc, qianwanFireWord, qianwanVetoWord     string
	shushiwanSrc, percentWord                        string

	// halfShichenRE / halfShichenFireWord are the FRACTIONAL-unit probe of the DC1 family: the count group of
	// shichenRE accepts a numeral only, so a fractional unit word (半 «half») never reaches the count parser and
	// the most frequent form of the unit is invisible to the whole class. The probe is a plain src-pattern +
	// target fire-word pair (the DC2 idiom), so the pair supplies BOTH or the sub-check stays inert — a pack
	// without these keys behaves exactly as before this field existed.
	halfShichenRE       *regexp.Regexp
	halfShichenFireWord string
	// hourWordRE is the target's BARE hour word (both boundaries explicit). It is not a detection pattern:
	// it is the positive post-condition a fractional-unit REPAIR must satisfy, and it lives with its
	// siblings because "how this target renders the double-hour" is one fact with one home.
	hourWordRE *regexp.Regexp

	// dc1UnitHours is how many target hours ONE source time-unit is worth (zh 时辰 = 2). It used to be a
	// literal `n * 2` in the DC1 algorithm — a pair fact hiding in generic Go, which would silently lie to a
	// pair whose unit is not a double-hour. Data now (nit of generality, pre-run hygiene 25.07).
	dc1UnitHours int
	// msg holds the pair's DETAIL templates ({name} placeholders). Every pair-gated checker renders its
	// detail line from here instead of a Go format string, so the engine file carries no pair literal.
	msg map[string]string
}

// CompileCheckers resolves the checker spec from the pair pack (dc) and the target data (tc). A malformed
// regex in the pack is a corrupt pack → panic (deterministic, caught by the checker/golden tests). dc==nil
// (a no-pack book) → the pair sub-checkers are inert; tc still supplies the target-general lists.
func CompileCheckers(dc *lang.DCCheckerData, tc lang.TargetChecks) *Checkers {
	c := &Checkers{
		brokenSuffix:  tc.List("broken_suffix"),
		speechVerb:    tc.List("speech_verb"),
		innerMarker:   tc.List("inner_marker"),
		yoHomograph:   listToSet(tc.List("yo_homograph")),
		magnitudeStem: listToStemExp(tc.List("magnitude_stem")),
	}
	if dc != nil {
		c.numeral, c.ruHours, c.registerNeg = dc.Numeral, dc.RuHours, dc.RegisterNeg
		p := dc.Patterns
		c.shichenRE = mustPairRE(p, "shichen_re")
		c.ruHoursRE = mustPairRE(p, "ru_hours_re")
		c.chengRE = mustPairRE(p, "cheng_re")
		c.decimalFractionRE = mustPairRE(p, "decimal_fraction_re")
		c.qianwanOKRE = mustPairRE(p, "qianwan_ok_re")
		c.shushiwanOKRE = mustPairRE(p, "shushiwan_ok_re")
		c.shushiwanFireRE = mustPairRE(p, "shushiwan_fire_re")
		c.qianwanSrc, c.qianwanFireWord, c.qianwanVetoWord = p["qianwan_src"], p["qianwan_fire_word"], p["qianwan_veto_word"]
		c.shushiwanSrc, c.percentWord = p["shushiwan_src"], p["percent_word"]
		c.halfShichenRE = mustPairRE(p, "halfshichen_re")
		c.halfShichenFireWord = p["halfshichen_fire_word"]
		c.hourWordRE = mustPairRE(p, "hour_word_re")
		c.dc1UnitHours = mustPairRatio(dc.Ratios, "dc1_unit_in_hours")
		c.msg = mustPairMessages(dc.Messages,
			"dc1_fractional", "dc1_counted", "dc2_qianwan", "dc2_shushiwan", "dc6_register", "percent_scale")
	}
	return c
}

// mustPairRatio reads a required numeric pair relation. A missing/zero ratio is a corrupt pack, not a
// tolerable default: the DC1 comparison is MEANINGLESS without it (a silent 0 would make every rendering
// look wrong), so it fails the same way a malformed regex does — loudly, at compile, named.
func mustPairRatio(ratios map[string]int, key string) int {
	v, ok := ratios[key]
	if !ok || v <= 0 {
		panic(fmt.Sprintf("checks: pair pack dc-checkers is missing the positive ratio %q (a detector without its unit relation cannot judge anything)", key))
	}
	return v
}

// mustPairMessages reads the required DETAIL templates. A pair that ships detection data but no message
// for it is a corrupt pack by the same argument: a checker that fires and cannot say WHAT it found produces
// an empty observability line, which reads downstream as "nothing was wrong".
func mustPairMessages(msgs map[string]string, keys ...string) map[string]string {
	out := make(map[string]string, len(keys))
	for _, k := range keys {
		v := msgs[k]
		if strings.TrimSpace(v) == "" {
			panic(fmt.Sprintf("checks: pair pack dc-checkers is missing the detail template %q (a detector with no message is a silent detector)", k))
		}
		out[k] = v
	}
	return out
}

// renderMsg substitutes `{name}` placeholders in a pair DETAIL template. kv is a flat name,value sequence
// (an odd tail is a programming error and panics — the templates and their call sites are compiled
// together). Pure and deterministic: one pass, no map iteration.
func renderMsg(tpl string, kv ...string) string {
	if len(kv)%2 != 0 {
		panic("checks: renderMsg wants name,value pairs")
	}
	if len(kv) == 0 {
		return tpl
	}
	rep := make([]string, 0, len(kv))
	for i := 0; i < len(kv); i += 2 {
		rep = append(rep, "{"+kv[i]+"}", kv[i+1])
	}
	return strings.NewReplacer(rep...).Replace(tpl)
}

// listToSet turns an ordered value list into a membership set (target wordlists).
func listToSet(xs []string) map[string]bool {
	m := make(map[string]bool, len(xs))
	for _, x := range xs {
		m[x] = true
	}
	return m
}

// listToStemExp parses `stem<TAB>exp` values (the magnitude_stem list carries a second tab-field) into a
// stem→exponent map. A malformed value is a corrupt embed → panic (deterministic, caught by tests).
func listToStemExp(xs []string) map[string]int {
	m := make(map[string]int, len(xs))
	for _, x := range xs {
		f := strings.SplitN(x, "\t", 2)
		if len(f) != 2 {
			panic(fmt.Sprintf("checks: magnitude_stem wants `stem<TAB>exp`, got %q", x))
		}
		v, err := strconv.Atoi(strings.TrimSpace(f[1]))
		if err != nil {
			panic(fmt.Sprintf("checks: magnitude_stem exponent %q: %v", f[1], err))
		}
		m[f[0]] = v
	}
	return m
}

// DCCheckerData returns the pack's checker data, or nil when the book has no pack (nil-safe helper for the
// compile step, which runs even for a no-langpack book).
func DCCheckerData(p *lang.Pack) *lang.DCCheckerData {
	if p != nil {
		return p.DCCheckers
	}
	return nil
}

// mustPairRE compiles a pair detection pattern by key; a missing key → nil (inert sub-checker), a malformed
// regex → panic (a corrupt pack, not a silent no-op — the same fail-loud discipline the loader keeps).
func mustPairRE(p map[string]string, key string) *regexp.Regexp {
	s := p[key]
	if s == "" {
		return nil
	}
	re, err := regexp.Compile(s)
	if err != nil {
		panic(fmt.Sprintf("checks: langpack checker pattern %q is not a valid regex: %v", key, err))
	}
	return re
}

// --- DC-1: 时辰 (double-hour) unit checker (ws5.shichen_checker) -----------------------------------

// lintTimeUnits flags a 时辰 (=2h) unit error: N个时辰 rendered as N часов (the count copied as hours)
// instead of ~2N hours (三个时辰 → «три часа» should be ~6h). It fires ONLY on an explicit mismatch — a
// paraphrase with no hours count is a valid rendering, not a defect. Pure and deterministic. Inert when the
// pair ships no DC1 pattern (the shichen_re / ru_hours_re detection patterns are pair langpack DATA).
func (c *Checkers) lintTimeUnits(source, final string) (int, []string) {
	if c == nil {
		return 0, nil
	}
	// Fractional-unit probe FIRST: 半个时辰 carries no numeral, so the counted branch below can never see it
	// (its count group matches a numeral only) — yet 半 is the most frequent 时辰 form in a real corpus.
	// Rendering it as «полчаса» halves the duration. Data-gated on BOTH keys (empty-probe guard, pack-15):
	// a pair that ships neither key runs this inert, exactly as before the probe existed.
	if c.halfShichenRE != nil && c.halfShichenFireWord != "" &&
		c.halfShichenRE.MatchString(source) && strings.Contains(final, c.halfShichenFireWord) {
		return 1, []string{renderMsg(c.msg["dc1_fractional"], "fire_word", c.halfShichenFireWord)}
	}
	if c.shichenRE == nil || c.ruHoursRE == nil {
		return 0, nil
	}
	m := c.shichenRE.FindStringSubmatch(source)
	if m == nil {
		return 0, nil
	}
	n, ok := dcParseCount(m[1], c.numeral)
	if !ok {
		return 0, nil
	}
	hm := c.ruHoursRE.FindStringSubmatch(final)
	if hm == nil {
		return 0, nil // no explicit hours rendering → a valid paraphrase, not a defect
	}
	ruNum, ok := dcParseRuHours(hm[1], c.ruHours)
	if !ok {
		return 0, nil
	}
	expectedHours := n * c.dc1UnitHours
	if ruNum == n && ruNum != expectedHours {
		return 1, []string{renderMsg(c.msg["dc1_counted"],
			"n", strconv.Itoa(n), "rendered", strconv.Itoa(ruNum),
			"expected", strconv.Itoa(expectedHours), "ratio", strconv.Itoa(c.dc1UnitHours))}
	}
	return 0, nil
}

// dcParseCount parses the DC1 count group: an Arabic digit string or a single small CJK numeral (looked up
// in the pair's DCCheckerData.Numeral, empty for a no-pack book → CJK counts don't resolve, the check is inert).
func dcParseCount(s string, dcNum map[rune]int) (int, bool) {
	if v, err := strconv.Atoi(s); err == nil {
		return v, true
	}
	r := []rune(s)
	if len(r) == 1 {
		if v, ok := dcNum[r[0]]; ok {
			return v, true
		}
	}
	return 0, false
}

// dcParseRuHours parses the DC1 hours group: an Arabic digit string or a Russian count word (pair data).
func dcParseRuHours(s string, dcRu map[string]int) (int, bool) {
	if v, err := strconv.Atoi(s); err == nil {
		return v, true
	}
	if v, ok := dcRu[s]; ok {
		return v, true
	}
	return 0, false
}

// --- DC-2: number-scale magnitude checker (ws5.magnitude_checker, 千万 / 数十万) --------------------

// lintMagnitudeScale flags a 千万 (10^7) / 数十万 (~several×10^5) magnitude rendered at a WRONG smaller
// scale. A CORRECT rendering anywhere in the chunk (the ok-suppressor) suppresses the flag (ws5 reference
// parity). The ok-suppressors are case-INsensitive (their data carries the (?i)); the FIRE predicates are
// case-SENSITIVE literal Contains (the reference does not pass re.I to the inner searches). ⚠ 千万 is also
// stock HYPERBOLE whose «тысячи» rendering is in-register (§5 A4). Observability only. All probes/patterns
// are pair langpack DATA — inert when the pair ships no DC2.
func (c *Checkers) lintMagnitudeScale(source, final string) (int, []string) {
	if c == nil {
		return 0, nil
	}
	var flags []string
	// EMPTY-PROBE GUARD (pack-15): a probe word missing from the pair data must leave the sub-check
	// INERT, never spurious. strings.Contains(x, "") is TRUE for every x, so an empty fire word would fire
	// this flag on every chunk whose source carries 千万, and an empty veto word would (silently) disable it
	// — a data slip that reads as a checker bug. Both words are required for the probe to run at all.
	if c.qianwanOKRE != nil && c.qianwanSrc != "" && c.qianwanFireWord != "" && c.qianwanVetoWord != "" &&
		strings.Contains(source, c.qianwanSrc) && !c.qianwanOKRE.MatchString(final) {
		if strings.Contains(final, c.qianwanFireWord) && !strings.Contains(final, c.qianwanVetoWord) { // case-sensitive, per reference
			flags = append(flags, c.msg["dc2_qianwan"])
		}
	}
	if c.shushiwanOKRE != nil && c.shushiwanFireRE != nil && c.shushiwanSrc != "" && strings.Contains(source, c.shushiwanSrc) && !c.shushiwanOKRE.MatchString(final) {
		if c.shushiwanFireRE.MatchString(final) {
			flags = append(flags, c.msg["dc2_shushiwan"])
		}
	}
	return len(flags), flags
}

// --- DC-6: register-lexicon negative-list (ws5.register_checker) ----------------------------------

// lintRegisterLexicon flags whole-word occurrences of a register negative-list lexeme in the FINAL text.
// The negative-list is pair langpack DATA (DCCheckerData.RegisterNeg): fairy-tale-Russian / chancery lexemes
// that break the xianxia register. Lower-cased; whole-word matched. Empty (a no-pack book) → nothing to flag.
func (c *Checkers) lintRegisterLexicon(final string) (int, []string) {
	if c == nil || len(c.registerNeg) == 0 {
		return 0, nil
	}
	low := []rune(strings.ToLower(final))
	hitSet := map[string]bool{}
	for _, w := range c.registerNeg {
		wr := []rune(w)
		for i := 0; i+len(wr) <= len(low); i++ {
			if !text.RunesEqual(low[i:i+len(wr)], wr) {
				continue
			}
			if (i == 0 || !isCyrLetter(low[i-1])) && (i+len(wr) == len(low) || !isCyrLetter(low[i+len(wr)])) {
				hitSet[w] = true
			}
		}
	}
	if len(hitSet) == 0 {
		return 0, nil
	}
	hits := make([]string, 0, len(hitSet))
	for w := range hitSet {
		hits = append(hits, w)
	}
	sort.Strings(hits)
	return len(hits), []string{renderMsg(c.msg["dc6_register"], "hits", strings.Join(hits, ", "))}
}

// isCyrLetter reports whether r is a Cyrillic letter (the word boundary for the register match).
func isCyrLetter(r rune) bool { return unicode.IsLetter(r) && unicode.Is(unicode.Cyrillic, r) }

// --- percent-scale checker (成 = tenths) -----------------------------------------------------------
//
// In Chinese, 成 is one tenth: 六成 = 60%, 六成六 = 66%. A common error renders this as a decimal FRACTION
// instead of a percentage (a ~100× scale error). Precision over recall: it fires only when the source has a
// «<count>成[<count>]» (cheng_re, pair data), the output has NO percent form (percent_word / «%»), AND the
// output carries a decimal-fraction cue (decimal_fraction_re). Inert when the pair ships no percent patterns.
func (c *Checkers) lintPercentScale(source, final string) (int, []string) {
	if c == nil || c.chengRE == nil || c.decimalFractionRE == nil {
		return 0, nil
	}
	m := c.chengRE.FindStringSubmatch(source)
	if m == nil {
		return 0, nil
	}
	low := strings.ToLower(final)
	if (c.percentWord != "" && strings.Contains(low, c.percentWord)) || strings.Contains(final, "%") {
		return 0, nil // the output uses a percent form — the scale is handled correctly
	}
	if !c.decimalFractionRE.MatchString(low) {
		return 0, nil // no fraction cue — the magnitude was paraphrased, not mis-scaled
	}
	tens, _ := dcParseCount(m[1], c.numeral)
	pct := tens * 10
	if m[2] != "" {
		if ones, ok := dcParseCount(m[2], c.numeral); ok {
			pct += ones
		}
	}
	return 1, []string{renderMsg(c.msg["percent_scale"],
		"tens", m[1], "ones", m[2], "pct", strconv.Itoa(pct))}
}

// --- Latin residue in the Russian output -----------------------------------------------------------
//
// A whole Latin WORD left untranslated in the output (e.g. «открыл их again, …»). Language-general (Latin is
// not pair data): it splits the output into maximal alphanumeric tokens and flags an all-lowercase Latin
// token (leaked prose is lowercase; a capital signals a proper noun/brand) with no digit, ≥ minLatinResidueLen
// letters, not a Roman numeral, not on the per-project allowlist. Precision over recall.
const minLatinResidueLen = 3

func lintLatinResidue(final string, allow map[string]bool) (int, []string) {
	hits := map[string]bool{}
	rs := []rune(final)
	for i := 0; i < len(rs); {
		if !isLatinLetterOrDigit(rs[i]) {
			i++
			continue
		}
		j := i
		reject := false // set on any digit or uppercase letter — not a lowercase leaked word
		for j < len(rs) && isLatinLetterOrDigit(rs[j]) {
			if (rs[j] >= '0' && rs[j] <= '9') || (rs[j] >= 'A' && rs[j] <= 'Z') {
				reject = true
			}
			j++
		}
		tok := string(rs[i:j])
		i = j
		if !reject && len([]rune(tok)) >= minLatinResidueLen && !isRomanNumeral(tok) && !allow[tok] {
			hits[tok] = true
		}
	}
	if len(hits) == 0 {
		return 0, nil
	}
	surfaces := make([]string, 0, len(hits))
	for s := range hits {
		surfaces = append(surfaces, s)
	}
	sort.Strings(surfaces)
	return len(surfaces), []string{"Latin word left untranslated in the target output: " + strings.Join(surfaces, ", ")}
}

// isLatinLetterOrDigit reports whether r is an ASCII Latin letter or digit (the alphanumeric-token alphabet).
func isLatinLetterOrDigit(r rune) bool {
	return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
}

// isRomanNumeral reports whether a lowercase token is a Roman numeral (all chars in ivxlcdm) — «iii» reads
// as a numeral, not a leaked word; excluded to hold precision. (Uppercase «II» is already skipped as a cap.)
func isRomanNumeral(tok string) bool {
	for _, r := range tok {
		switch r {
		case 'i', 'v', 'x', 'l', 'c', 'd', 'm':
		default:
			return false
		}
	}
	return true
}

// --- broken word forms (target-general) ------------------------------------------------------------
//
// Flags a target word ending in a structurally-impossible suffix — for ru, «-йть», which no well-formed
// Russian word does (the shape of a mangled infinitive, «войть» for «войти»). The suffix set is TARGET data
// (lang.TargetChecks "broken_suffix"), so the ALGORITHM is language-general; a target with no suffix data
// flags nothing. Zero false-positive by construction (only a structural signature, no dictionary).
func (c *Checkers) lintBrokenWord(final string) (int, []string) {
	if c == nil || len(c.brokenSuffix) == 0 {
		return 0, nil
	}
	seen := map[string]bool{}
	var det []string
	for _, w := range text.TokenizeCyrillic(final) {
		wl := len([]rune(w))
		for _, suf := range c.brokenSuffix {
			if wl >= 4 && strings.HasSuffix(w, suf) && !seen[w] {
				seen[w] = true
				det = append(det, "malformed word: the target has no well-formed word ending in «-"+suf+"»: "+w)
			}
		}
	}
	sort.Strings(det)
	return len(det), det
}

// isSpokenChevronLine reports whether a chevron-led line is SPOKEN dialogue, by looking for one of the
// target's speech-attribution verbs in it.
//
// The polarity is the load-bearing choice. Asking "is this a THOUGHT?" needs an open-ended list — a live
// 2-chapter run produced «понимал» and «размышлял» within minutes, and thought verbs have no natural
// boundary — and every miss becomes a false flag on correct typography. Asking "is this SPOKEN?" needs a
// small closed list, and every miss becomes SILENCE on a real style clash. Precision over recall is the
// gate's stated bias, so the second failure mode is the right one to have.
//
// A nil receiver / a target with no `speech_verb` data returns false: the mixing rule then never fires,
// which is the same "inert without data" contract the pair checkers follow.
func (c *Checkers) isSpokenChevronLine(line string) bool {
	if c == nil || len(c.speechVerb) == 0 {
		return false
	}
	low := strings.ToLower(line)
	for _, marker := range c.innerMarker {
		if strings.Contains(low, marker) {
			return false // «пробормотал ПРО СЕБЯ» — a speaking verb qualified into a thought
		}
	}
	for _, verb := range c.speechVerb {
		if strings.Contains(low, verb) {
			return true
		}
	}
	return false
}