textmachine/backend/internal/checks/voice.go

package checks

import (
	"fmt"
	"sort"
	"strings"

	"textmachine/backend/internal/text"
)

// voice.go: the deterministic $0 VOICE flagger (pack-19, ratified D39.55) — the layer-1 half of D21's
// voice detector. Like every checker here it is PURE observability, never a disposition, and tuned
// PRECISION over recall: a miss is silence on a real defect, a false positive is noise on good prose.
//
// WHAT IT CAN SEE, AND WHY THAT IS THE SHAPE IT IS. The check D21 describes — "a ты/вы flip in an
// attributed reply against the registry of PAIRS" — needs BOTH ends of a pair, and only one end is
// deterministically reachable without the Annotator this pack does not build (a speaker cue names the
// speaker; the addressee is named by almost nothing). So the flagger stands on three axes that need no
// addressee at all, and reports the fourth separately:
//
//	A. tv_contradiction — one reply carries both a T and a V surface. Needs neither end.
//	B. self_ref_flat    — an attributed speaker with a declared self-designation refers to herself with
//	                      a plain pronoun instead (the measured flattening class, research/15 §Пробы).
//	C. ng_lexicon       — an attributed speaker says a word her profile forbids. Needs the speaker only.
//	D. pair_register    — the registry check. Reported as an INDICATOR, excluded from Total(), because
//	                      its addressee comes from a heuristic (exactly two characters on stage), not
//	                      from attribution.
//
// ATTRIBUTION IS TARGET-SIDE, and that is a refinement of the phase-1 design (which sketched a
// source-side 说/道 pre-gate): a source cue names a speaker in the SOURCE, while the reply to check is in
// the TARGET, and this engine has no source↔target alignment to carry one to the other. The target side
// needs none — the attribution tail of a Russian dash line already names the speaker in words the bank
// can match. The source cue data is still read, for the honest denominator only (SourceReplies).
//
// EVERYTHING language-specific is DATA: T/V surfaces, the plural veto, the generic self-reference
// pronouns and the reply markers are TARGET data (lang.TargetChecks); the speech cues and quote marks
// are SOURCE data (lang.SpeechCue). A target that ships no T/V rows runs the whole flagger inert.

// VoiceCheckVersion versions the rules in this file. It is deliberately NOT snapshot-folded (see
// config.VoiceGate) — it is logged with the run instead, exactly as terminologyVersion is.
const VoiceCheckVersion = "voice-v1-tv+selfref+ng+pair-indicator"

// VoiceCharacter is one character as the flagger sees it: an opaque identity, the target surfaces that
// name it in the output, and the two profile fields the checks read. Built by the driver from the bank
// (surfaces) and the voice profiles (self-ref / ng); this package never touches the store.
type VoiceCharacter struct {
	Key     string   // stable identity (src\x1fsense) — compared, never rendered into a message
	Name    string   // human-readable label for the detail line
	Forms   []string // NORMALIZED target surfaces (base dst + decl forms)
	SelfRef string   // NORMALIZED declared self-designation; "" = the profile states none
	NG      []string // NORMALIZED forbidden lexemes
}

// VoiceAddressPair is one ordered register record projected onto the current chapter.
type VoiceAddressPair struct {
	Speaker, Addressee string // VoiceCharacter.Key
	Register           string // "informal" | "formal" — abstract; the surfaces are target data
}

// VoiceRegistry is the per-unit projection the flagger runs against.
type VoiceRegistry struct {
	Chars []VoiceCharacter
	Pairs []VoiceAddressPair
}

// HasData reports whether there is anything to check against.
func (r VoiceRegistry) HasData() bool { return len(r.Chars) > 0 }

// VoiceResult is one unit's flagger outcome. Every field is omitempty, so a clean unit (and any unit of
// a book that does not enable the gate) serialises to nothing.
type VoiceResult struct {
	TVContradiction int `json:"tv_contradiction,omitempty"`
	SelfRefFlat     int `json:"self_ref_flat,omitempty"`
	NGLexicon       int `json:"ng_lexicon,omitempty"`
	// PairRegister is the axis-D INDICATOR: excluded from Total() on purpose. Its addressee is inferred
	// from "exactly two characters are on stage", which is a heuristic — mixing it into the headline
	// count would let a structurally weaker signal inflate a number the operator reads as defects.
	PairRegister int `json:"pair_register,omitempty"`
	// Replies / Attributed are the denominators. Without them a count is unreadable: 3 flags out of 4
	// attributed replies is a broken voice, 3 out of 300 is noise.
	Replies    int `json:"replies,omitempty"`
	Attributed int `json:"attributed,omitempty"`
	// SourceReplies is the source-side quoted-turn count (pair data). 0 when the source ships no
	// speech-cue file — which is "not measured", not "none found".
	SourceReplies int      `json:"source_replies,omitempty"`
	Detail        []string `json:"detail,omitempty"`
}

// Total is the flagged count that reaches n_voice_flags: axes A–C only (see PairRegister).
func (v VoiceResult) Total() int { return v.TVContradiction + v.SelfRefFlat + v.NGLexicon }

// voiceReply is one segmented turn: what was said, and the attribution around it.
type voiceReply struct {
	speech string
	attr   string
}

// RunVoiceChecks runs the flagger over one unit's SOURCE and FINAL text against the projected registry.
// Pure and deterministic (sorted detail, no map iteration in output). A nil spec, an empty registry or a
// target with no T/V data yields a zero result — the inert path a pair without the distinction takes.
func RunVoiceChecks(source, final string, reg VoiceRegistry, c *Checkers) VoiceResult {
	var res VoiceResult
	if c == nil {
		return res
	}
	res.SourceReplies = c.countSourceReplies(source)
	if !reg.HasData() || (len(c.tvInformal) == 0 && len(c.tvFormal) == 0) {
		return res
	}
	replies := c.splitReplies(final)
	res.Replies = len(replies)
	if len(replies) == 0 {
		return res
	}
	// Who is on stage at all — the basis of the axis-D addressee heuristic. Computed once over the whole
	// unit, because a speaker named in one reply's attribution is still present for the next one.
	onStage := reg.present(final)

	detail := map[string]bool{}
	for _, rp := range replies {
		speech := []rune(text.NormalizeTargetForm(rp.speech))
		if len(speech) == 0 {
			continue
		}
		hasT := anyWholeWord(speech, c.tvInformal)
		hasV := anyWholeWord(speech, c.tvFormal)
		// Axis A. «Ты иди, а вы оба ждите» is correct Russian addressing two parties, so a plural marker
		// vetoes the contradiction — the same shape inner_marker has over speech_verb.
		if hasT && hasV && !anyWholeWord(speech, c.tvPluralVeto) {
			res.TVContradiction++
			detail[fmt.Sprintf("voice: one reply mixes an informal and a formal address: %s", previewRunes(speech))] = true
		}
		sp, ok := reg.speakerOf(rp.attr, c)
		if !ok {
			continue
		}
		res.Attributed++
		// Axis B. The evidence is that she referred to herself AT ALL (a generic pronoun) while her
		// declared designation is absent — not that she used a pronoun, which everyone does.
		if sp.SelfRef != "" && anyWholeWord(speech, c.selfRefGeneric) && !wholeWord(speech, sp.SelfRef) {
			res.SelfRefFlat++
			detail[fmt.Sprintf("voice: %s refers to herself plainly; the profile declares %q", sp.Name, sp.SelfRef)] = true
		}
		// Axis C.
		for _, ng := range sp.NG {
			if wholeWord(speech, ng) {
				res.NGLexicon++
				detail[fmt.Sprintf("voice: %s says %q, which the profile forbids", sp.Name, ng)] = true
			}
		}
		// Axis D (indicator). Only when the addressee is unambiguous by the two-on-stage heuristic.
		if !hasT && !hasV {
			continue
		}
		addressee, ok := onStage.other(sp.Key)
		if !ok {
			continue
		}
		want, ok := reg.registerFor(sp.Key, addressee)
		if !ok {
			continue
		}
		got := registerFormal
		if hasT {
			got = registerInformal
		}
		if got != want {
			res.PairRegister++
			detail[fmt.Sprintf("voice(indicator): %s addresses the other character as %q, the registry says %q", sp.Name, got, want)] = true
		}
	}
	res.Detail = sortedKeys(detail)
	return res
}

// registerInformal / registerFormal mirror the abstract vocabulary membank validates the seed against.
// They are category names, not target words.
const (
	registerInformal = "informal"
	registerFormal   = "formal"
)

// stageSet is the set of characters whose surfaces appear in a unit, in registry order.
type stageSet []string

// other returns the single OTHER character on stage, and whether the "exactly two, one of them the
// speaker" precondition of the axis-D heuristic holds. Any other cast size gives no addressee.
func (s stageSet) other(speaker string) (string, bool) {
	if len(s) != 2 {
		return "", false
	}
	switch {
	case s[0] == speaker:
		return s[1], true
	case s[1] == speaker:
		return s[0], true
	}
	return "", false
}

// present returns the characters whose surfaces occur in the text, in registry order (deterministic).
func (r VoiceRegistry) present(final string) stageSet {
	norm := []rune(text.NormalizeTargetForm(final))
	var out stageSet
	for _, ch := range r.Chars {
		if anyWholeWord(norm, ch.Forms) {
			out = append(out, ch.Key)
		}
	}
	return out
}

// speakerOf resolves a reply's attribution to a character. It requires BOTH a target speech verb (so a
// narrative sentence that merely names somebody is not an attribution) and EXACTLY ONE character
// surface: «— …, — сказал Фан Юань Бай Нинбин» names two, and guessing which one speaks is precisely the
// coreference this pack does not have. Ambiguity yields no speaker, which is the silent, precise answer.
func (r VoiceRegistry) speakerOf(attr string, c *Checkers) (VoiceCharacter, bool) {
	if strings.TrimSpace(attr) == "" {
		return VoiceCharacter{}, false
	}
	low := []rune(text.NormalizeTargetForm(attr))
	if !anySubstring(low, c.speechVerb) || anySubstring(low, c.innerMarker) {
		return VoiceCharacter{}, false
	}
	var found VoiceCharacter
	n := 0
	for _, ch := range r.Chars {
		if anyWholeWord(low, ch.Forms) {
			found, n = ch, n+1
		}
	}
	if n != 1 {
		return VoiceCharacter{}, false
	}
	return found, true
}

// registerFor returns the projected register for an ordered pair.
func (r VoiceRegistry) registerFor(speaker, addressee string) (string, bool) {
	for _, p := range r.Pairs {
		if p.Speaker == speaker && p.Addressee == addressee {
			return p.Register, true
		}
	}
	return "", false
}

// splitReplies segments the target text into turns using TARGET data alone.
//
// A dash line is one speaker's whole turn: splitting «— A, — сказал X. — B» on the marker alternates
// speech (odd segments) and attribution (even ones), which is the Russian convention as a rule rather
// than as a Go literal. A quoted span is a turn only when its tail carries a speech verb — a bare
// «…» is as likely a citation or a thought, and the inner-speech veto removes the rest.
func (c *Checkers) splitReplies(final string) []voiceReply {
	if c == nil || (c.replyDash == "" && (c.replyOpen == "" || c.replyClose == "")) {
		return nil
	}
	var out []voiceReply
	for _, raw := range strings.Split(final, "\n") {
		line := strings.TrimSpace(raw)
		if line == "" {
			continue
		}
		if c.replyDash != "" && strings.HasPrefix(line, c.replyDash) {
			segs := strings.Split(line, c.replyDash)
			var speech, attr []string
			for i, s := range segs {
				switch {
				case i == 0: // text before the opening marker (empty by the HasPrefix above)
				case i%2 == 1:
					speech = append(speech, s)
				default:
					attr = append(attr, s)
				}
			}
			out = append(out, voiceReply{speech: strings.Join(speech, " "), attr: strings.Join(attr, " ")})
			continue
		}
		if c.replyOpen == "" || c.replyClose == "" {
			continue
		}
		rest := line
		for {
			i := strings.Index(rest, c.replyOpen)
			if i < 0 {
				break
			}
			after := rest[i+len(c.replyOpen):]
			j := strings.Index(after, c.replyClose)
			if j < 0 {
				break
			}
			speech := after[:j]
			tail := after[j+len(c.replyClose):]
			// The attribution of a quoted turn is its tail up to the next quoted turn on the line.
			attr := tail
			if k := strings.Index(tail, c.replyOpen); k >= 0 {
				attr = tail[:k]
			}
			low := []rune(text.NormalizeTargetForm(attr))
			if anySubstring(low, c.speechVerb) && !anySubstring(low, c.innerMarker) {
				out = append(out, voiceReply{speech: speech, attr: attr})
			}
			rest = tail
		}
	}
	return out
}

// countSourceReplies counts quoted turns in the SOURCE that a cue attributes — the denominator that says
// how much direct speech the chapter had, independently of what the translation produced. 0 when the
// source ships no speech-cue data, which the caller reads as "not measured".
//
// A cue attributes AT MOST ONE turn. The scan takes the cue-AFTER position first (the zh «…」X说道» shape)
// and marks the text it consumed, so the next turn's cue-BEFORE scan cannot claim the same words again —
// without that, `“来吧。”方源说道。他走了。“不。”` counted two attributed turns on one cue (measured while
// building this). Both positions are accepted so a cue-before language needs no Go branch.
func (c *Checkers) countSourceReplies(source string) int {
	if c == nil || c.speechCue == nil {
		return 0
	}
	sc := c.speechCue
	isQuote := func(r rune) bool { return sc.QuoteOpen[r] || sc.QuoteClose[r] }
	hasCue := func(s string) bool {
		for _, cue := range sc.Cues {
			if strings.Contains(s, cue) {
				return true
			}
		}
		return false
	}
	rs := []rune(source)
	n, consumed := 0, 0
	for i := 0; i < len(rs); i++ {
		if !sc.QuoteOpen[rs[i]] {
			continue
		}
		j := i + 1
		for ; j < len(rs) && !sc.QuoteClose[rs[j]]; j++ {
		}
		if j >= len(rs) {
			break // an unclosed quote: no turn to attribute
		}
		// Cue AFTER the closing mark, up to the next quote mark or the window edge.
		hi := j + 1
		for hi < len(rs) && hi-(j+1) < sc.Window && !isQuote(rs[hi]) {
			hi++
		}
		// Cue BEFORE the opening mark, back to the previous quote mark, the window edge, or whatever an
		// earlier turn already claimed — whichever comes first.
		lo := i
		for lo > consumed && i-lo < sc.Window && !isQuote(rs[lo-1]) {
			lo--
		}
		switch {
		case hasCue(string(rs[j+1 : hi])):
			n++
			consumed = hi
		case hasCue(string(rs[lo:i])):
			n++
			consumed = i
		}
		i = j
	}
	return n
}

// DECL-AWARE STEMMING IS DELIBERATELY NOT WIRED INTO THESE MATCHERS (fix-pack §е, D39.71). The bank's
// stemmer (lang.TargetStemmer) is the right tool for the POST-CHECK, where a miss is uniformly the safe
// direction (a false omission flag) — and where character NAMES already arrive declension-aware, because
// Forms carries the base dst plus its enumerated decl forms (the same decl.forms[] the bank uses). Stemming
// on top of that would only add stem-collision MIS-ATTRIBUTION risk to a checker whose whole doctrine is
// precision over recall. For SelfRef/NG (single surfaces) the accept-biased stemmer is not uniformly safe
// either: stemming SelfRef would suppress a false self-ref flag on a declined designation (safe), but stemming
// an NG lexeme would EXPAND the forbidden-word match (a recall boost that manufactures noise) — opposite
// directions on one checker. A declension-aware SelfRef/NG belongs in the DRIVER (enumerate forms from the
// seed, as Forms already are), not in a match-time stemmer whose safe direction flips per axis. Recorded here
// so the decision stays closed rather than silently re-opened.

// wholeWord reports whether form occurs in the normalized haystack bounded by non-word runes.
func wholeWord(hay []rune, form string) bool {
	f := []rune(form)
	if len(f) == 0 || len(f) > len(hay) {
		return false
	}
	for i := 0; i+len(f) <= len(hay); i++ {
		if !text.RunesEqual(hay[i:i+len(f)], f) {
			continue
		}
		if (i == 0 || !isWordRune(hay[i-1])) && (i+len(f) == len(hay) || !isWordRune(hay[i+len(f)])) {
			return true
		}
	}
	return false
}

func anyWholeWord(hay []rune, forms []string) bool {
	for _, f := range forms {
		if wholeWord(hay, f) {
			return true
		}
	}
	return false
}

// anySubstring is the loose test used for ATTRIBUTION vocabulary (speech verbs, inner markers), matching
// how the dialogue-dash rule already reads those same lists: a verb may carry a prefix or a suffix the
// list does not enumerate.
func anySubstring(hay []rune, needles []string) bool {
	s := string(hay)
	for _, n := range needles {
		if n != "" && strings.Contains(s, n) {
			return true
		}
	}
	return false
}

func sortedKeys(m map[string]bool) []string {
	if len(m) == 0 {
		return nil
	}
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

func previewRunes(rs []rune) string { return preview(string(rs)) }