package checks import ( "fmt" "sort" "strings" "textmachine/backend/internal/text" ) // voice.go: the deterministic $0 VOICE flagger (pack-19, ratified D39.55) — the layer-1 half of D21's // voice detector. Like every checker here it is PURE observability, never a disposition, and tuned // PRECISION over recall: a miss is silence on a real defect, a false positive is noise on good prose. // // WHAT IT CAN SEE, AND WHY THAT IS THE SHAPE IT IS. The check D21 describes — "a ты/вы flip in an // attributed reply against the registry of PAIRS" — needs BOTH ends of a pair, and only one end is // deterministically reachable without the Annotator this pack does not build (a speaker cue names the // speaker; the addressee is named by almost nothing). So the flagger stands on three axes that need no // addressee at all, and reports the fourth separately: // // A. tv_contradiction — one reply carries both a T and a V surface. Needs neither end. // B. self_ref_flat — an attributed speaker with a declared self-designation refers to herself with // a plain pronoun instead (the measured flattening class, research/15 §Пробы). // C. ng_lexicon — an attributed speaker says a word her profile forbids. Needs the speaker only. // D. pair_register — the registry check. Reported as an INDICATOR, excluded from Total(), because // its addressee comes from a heuristic (exactly two characters on stage), not // from attribution. // // ATTRIBUTION IS TARGET-SIDE, and that is a refinement of the phase-1 design (which sketched a // source-side 说/道 pre-gate): a source cue names a speaker in the SOURCE, while the reply to check is in // the TARGET, and this engine has no source↔target alignment to carry one to the other. The target side // needs none — the attribution tail of a Russian dash line already names the speaker in words the bank // can match. The source cue data is still read, for the honest denominator only (SourceReplies). // // EVERYTHING language-specific is DATA: T/V surfaces, the plural veto, the generic self-reference // pronouns and the reply markers are TARGET data (lang.TargetChecks); the speech cues and quote marks // are SOURCE data (lang.SpeechCue). A target that ships no T/V rows runs the whole flagger inert. // VoiceCheckVersion versions the rules in this file. It is deliberately NOT snapshot-folded (see // config.VoiceGate) — it is logged with the run instead, exactly as terminologyVersion is. const VoiceCheckVersion = "voice-v1-tv+selfref+ng+pair-indicator" // VoiceCharacter is one character as the flagger sees it: an opaque identity, the target surfaces that // name it in the output, and the two profile fields the checks read. Built by the driver from the bank // (surfaces) and the voice profiles (self-ref / ng); this package never touches the store. type VoiceCharacter struct { Key string // stable identity (src\x1fsense) — compared, never rendered into a message Name string // human-readable label for the detail line Forms []string // NORMALIZED target surfaces (base dst + decl forms) SelfRef string // NORMALIZED declared self-designation; "" = the profile states none NG []string // NORMALIZED forbidden lexemes } // VoiceAddressPair is one ordered register record projected onto the current chapter. type VoiceAddressPair struct { Speaker, Addressee string // VoiceCharacter.Key Register string // "informal" | "formal" — abstract; the surfaces are target data } // VoiceRegistry is the per-unit projection the flagger runs against. type VoiceRegistry struct { Chars []VoiceCharacter Pairs []VoiceAddressPair } // HasData reports whether there is anything to check against. func (r VoiceRegistry) HasData() bool { return len(r.Chars) > 0 } // VoiceResult is one unit's flagger outcome. Every field is omitempty, so a clean unit (and any unit of // a book that does not enable the gate) serialises to nothing. type VoiceResult struct { TVContradiction int `json:"tv_contradiction,omitempty"` SelfRefFlat int `json:"self_ref_flat,omitempty"` NGLexicon int `json:"ng_lexicon,omitempty"` // PairRegister is the axis-D INDICATOR: excluded from Total() on purpose. Its addressee is inferred // from "exactly two characters are on stage", which is a heuristic — mixing it into the headline // count would let a structurally weaker signal inflate a number the operator reads as defects. PairRegister int `json:"pair_register,omitempty"` // Replies / Attributed are the denominators. Without them a count is unreadable: 3 flags out of 4 // attributed replies is a broken voice, 3 out of 300 is noise. Replies int `json:"replies,omitempty"` Attributed int `json:"attributed,omitempty"` // SourceReplies is the source-side quoted-turn count (pair data). 0 when the source ships no // speech-cue file — which is "not measured", not "none found". SourceReplies int `json:"source_replies,omitempty"` Detail []string `json:"detail,omitempty"` } // Total is the flagged count that reaches n_voice_flags: axes A–C only (see PairRegister). func (v VoiceResult) Total() int { return v.TVContradiction + v.SelfRefFlat + v.NGLexicon } // voiceReply is one segmented turn: what was said, and the attribution around it. type voiceReply struct { speech string attr string } // RunVoiceChecks runs the flagger over one unit's SOURCE and FINAL text against the projected registry. // Pure and deterministic (sorted detail, no map iteration in output). A nil spec, an empty registry or a // target with no T/V data yields a zero result — the inert path a pair without the distinction takes. func RunVoiceChecks(source, final string, reg VoiceRegistry, c *Checkers) VoiceResult { var res VoiceResult if c == nil { return res } res.SourceReplies = c.countSourceReplies(source) if !reg.HasData() || (len(c.tvInformal) == 0 && len(c.tvFormal) == 0) { return res } replies := c.splitReplies(final) res.Replies = len(replies) if len(replies) == 0 { return res } // Who is on stage at all — the basis of the axis-D addressee heuristic. Computed once over the whole // unit, because a speaker named in one reply's attribution is still present for the next one. onStage := reg.present(final) detail := map[string]bool{} for _, rp := range replies { speech := []rune(text.NormalizeTargetForm(rp.speech)) if len(speech) == 0 { continue } hasT := anyWholeWord(speech, c.tvInformal) hasV := anyWholeWord(speech, c.tvFormal) // Axis A. «Ты иди, а вы оба ждите» is correct Russian addressing two parties, so a plural marker // vetoes the contradiction — the same shape inner_marker has over speech_verb. if hasT && hasV && !anyWholeWord(speech, c.tvPluralVeto) { res.TVContradiction++ detail[fmt.Sprintf("voice: one reply mixes an informal and a formal address: %s", previewRunes(speech))] = true } sp, ok := reg.speakerOf(rp.attr, c) if !ok { continue } res.Attributed++ // Axis B. The evidence is that she referred to herself AT ALL (a generic pronoun) while her // declared designation is absent — not that she used a pronoun, which everyone does. if sp.SelfRef != "" && anyWholeWord(speech, c.selfRefGeneric) && !wholeWord(speech, sp.SelfRef) { res.SelfRefFlat++ detail[fmt.Sprintf("voice: %s refers to herself plainly; the profile declares %q", sp.Name, sp.SelfRef)] = true } // Axis C. for _, ng := range sp.NG { if wholeWord(speech, ng) { res.NGLexicon++ detail[fmt.Sprintf("voice: %s says %q, which the profile forbids", sp.Name, ng)] = true } } // Axis D (indicator). Only when the addressee is unambiguous by the two-on-stage heuristic. if !hasT && !hasV { continue } addressee, ok := onStage.other(sp.Key) if !ok { continue } want, ok := reg.registerFor(sp.Key, addressee) if !ok { continue } got := registerFormal if hasT { got = registerInformal } if got != want { res.PairRegister++ detail[fmt.Sprintf("voice(indicator): %s addresses the other character as %q, the registry says %q", sp.Name, got, want)] = true } } res.Detail = sortedKeys(detail) return res } // registerInformal / registerFormal mirror the abstract vocabulary membank validates the seed against. // They are category names, not target words. const ( registerInformal = "informal" registerFormal = "formal" ) // stageSet is the set of characters whose surfaces appear in a unit, in registry order. type stageSet []string // other returns the single OTHER character on stage, and whether the "exactly two, one of them the // speaker" precondition of the axis-D heuristic holds. Any other cast size gives no addressee. func (s stageSet) other(speaker string) (string, bool) { if len(s) != 2 { return "", false } switch { case s[0] == speaker: return s[1], true case s[1] == speaker: return s[0], true } return "", false } // present returns the characters whose surfaces occur in the text, in registry order (deterministic). func (r VoiceRegistry) present(final string) stageSet { norm := []rune(text.NormalizeTargetForm(final)) var out stageSet for _, ch := range r.Chars { if anyWholeWord(norm, ch.Forms) { out = append(out, ch.Key) } } return out } // speakerOf resolves a reply's attribution to a character. It requires BOTH a target speech verb (so a // narrative sentence that merely names somebody is not an attribution) and EXACTLY ONE character // surface: «— …, — сказал Фан Юань Бай Нинбин» names two, and guessing which one speaks is precisely the // coreference this pack does not have. Ambiguity yields no speaker, which is the silent, precise answer. func (r VoiceRegistry) speakerOf(attr string, c *Checkers) (VoiceCharacter, bool) { if strings.TrimSpace(attr) == "" { return VoiceCharacter{}, false } low := []rune(text.NormalizeTargetForm(attr)) if !anySubstring(low, c.speechVerb) || anySubstring(low, c.innerMarker) { return VoiceCharacter{}, false } var found VoiceCharacter n := 0 for _, ch := range r.Chars { if anyWholeWord(low, ch.Forms) { found, n = ch, n+1 } } if n != 1 { return VoiceCharacter{}, false } return found, true } // registerFor returns the projected register for an ordered pair. func (r VoiceRegistry) registerFor(speaker, addressee string) (string, bool) { for _, p := range r.Pairs { if p.Speaker == speaker && p.Addressee == addressee { return p.Register, true } } return "", false } // splitReplies segments the target text into turns using TARGET data alone. // // A dash line is one speaker's whole turn: splitting «— A, — сказал X. — B» on the marker alternates // speech (odd segments) and attribution (even ones), which is the Russian convention as a rule rather // than as a Go literal. A quoted span is a turn only when its tail carries a speech verb — a bare // «…» is as likely a citation or a thought, and the inner-speech veto removes the rest. func (c *Checkers) splitReplies(final string) []voiceReply { if c == nil || (c.replyDash == "" && (c.replyOpen == "" || c.replyClose == "")) { return nil } var out []voiceReply for _, raw := range strings.Split(final, "\n") { line := strings.TrimSpace(raw) if line == "" { continue } if c.replyDash != "" && strings.HasPrefix(line, c.replyDash) { segs := strings.Split(line, c.replyDash) var speech, attr []string for i, s := range segs { switch { case i == 0: // text before the opening marker (empty by the HasPrefix above) case i%2 == 1: speech = append(speech, s) default: attr = append(attr, s) } } out = append(out, voiceReply{speech: strings.Join(speech, " "), attr: strings.Join(attr, " ")}) continue } if c.replyOpen == "" || c.replyClose == "" { continue } rest := line for { i := strings.Index(rest, c.replyOpen) if i < 0 { break } after := rest[i+len(c.replyOpen):] j := strings.Index(after, c.replyClose) if j < 0 { break } speech := after[:j] tail := after[j+len(c.replyClose):] // The attribution of a quoted turn is its tail up to the next quoted turn on the line. attr := tail if k := strings.Index(tail, c.replyOpen); k >= 0 { attr = tail[:k] } low := []rune(text.NormalizeTargetForm(attr)) if anySubstring(low, c.speechVerb) && !anySubstring(low, c.innerMarker) { out = append(out, voiceReply{speech: speech, attr: attr}) } rest = tail } } return out } // countSourceReplies counts quoted turns in the SOURCE that a cue attributes — the denominator that says // how much direct speech the chapter had, independently of what the translation produced. 0 when the // source ships no speech-cue data, which the caller reads as "not measured". // // A cue attributes AT MOST ONE turn. The scan takes the cue-AFTER position first (the zh «…」X说道» shape) // and marks the text it consumed, so the next turn's cue-BEFORE scan cannot claim the same words again — // without that, `“来吧。”方源说道。他走了。“不。”` counted two attributed turns on one cue (measured while // building this). Both positions are accepted so a cue-before language needs no Go branch. func (c *Checkers) countSourceReplies(source string) int { if c == nil || c.speechCue == nil { return 0 } sc := c.speechCue isQuote := func(r rune) bool { return sc.QuoteOpen[r] || sc.QuoteClose[r] } hasCue := func(s string) bool { for _, cue := range sc.Cues { if strings.Contains(s, cue) { return true } } return false } rs := []rune(source) n, consumed := 0, 0 for i := 0; i < len(rs); i++ { if !sc.QuoteOpen[rs[i]] { continue } j := i + 1 for ; j < len(rs) && !sc.QuoteClose[rs[j]]; j++ { } if j >= len(rs) { break // an unclosed quote: no turn to attribute } // Cue AFTER the closing mark, up to the next quote mark or the window edge. hi := j + 1 for hi < len(rs) && hi-(j+1) < sc.Window && !isQuote(rs[hi]) { hi++ } // Cue BEFORE the opening mark, back to the previous quote mark, the window edge, or whatever an // earlier turn already claimed — whichever comes first. lo := i for lo > consumed && i-lo < sc.Window && !isQuote(rs[lo-1]) { lo-- } switch { case hasCue(string(rs[j+1 : hi])): n++ consumed = hi case hasCue(string(rs[lo:i])): n++ consumed = i } i = j } return n } // DECL-AWARE STEMMING IS DELIBERATELY NOT WIRED INTO THESE MATCHERS (fix-pack §е, D39.71). The bank's // stemmer (lang.TargetStemmer) is the right tool for the POST-CHECK, where a miss is uniformly the safe // direction (a false omission flag) — and where character NAMES already arrive declension-aware, because // Forms carries the base dst plus its enumerated decl forms (the same decl.forms[] the bank uses). Stemming // on top of that would only add stem-collision MIS-ATTRIBUTION risk to a checker whose whole doctrine is // precision over recall. For SelfRef/NG (single surfaces) the accept-biased stemmer is not uniformly safe // either: stemming SelfRef would suppress a false self-ref flag on a declined designation (safe), but stemming // an NG lexeme would EXPAND the forbidden-word match (a recall boost that manufactures noise) — opposite // directions on one checker. A declension-aware SelfRef/NG belongs in the DRIVER (enumerate forms from the // seed, as Forms already are), not in a match-time stemmer whose safe direction flips per axis. Recorded here // so the decision stays closed rather than silently re-opened. // wholeWord reports whether form occurs in the normalized haystack bounded by non-word runes. func wholeWord(hay []rune, form string) bool { f := []rune(form) if len(f) == 0 || len(f) > len(hay) { return false } for i := 0; i+len(f) <= len(hay); i++ { if !text.RunesEqual(hay[i:i+len(f)], f) { continue } if (i == 0 || !isWordRune(hay[i-1])) && (i+len(f) == len(hay) || !isWordRune(hay[i+len(f)])) { return true } } return false } func anyWholeWord(hay []rune, forms []string) bool { for _, f := range forms { if wholeWord(hay, f) { return true } } return false } // anySubstring is the loose test used for ATTRIBUTION vocabulary (speech verbs, inner markers), matching // how the dialogue-dash rule already reads those same lists: a verb may carry a prefix or a suffix the // list does not enumerate. func anySubstring(hay []rune, needles []string) bool { s := string(hay) for _, n := range needles { if n != "" && strings.Contains(s, n) { return true } } return false } func sortedKeys(m map[string]bool) []string { if len(m) == 0 { return nil } out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out } func previewRunes(rs []rune) string { return preview(string(rs)) }