diff --git a/backend/internal/checks/checkers.go b/backend/internal/checks/checkers.go index c8801ad1..764e1789 100644 --- a/backend/internal/checks/checkers.go +++ b/backend/internal/checks/checkers.go @@ -45,6 +45,15 @@ type Checkers struct { innerMarker []string yoHomograph map[string]bool // target-general: ё↔е homograph whitelist (cheapgates yofikator) magnitudeStem map[string]int // target-general: ru magnitude word stem → base-10 exponent (cheapgates) + // The pack-19 voice flagger's TARGET data (voice.go): the T/V surfaces and the plural veto over + // them, the generic self-reference pronouns, and the reply markers this target sets speech with. + // All NORMALIZED at compile time so they compare against normalized output directly. A target that + // ships none of them leaves the flagger inert. + tvInformal, tvFormal, tvPluralVeto, selfRefGeneric []string + replyDash, replyOpen, replyClose string + // speechCue is the SOURCE-side quoted-turn alphabet (pair langpack). nil → the source-reply + // denominator is not measured. + speechCue *lang.SpeechCue shichenRE, ruHoursRE, chengRE, decimalFractionRE *regexp.Regexp qianwanOKRE, shushiwanOKRE, shushiwanFireRE *regexp.Regexp @@ -82,6 +91,16 @@ func CompileCheckers(dc *lang.DCCheckerData, tc lang.TargetChecks) *Checkers { innerMarker: tc.List("inner_marker"), yoHomograph: listToSet(tc.List("yo_homograph")), magnitudeStem: listToStemExp(tc.List("magnitude_stem")), + // pack-19 voice data. Normalized here so every comparison in voice.go is against the same form + // the output is normalized into; the reply markers stay RAW because they are matched on the + // un-normalized line (normalization folds the dash variants and eats the line breaks). + tvInformal: normalizeList(tc.List("tv_informal")), + tvFormal: normalizeList(tc.List("tv_formal")), + tvPluralVeto: normalizeList(tc.List("tv_plural_veto")), + selfRefGeneric: normalizeList(tc.List("self_ref_generic")), + replyDash: first(tc.List("reply_dash")), + replyOpen: first(tc.List("reply_open")), + replyClose: first(tc.List("reply_close")), } if dc != nil { c.numeral, c.ruHours, c.registerNeg = dc.Numeral, dc.RuHours, dc.RegisterNeg @@ -184,6 +203,42 @@ func DCCheckerData(p *lang.Pack) *lang.DCCheckerData { return nil } +// CompileCheckersFor is CompileCheckers over a whole pack — the form the runner uses, because the +// pack-19 voice flagger needs the SOURCE-side speech-cue table too and a nil pack must still yield the +// target-general checkers. Kept as a second entry point rather than a third parameter so no existing +// caller (every checker test) has to learn about a table it does not use. +func CompileCheckersFor(p *lang.Pack, tc lang.TargetChecks) *Checkers { + c := CompileCheckers(DCCheckerData(p), tc) + if p != nil { + c.speechCue = p.SpeechCue + } + return c +} + +// normalizeList folds a target wordlist into the matching form (NFC, lower, ё→е), dropping entries that +// normalize to nothing. Deduped, authored order preserved. +func normalizeList(xs []string) []string { + var out []string + seen := map[string]bool{} + for _, x := range xs { + n := text.NormalizeTargetForm(x) + if n == "" || seen[n] { + continue + } + seen[n] = true + out = append(out, n) + } + return out +} + +// first returns the first entry of a single-valued data category, or "" when the target states none. +func first(xs []string) string { + if len(xs) == 0 { + return "" + } + return xs[0] +} + // mustPairRE compiles a pair detection pattern by key; a missing key → nil (inert sub-checker), a malformed // regex → panic (a corrupt pack, not a silent no-op — the same fail-loud discipline the loader keeps). func mustPairRE(p map[string]string, key string) *regexp.Regexp { diff --git a/backend/internal/checks/voice.go b/backend/internal/checks/voice.go new file mode 100644 index 00000000..0f131d58 --- /dev/null +++ b/backend/internal/checks/voice.go @@ -0,0 +1,419 @@ +package checks + +import ( + "fmt" + "sort" + "strings" + + "textmachine/backend/internal/text" +) + +// voice.go: the deterministic $0 VOICE flagger (pack-19, ratified D39.55) — the layer-1 half of D21's +// voice detector. Like every checker here it is PURE observability, never a disposition, and tuned +// PRECISION over recall: a miss is silence on a real defect, a false positive is noise on good prose. +// +// WHAT IT CAN SEE, AND WHY THAT IS THE SHAPE IT IS. The check D21 describes — "a ты/вы flip in an +// attributed reply against the registry of PAIRS" — needs BOTH ends of a pair, and only one end is +// deterministically reachable without the Annotator this pack does not build (a speaker cue names the +// speaker; the addressee is named by almost nothing). So the flagger stands on three axes that need no +// addressee at all, and reports the fourth separately: +// +// A. tv_contradiction — one reply carries both a T and a V surface. Needs neither end. +// B. self_ref_flat — an attributed speaker with a declared self-designation refers to herself with +// a plain pronoun instead (the measured flattening class, research/15 §Пробы). +// C. ng_lexicon — an attributed speaker says a word her profile forbids. Needs the speaker only. +// D. pair_register — the registry check. Reported as an INDICATOR, excluded from Total(), because +// its addressee comes from a heuristic (exactly two characters on stage), not +// from attribution. +// +// ATTRIBUTION IS TARGET-SIDE, and that is a refinement of the phase-1 design (which sketched a +// source-side 说/道 pre-gate): a source cue names a speaker in the SOURCE, while the reply to check is in +// the TARGET, and this engine has no source↔target alignment to carry one to the other. The target side +// needs none — the attribution tail of a Russian dash line already names the speaker in words the bank +// can match. The source cue data is still read, for the honest denominator only (SourceReplies). +// +// EVERYTHING language-specific is DATA: T/V surfaces, the plural veto, the generic self-reference +// pronouns and the reply markers are TARGET data (lang.TargetChecks); the speech cues and quote marks +// are SOURCE data (lang.SpeechCue). A target that ships no T/V rows runs the whole flagger inert. + +// VoiceCheckVersion versions the rules in this file. It is deliberately NOT snapshot-folded (see +// config.VoiceGate) — it is logged with the run instead, exactly as terminologyVersion is. +const VoiceCheckVersion = "voice-v1-tv+selfref+ng+pair-indicator" + +// VoiceCharacter is one character as the flagger sees it: an opaque identity, the target surfaces that +// name it in the output, and the two profile fields the checks read. Built by the driver from the bank +// (surfaces) and the voice profiles (self-ref / ng); this package never touches the store. +type VoiceCharacter struct { + Key string // stable identity (src\x1fsense) — compared, never rendered into a message + Name string // human-readable label for the detail line + Forms []string // NORMALIZED target surfaces (base dst + decl forms) + SelfRef string // NORMALIZED declared self-designation; "" = the profile states none + NG []string // NORMALIZED forbidden lexemes +} + +// VoiceAddressPair is one ordered register record projected onto the current chapter. +type VoiceAddressPair struct { + Speaker, Addressee string // VoiceCharacter.Key + Register string // "informal" | "formal" — abstract; the surfaces are target data +} + +// VoiceRegistry is the per-unit projection the flagger runs against. +type VoiceRegistry struct { + Chars []VoiceCharacter + Pairs []VoiceAddressPair +} + +// HasData reports whether there is anything to check against. +func (r VoiceRegistry) HasData() bool { return len(r.Chars) > 0 } + +// VoiceResult is one unit's flagger outcome. Every field is omitempty, so a clean unit (and any unit of +// a book that does not enable the gate) serialises to nothing. +type VoiceResult struct { + TVContradiction int `json:"tv_contradiction,omitempty"` + SelfRefFlat int `json:"self_ref_flat,omitempty"` + NGLexicon int `json:"ng_lexicon,omitempty"` + // PairRegister is the axis-D INDICATOR: excluded from Total() on purpose. Its addressee is inferred + // from "exactly two characters are on stage", which is a heuristic — mixing it into the headline + // count would let a structurally weaker signal inflate a number the operator reads as defects. + PairRegister int `json:"pair_register,omitempty"` + // Replies / Attributed are the denominators. Without them a count is unreadable: 3 flags out of 4 + // attributed replies is a broken voice, 3 out of 300 is noise. + Replies int `json:"replies,omitempty"` + Attributed int `json:"attributed,omitempty"` + // SourceReplies is the source-side quoted-turn count (pair data). 0 when the source ships no + // speech-cue file — which is "not measured", not "none found". + SourceReplies int `json:"source_replies,omitempty"` + Detail []string `json:"detail,omitempty"` +} + +// Total is the flagged count that reaches n_voice_flags: axes A–C only (see PairRegister). +func (v VoiceResult) Total() int { return v.TVContradiction + v.SelfRefFlat + v.NGLexicon } + +// voiceReply is one segmented turn: what was said, and the attribution around it. +type voiceReply struct { + speech string + attr string +} + +// RunVoiceChecks runs the flagger over one unit's SOURCE and FINAL text against the projected registry. +// Pure and deterministic (sorted detail, no map iteration in output). A nil spec, an empty registry or a +// target with no T/V data yields a zero result — the inert path a pair without the distinction takes. +func RunVoiceChecks(source, final string, reg VoiceRegistry, c *Checkers) VoiceResult { + var res VoiceResult + if c == nil { + return res + } + res.SourceReplies = c.countSourceReplies(source) + if !reg.HasData() || (len(c.tvInformal) == 0 && len(c.tvFormal) == 0) { + return res + } + replies := c.splitReplies(final) + res.Replies = len(replies) + if len(replies) == 0 { + return res + } + // Who is on stage at all — the basis of the axis-D addressee heuristic. Computed once over the whole + // unit, because a speaker named in one reply's attribution is still present for the next one. + onStage := reg.present(final) + + detail := map[string]bool{} + for _, rp := range replies { + speech := []rune(text.NormalizeTargetForm(rp.speech)) + if len(speech) == 0 { + continue + } + hasT := anyWholeWord(speech, c.tvInformal) + hasV := anyWholeWord(speech, c.tvFormal) + // Axis A. «Ты иди, а вы оба ждите» is correct Russian addressing two parties, so a plural marker + // vetoes the contradiction — the same shape inner_marker has over speech_verb. + if hasT && hasV && !anyWholeWord(speech, c.tvPluralVeto) { + res.TVContradiction++ + detail[fmt.Sprintf("voice: one reply mixes an informal and a formal address: %s", previewRunes(speech))] = true + } + sp, ok := reg.speakerOf(rp.attr, c) + if !ok { + continue + } + res.Attributed++ + // Axis B. The evidence is that she referred to herself AT ALL (a generic pronoun) while her + // declared designation is absent — not that she used a pronoun, which everyone does. + if sp.SelfRef != "" && anyWholeWord(speech, c.selfRefGeneric) && !wholeWord(speech, sp.SelfRef) { + res.SelfRefFlat++ + detail[fmt.Sprintf("voice: %s refers to herself plainly; the profile declares %q", sp.Name, sp.SelfRef)] = true + } + // Axis C. + for _, ng := range sp.NG { + if wholeWord(speech, ng) { + res.NGLexicon++ + detail[fmt.Sprintf("voice: %s says %q, which the profile forbids", sp.Name, ng)] = true + } + } + // Axis D (indicator). Only when the addressee is unambiguous by the two-on-stage heuristic. + if !hasT && !hasV { + continue + } + addressee, ok := onStage.other(sp.Key) + if !ok { + continue + } + want, ok := reg.registerFor(sp.Key, addressee) + if !ok { + continue + } + got := registerFormal + if hasT { + got = registerInformal + } + if got != want { + res.PairRegister++ + detail[fmt.Sprintf("voice(indicator): %s addresses the other character as %q, the registry says %q", sp.Name, got, want)] = true + } + } + res.Detail = sortedKeys(detail) + return res +} + +// registerInformal / registerFormal mirror the abstract vocabulary membank validates the seed against. +// They are category names, not target words. +const ( + registerInformal = "informal" + registerFormal = "formal" +) + +// stageSet is the set of characters whose surfaces appear in a unit, in registry order. +type stageSet []string + +// other returns the single OTHER character on stage, and whether the "exactly two, one of them the +// speaker" precondition of the axis-D heuristic holds. Any other cast size gives no addressee. +func (s stageSet) other(speaker string) (string, bool) { + if len(s) != 2 { + return "", false + } + switch { + case s[0] == speaker: + return s[1], true + case s[1] == speaker: + return s[0], true + } + return "", false +} + +// present returns the characters whose surfaces occur in the text, in registry order (deterministic). +func (r VoiceRegistry) present(final string) stageSet { + norm := []rune(text.NormalizeTargetForm(final)) + var out stageSet + for _, ch := range r.Chars { + if anyWholeWord(norm, ch.Forms) { + out = append(out, ch.Key) + } + } + return out +} + +// speakerOf resolves a reply's attribution to a character. It requires BOTH a target speech verb (so a +// narrative sentence that merely names somebody is not an attribution) and EXACTLY ONE character +// surface: «— …, — сказал Фан Юань Бай Нинбин» names two, and guessing which one speaks is precisely the +// coreference this pack does not have. Ambiguity yields no speaker, which is the silent, precise answer. +func (r VoiceRegistry) speakerOf(attr string, c *Checkers) (VoiceCharacter, bool) { + if strings.TrimSpace(attr) == "" { + return VoiceCharacter{}, false + } + low := []rune(text.NormalizeTargetForm(attr)) + if !anySubstring(low, c.speechVerb) || anySubstring(low, c.innerMarker) { + return VoiceCharacter{}, false + } + var found VoiceCharacter + n := 0 + for _, ch := range r.Chars { + if anyWholeWord(low, ch.Forms) { + found, n = ch, n+1 + } + } + if n != 1 { + return VoiceCharacter{}, false + } + return found, true +} + +// registerFor returns the projected register for an ordered pair. +func (r VoiceRegistry) registerFor(speaker, addressee string) (string, bool) { + for _, p := range r.Pairs { + if p.Speaker == speaker && p.Addressee == addressee { + return p.Register, true + } + } + return "", false +} + +// splitReplies segments the target text into turns using TARGET data alone. +// +// A dash line is one speaker's whole turn: splitting «— A, — сказал X. — B» on the marker alternates +// speech (odd segments) and attribution (even ones), which is the Russian convention as a rule rather +// than as a Go literal. A quoted span is a turn only when its tail carries a speech verb — a bare +// «…» is as likely a citation or a thought, and the inner-speech veto removes the rest. +func (c *Checkers) splitReplies(final string) []voiceReply { + if c == nil || (c.replyDash == "" && (c.replyOpen == "" || c.replyClose == "")) { + return nil + } + var out []voiceReply + for _, raw := range strings.Split(final, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + if c.replyDash != "" && strings.HasPrefix(line, c.replyDash) { + segs := strings.Split(line, c.replyDash) + var speech, attr []string + for i, s := range segs { + switch { + case i == 0: // text before the opening marker (empty by the HasPrefix above) + case i%2 == 1: + speech = append(speech, s) + default: + attr = append(attr, s) + } + } + out = append(out, voiceReply{speech: strings.Join(speech, " "), attr: strings.Join(attr, " ")}) + continue + } + if c.replyOpen == "" || c.replyClose == "" { + continue + } + rest := line + for { + i := strings.Index(rest, c.replyOpen) + if i < 0 { + break + } + after := rest[i+len(c.replyOpen):] + j := strings.Index(after, c.replyClose) + if j < 0 { + break + } + speech := after[:j] + tail := after[j+len(c.replyClose):] + // The attribution of a quoted turn is its tail up to the next quoted turn on the line. + attr := tail + if k := strings.Index(tail, c.replyOpen); k >= 0 { + attr = tail[:k] + } + low := []rune(text.NormalizeTargetForm(attr)) + if anySubstring(low, c.speechVerb) && !anySubstring(low, c.innerMarker) { + out = append(out, voiceReply{speech: speech, attr: attr}) + } + rest = tail + } + } + return out +} + +// countSourceReplies counts quoted turns in the SOURCE that a cue attributes — the denominator that says +// how much direct speech the chapter had, independently of what the translation produced. 0 when the +// source ships no speech-cue data, which the caller reads as "not measured". +// +// A cue attributes AT MOST ONE turn. The scan takes the cue-AFTER position first (the zh «…」X说道» shape) +// and marks the text it consumed, so the next turn's cue-BEFORE scan cannot claim the same words again — +// without that, `“来吧。”方源说道。他走了。“不。”` counted two attributed turns on one cue (measured while +// building this). Both positions are accepted so a cue-before language needs no Go branch. +func (c *Checkers) countSourceReplies(source string) int { + if c == nil || c.speechCue == nil { + return 0 + } + sc := c.speechCue + isQuote := func(r rune) bool { return sc.QuoteOpen[r] || sc.QuoteClose[r] } + hasCue := func(s string) bool { + for _, cue := range sc.Cues { + if strings.Contains(s, cue) { + return true + } + } + return false + } + rs := []rune(source) + n, consumed := 0, 0 + for i := 0; i < len(rs); i++ { + if !sc.QuoteOpen[rs[i]] { + continue + } + j := i + 1 + for ; j < len(rs) && !sc.QuoteClose[rs[j]]; j++ { + } + if j >= len(rs) { + break // an unclosed quote: no turn to attribute + } + // Cue AFTER the closing mark, up to the next quote mark or the window edge. + hi := j + 1 + for hi < len(rs) && hi-(j+1) < sc.Window && !isQuote(rs[hi]) { + hi++ + } + // Cue BEFORE the opening mark, back to the previous quote mark, the window edge, or whatever an + // earlier turn already claimed — whichever comes first. + lo := i + for lo > consumed && i-lo < sc.Window && !isQuote(rs[lo-1]) { + lo-- + } + switch { + case hasCue(string(rs[j+1 : hi])): + n++ + consumed = hi + case hasCue(string(rs[lo:i])): + n++ + consumed = i + } + i = j + } + return n +} + +// wholeWord reports whether form occurs in the normalized haystack bounded by non-word runes. +func wholeWord(hay []rune, form string) bool { + f := []rune(form) + if len(f) == 0 || len(f) > len(hay) { + return false + } + for i := 0; i+len(f) <= len(hay); i++ { + if !text.RunesEqual(hay[i:i+len(f)], f) { + continue + } + if (i == 0 || !isWordRune(hay[i-1])) && (i+len(f) == len(hay) || !isWordRune(hay[i+len(f)])) { + return true + } + } + return false +} + +func anyWholeWord(hay []rune, forms []string) bool { + for _, f := range forms { + if wholeWord(hay, f) { + return true + } + } + return false +} + +// anySubstring is the loose test used for ATTRIBUTION vocabulary (speech verbs, inner markers), matching +// how the dialogue-dash rule already reads those same lists: a verb may carry a prefix or a suffix the +// list does not enumerate. +func anySubstring(hay []rune, needles []string) bool { + s := string(hay) + for _, n := range needles { + if n != "" && strings.Contains(s, n) { + return true + } + } + return false +} + +func sortedKeys(m map[string]bool) []string { + if len(m) == 0 { + return nil + } + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func previewRunes(rs []rune) string { return preview(string(rs)) } diff --git a/backend/internal/checks/voice_test.go b/backend/internal/checks/voice_test.go new file mode 100644 index 00000000..bfea4809 --- /dev/null +++ b/backend/internal/checks/voice_test.go @@ -0,0 +1,220 @@ +package checks + +import ( + "strings" + "testing" + + "textmachine/backend/internal/lang" +) + +// voice_test.go pins the four axes AND the inert paths — the second half matters as much as the first, +// because "a pair without T/V activates nothing without a Go edit" is the §0 promise this flagger makes. + +func ruVoiceCheckers() *Checkers { return CompileCheckers(nil, lang.TargetChecksFor("ru")) } + +// служанка is the worked example of research/15: a declared self-designation, a forbidden lexeme, and a +// formal register toward her master. +func maid() VoiceCharacter { + return VoiceCharacter{Key: "maid", Name: "Бай Нинбин", Forms: []string{"бай нинбин", "бай нинбин"}, + SelfRef: "ваша служанка", NG: []string{"ладно"}} +} + +// The declined forms are what the bank actually supplies (decl), and axis-D/attribution precision +// depends on them: without «фан юаню» the ambiguous-attribution case below would read as unambiguous. +func master() VoiceCharacter { + return VoiceCharacter{Key: "master", Name: "Фан Юань", Forms: []string{"фан юань", "фан юаню", "фан юаня"}} +} + +func twoCharRegistry() VoiceRegistry { + return VoiceRegistry{ + Chars: []VoiceCharacter{maid(), master()}, + Pairs: []VoiceAddressPair{{Speaker: "maid", Addressee: "master", Register: "formal"}}, + } +} + +func TestVoiceAxisATVContradictionInOneReply(t *testing.T) { + c := ruVoiceCheckers() + got := RunVoiceChecks("", "— Ты придёшь, но вас я не звала, — сказала Бай Нинбин.", twoCharRegistry(), c) + if got.TVContradiction != 1 { + t.Fatalf("one reply carrying both registers must flag, got %+v", got) + } + if got.Total() == 0 { + t.Fatal("axis A must reach the headline count") + } + // The plural veto: addressing one person and a group in one turn is correct Russian, not a flip. + got = RunVoiceChecks("", "— Ты иди, а вы оба ждите, — сказала Бай Нинбин.", twoCharRegistry(), c) + if got.TVContradiction != 0 { + t.Fatalf("a plural-addressing marker must veto the contradiction, got %+v", got) + } + // One register alone is not a contradiction. + got = RunVoiceChecks("", "— Вы придёте? — спросила Бай Нинбин.", twoCharRegistry(), c) + if got.TVContradiction != 0 { + t.Fatalf("a single register is no contradiction, got %+v", got) + } +} + +func TestVoiceAxisBSelfReferenceFlattened(t *testing.T) { + c := ruVoiceCheckers() + got := RunVoiceChecks("", "— Я всё сделаю, — сказала Бай Нинбин.", twoCharRegistry(), c) + if got.SelfRefFlat != 1 { + t.Fatalf("a plain pronoun where the profile declares a self-designation must flag, got %+v", got) + } + // The declared designation present → no flag, even beside a pronoun. + got = RunVoiceChecks("", "— Ваша служанка всё сделает, я не подведу, — сказала Бай Нинбин.", twoCharRegistry(), c) + if got.SelfRefFlat != 0 { + t.Fatalf("the declared designation is present — nothing was flattened, got %+v", got) + } + // No self-reference at all → no evidence either way. + got = RunVoiceChecks("", "— Хорошо, — сказала Бай Нинбин.", twoCharRegistry(), c) + if got.SelfRefFlat != 0 { + t.Fatalf("a reply with no self-reference is not evidence of flattening, got %+v", got) + } + // A character with no declared self_ref is never checked on this axis. + reg := twoCharRegistry() + reg.Chars[0].SelfRef = "" + if got := RunVoiceChecks("", "— Я всё сделаю, — сказала Бай Нинбин.", reg, c); got.SelfRefFlat != 0 { + t.Fatalf("no declared self_ref means no axis-B check, got %+v", got) + } +} + +func TestVoiceAxisCForbiddenLexeme(t *testing.T) { + c := ruVoiceCheckers() + got := RunVoiceChecks("", "— Ладно, ваша служанка сделает, — сказала Бай Нинбин.", twoCharRegistry(), c) + if got.NGLexicon != 1 { + t.Fatalf("a forbidden lexeme in an attributed reply must flag, got %+v", got) + } + if len(got.Detail) == 0 || !strings.Contains(strings.Join(got.Detail, "|"), "Бай Нинбин") { + t.Fatalf("the detail must name the character, got %v", got.Detail) + } + // The SAME word in somebody else's reply is not this character's defect. + got = RunVoiceChecks("", "— Ладно, — сказал Фан Юань.", twoCharRegistry(), c) + if got.NGLexicon != 0 { + t.Fatalf("another speaker's word must not be charged to this profile, got %+v", got) + } +} + +func TestVoiceAxisDIsAnIndicatorOutsideTheCount(t *testing.T) { + c := ruVoiceCheckers() + // Both characters are on stage, the maid speaks, the registry says she owes him the formal register, + // and she uses the informal one. + got := RunVoiceChecks("", "— Ты придёшь, Фан Юань? — спросила Бай Нинбин.", twoCharRegistry(), c) + if got.PairRegister != 1 { + t.Fatalf("a register contradicting the registry must be indicated, got %+v", got) + } + if got.Total() != 0 { + t.Fatalf("axis D must stay OUT of the headline count, got total=%d (%+v)", got.Total(), got) + } + // Matching the registry → nothing. + got = RunVoiceChecks("", "— Вы придёте, Фан Юань? — спросила Бай Нинбин.", twoCharRegistry(), c) + if got.PairRegister != 0 { + t.Fatalf("the declared register must not be indicated, got %+v", got) + } + // A third character on stage breaks the two-on-stage heuristic → no addressee, no indicator. + reg := twoCharRegistry() + reg.Chars = append(reg.Chars, VoiceCharacter{Key: "third", Name: "Третий", Forms: []string{"третий"}}) + // The third name sits in the NARRATIVE, not in the attribution, so the speaker stays unambiguous and + // what is actually under test is the two-on-stage precondition rather than attribution ambiguity. + got = RunVoiceChecks("", "— Ты придёшь, Фан Юань? — спросила Бай Нинбин.\nТретий молчал.", reg, c) + if got.Attributed != 1 { + t.Fatalf("the speaker must still be attributed — otherwise this case tests the wrong rule: %+v", got) + } + if got.PairRegister != 0 { + t.Fatalf("with three characters on stage the addressee is unknown — no indicator, got %+v", got) + } +} + +func TestVoiceAttributionRequiresOneSpeakerAndASpeechVerb(t *testing.T) { + c := ruVoiceCheckers() + // No attribution at all: the reply still counts, but nothing speaker-scoped is checked. + got := RunVoiceChecks("", "— Я всё сделаю.", twoCharRegistry(), c) + if got.Replies != 1 || got.Attributed != 0 || got.SelfRefFlat != 0 { + t.Fatalf("an unattributed reply is counted but not charged to anybody, got %+v", got) + } + // TWO characters in the attribution: guessing which one speaks is the coreference this pack lacks. + got = RunVoiceChecks("", "— Я всё сделаю, — сказала Бай Нинбин Фан Юаню.", twoCharRegistry(), c) + if got.Attributed != 0 { + t.Fatalf("an ambiguous attribution must yield no speaker, got %+v", got) + } + // A narrative sentence that merely names somebody is not an attribution. + got = RunVoiceChecks("", "— Я всё сделаю, — Бай Нинбин смотрела в окно.", twoCharRegistry(), c) + if got.Attributed != 0 { + t.Fatalf("no speech verb means no attribution, got %+v", got) + } +} + +func TestVoiceQuotedRepliesNeedASpeechVerbAndRespectTheInnerVeto(t *testing.T) { + c := ruVoiceCheckers() + got := RunVoiceChecks("", "«Я всё сделаю», — сказала Бай Нинбин.", twoCharRegistry(), c) + if got.Replies != 1 || got.SelfRefFlat != 1 { + t.Fatalf("a quoted turn with a speech verb is a reply, got %+v", got) + } + // Inner speech is a thought, not an addressed turn: no register, no self-designation duty. + got = RunVoiceChecks("", "«Я всё сделаю», — пробормотала про себя Бай Нинбин.", twoCharRegistry(), c) + if got.Replies != 0 { + t.Fatalf("inner speech is not an addressed reply, got %+v", got) + } + // A bare citation is not a reply either. + got = RunVoiceChecks("", "На двери висела табличка «Я всё сделаю».", twoCharRegistry(), c) + if got.Replies != 0 { + t.Fatalf("a citation is not a reply, got %+v", got) + } +} + +// The §0 promise, executed: with no target T/V data the whole flagger is inert — no Go edit needed for a +// pair whose target has no such distinction. +func TestVoiceFlaggerIsInertWithoutTargetData(t *testing.T) { + bare := CompileCheckers(nil, lang.TargetChecks{}) + got := RunVoiceChecks("", "— Ты придёшь, но вас я не звала, — сказала Бай Нинбин.", twoCharRegistry(), bare) + if got.Total() != 0 || got.Replies != 0 || got.PairRegister != 0 { + t.Fatalf("a target with no voice data must flag nothing, got %+v", got) + } + // And with data but no registry (a book that authored no profiles) — equally inert. + if got := RunVoiceChecks("", "— Ты и вы, — сказал он.", VoiceRegistry{}, ruVoiceCheckers()); got.Total() != 0 { + t.Fatalf("an empty registry must flag nothing, got %+v", got) + } + if RunVoiceChecks("", "текст", twoCharRegistry(), nil).Total() != 0 { + t.Fatal("a nil checker spec must be inert, not a panic") + } + // The case that isolates the T/V guard from the segmentation guard: a target that DOES mark dialogue + // (so replies are found and attributed) but has NO T/V distinction — an en-like target. Nothing on the + // register axes may fire, with no Go edit anywhere. + noTV := &Checkers{ + replyDash: "—", + speechVerb: []string{"сказала"}, + selfRefGeneric: []string{"я"}, + } + got = RunVoiceChecks("", "— Ты придёшь, но вас я не звала, — сказала Бай Нинбин.", twoCharRegistry(), noTV) + if got.Total() != 0 || got.Replies != 0 || got.PairRegister != 0 { + t.Fatalf("a target with dialogue markers but no T/V distinction must flag nothing, got %+v", got) + } +} + +// The source-side denominator: read from PAIR data, and 0 (not "none") when the source ships no file. +func TestVoiceSourceReplyDenominator(t *testing.T) { + c := ruVoiceCheckers() + if got := RunVoiceChecks("“来吧。”方源说道。", "", twoCharRegistry(), c); got.SourceReplies != 0 { + t.Fatalf("without a speech-cue table the count is NOT MEASURED, got %d", got.SourceReplies) + } + c.speechCue = &lang.SpeechCue{ + Cues: []string{"说", "道"}, + QuoteOpen: map[rune]bool{'“': true}, + QuoteClose: map[rune]bool{'”': true}, + Window: 12, + } + got := RunVoiceChecks("“来吧。”方源说道。他走了。“不。”", "", twoCharRegistry(), c) + if got.SourceReplies != 1 { + t.Fatalf("only the cue-attributed quoted turn counts, got %d", got.SourceReplies) + } +} + +func TestVoiceResultIsDeterministic(t *testing.T) { + c := ruVoiceCheckers() + const final = "— Ладно, я всё сделаю, но вас я не звала, — сказала Бай Нинбин." + first := RunVoiceChecks("", final, twoCharRegistry(), c) + for i := 0; i < 20; i++ { + if got := RunVoiceChecks("", final, twoCharRegistry(), c); strings.Join(got.Detail, "|") != strings.Join(first.Detail, "|") || + got.Total() != first.Total() { + t.Fatalf("run %d diverged: %+v vs %+v", i, got, first) + } + } +} diff --git a/backend/internal/config/pipeline.go b/backend/internal/config/pipeline.go index dfeea145..48e54a04 100644 --- a/backend/internal/config/pipeline.go +++ b/backend/internal/config/pipeline.go @@ -251,6 +251,26 @@ type Gates struct { Banknote BanknoteGate `yaml:"banknote"` Repair RepairGate `yaml:"repair"` Terminology TerminologyGate `yaml:"terminology"` + Voice VoiceGate `yaml:"voice"` +} + +// VoiceGate controls the deterministic $0 voice flagger (pack-19, D39.55): T/V contradictions, +// flattened self-designations and forbidden lexemes in attributed replies, measured against the book's +// voice profiles and address-register journal. +// +// Opt-in and OFF by default like every other gate here, and — like gates.terminology, unlike coverage / +// sanitizer / banknote — deliberately NOT snapshot-folded even when on. The reason is the same one: it +// touches neither the wire nor a checkpoint's resolved verdict. It makes no call, changes no message and +// never becomes a disposition; its counters live in retrieval_state, which every run recomputes from the +// stored text and which self-heals on resume. Folding it would re-bill a whole wave for switching on a +// measurement — and folding its VERSION would re-bill every book of every pair for a rule edit, because +// StyleCheckVersion (the sibling it might otherwise have joined) is folded unconditionally. +// +// The accepted cost of not folding: editing a rule shifts recorded counts between runs under one +// snapshot. checks.VoiceCheckVersion is logged with the run and carried in the report so the numbers +// stay attributable to the rules that produced them — the mitigation the terminologist already uses. +type VoiceGate struct { + Enabled bool `yaml:"enabled"` } // TerminologyGate controls the TERMINOLOGIST role (pack-20, D39.42 п.1): between the draft wave and the diff --git a/backend/internal/lang/data/target-ru.txt b/backend/internal/lang/data/target-ru.txt index 7fd93207..5eebe5e4 100644 --- a/backend/internal/lang/data/target-ru.txt +++ b/backend/internal/lang/data/target-ru.txt @@ -107,3 +107,62 @@ sanitizer_preamble (?is)^\s*ниже\s+(?:привед[её]н|представ sanitizer_trailing_note (?i)^[*_>\s-]{0,4}(?:(?:примечани[ея]|заметк[аи]|комментари[йяю]|пояснени[ея]|сноск[аи])\s*:|(?:примечани[ея]|заметк[аи]|комментари[йяю])\s+(?:переводчик|редактор)[а-яё]*|прим\.\s*(?:перев|ред)\.?) sanitizer_edit_meta (?im)^[*_>\s-]{0,4}(?:(?:внесённ|внесен)[а-яё]+\s+правк|основн[а-яё]+\s+правк[а-яё]*(?:\s+для\s+справк[а-яё]+|\s*:)|что\s+(?:было\s+)?(?:изменен|исправлен)|список\s+(?:правок|изменений)|список\s+внесённых) sanitizer_invalid_sign (?i)(?:^|\P{L})[ъь][а-яё]|[ъь][ъь] +# --- pack-19 voice flagger (T/V + attributed-reply checks) ----------------------------------------- +# The T/V distinction is a fact about the TARGET language, so it lives here: a target that ships no +# tv_informal/tv_formal rows runs the whole voice flagger inert, with no Go edit. Surfaces are matched +# as WHOLE WORDS on the normalized output, lower-cased; only PRONOUNS and possessives are listed — +# verb agreement is morphology and belongs to the ru-target layer, not to a $0 flagger. +tv_informal ты +tv_informal тебя +tv_informal тебе +tv_informal тобой +tv_informal тобою +tv_informal твой +tv_informal твоя +tv_informal твоё +tv_informal твое +tv_informal твои +tv_informal твоего +tv_informal твоей +tv_informal твоих +tv_informal твоим +tv_informal твою +tv_formal вы +tv_formal вас +tv_formal вам +tv_formal вами +tv_formal ваш +tv_formal ваша +tv_formal ваше +tv_formal ваши +tv_formal вашего +tv_formal вашей +tv_formal ваших +tv_formal вашим +tv_formal вашу +# tv_plural_veto: Russian «вы» is ALSO plural-you, so a reply addressing several people legitimately +# mixes «ты» and «вы» («Ты иди, а вы оба ждите»). A reply carrying one of these markers is not counted +# as a T/V contradiction — the same veto shape inner_marker has over speech_verb, and for the same +# precision-over-recall reason: a miss here is silence on a real clash, not noise on good prose. +tv_plural_veto оба +tv_plural_veto обе +tv_plural_veto вдвоём +tv_plural_veto втроём +tv_plural_veto все вы +tv_plural_veto вы все +tv_plural_veto господа +tv_plural_veto каждый из вас +# self_ref_generic: the PLAIN first-person surfaces. They are the evidence that a character referred to +# herself at all — the check is «she said "я" where her profile says "ваша служанка"», i.e. the measured +# flattening class (research/15 §Пробы, marker 奴婢), never «she said "я"» on its own. +self_ref_generic я +self_ref_generic меня +self_ref_generic мне +self_ref_generic мной +self_ref_generic мною +# reply segmentation: how this target sets direct speech. reply_dash opens a dash-marked turn at line +# start; reply_open/reply_close bound a quoted turn. Data, not a Go literal, so a target with other +# conventions (ja 「」) states its own without touching the segmentation algorithm. +reply_dash — +reply_open « +reply_close » diff --git a/backend/internal/lang/langpack.go b/backend/internal/lang/langpack.go index 4f08bea3..6ace3e67 100644 --- a/backend/internal/lang/langpack.go +++ b/backend/internal/lang/langpack.go @@ -82,6 +82,12 @@ type Pack struct { // data/algorithm boundary the miner tables keep. Heading *HeadingRule + // SpeechCue is the OPTIONAL SOURCE-side direct-speech alphabet (configs/langpacks//speech-cue.txt); + // nil when the source ships no file, and then the source-side reply count is simply 0. It is source + // data because "a quoted turn is 「…」 with 说/道 beside it" is a fact about the source language, not + // about the pair or the algorithm. + SpeechCue *SpeechCue + // Terminology is the OPTIONAL pair sizing of the terminologist's source contexts // (configs/langpacks//terminology.txt); nil when the pair ships no file. It exists because the // KWIC width is measured in RUNES, and N runes buy a different amount of context in every script, so @@ -91,6 +97,16 @@ type Pack struct { version string } +// SpeechCue is the source language's direct-speech alphabet: the attribution cues and the quote marks +// that bound a quoted turn, plus how far from a quote mark a cue still counts as attributing it. Data +// only — the counting ALGORITHM lives in the pipeline, like every other table here. +type SpeechCue struct { + Cues []string // attribution cues in authored order (说/道/问 …) + QuoteOpen map[rune]bool // opening quote marks + QuoteClose map[rune]bool // closing quote marks + Window int // runes on either side of a quote mark within which a cue attributes it +} + // TerminologySizing is the pair's own "how much source context is one context". A zero field means // unstated and falls through to the engine default; the order lives in pipeline.terminologyOpts. type TerminologySizing struct { @@ -241,6 +257,20 @@ func Load(root, sourceLang, targetLang string) (*Pack, error) { p.DCCheckers = dc } + // Optional per-SOURCE speech-cue alphabet (pack-19). Same optional contract as heading.txt, but read + // from the SOURCE directory: 说/道 and 「」 are facts about the source language, not about the pair. + if sb, ok, serr := readOptional(root, sourceLang, "speech-cue.txt"); serr != nil { + return nil, fmt.Errorf("langpack %q speech-cue.txt: %w", pair, serr) + } else if ok { + h.Write([]byte("\x00" + sourceLang + "/speech-cue.txt\x00")) + h.Write(sb) + sc, perr := parseSpeechCue(sb) + if perr != nil { + return nil, fmt.Errorf("langpack %q speech-cue.txt: %w", pair, perr) + } + p.SpeechCue = sc + } + // Optional per-pair TERMINOLOGIST sizing. Same contract as heading.txt: ABSENT → nil, the engine // defaults apply and Version() is byte-stable, so shipping the mechanism re-bills nobody. A pair whose // measured value equals the default deliberately ships no file — the bytes would move the pack version @@ -588,6 +618,54 @@ func parseHeading(b []byte) (*HeadingRule, error) { return hr, nil } +// parseSpeechCue reads speech-cue.txt into a SpeechCue. Format: `keyvalue` per non-comment line, +// keys cue | quote_open | quote_close | window. A file that ships cues but no quote marks (or the +// reverse) can never attribute anything, so it is refused rather than loaded inert — the same +// "never silently empty" contract the required tables keep. +func parseSpeechCue(b []byte) (*SpeechCue, error) { + sc := &SpeechCue{QuoteOpen: map[rune]bool{}, QuoteClose: map[rune]bool{}} + for i, raw := range strings.Split(string(b), "\n") { + t := strings.TrimSpace(strings.TrimRight(raw, "\r")) + if t == "" || strings.HasPrefix(t, "#") { + continue + } + f := strings.SplitN(t, "\t", 2) + if len(f) != 2 || strings.TrimSpace(f[1]) == "" { + return nil, fmt.Errorf("line %d: want `keyvalue` with a non-empty value (%q)", i+1, t) + } + key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1]) + switch key { + case "cue": + sc.Cues = append(sc.Cues, val) + case "quote_open", "quote_close": + r := []rune(val) + if len(r) != 1 { + return nil, fmt.Errorf("line %d: %s %q must be a single rune", i+1, key, val) + } + if key == "quote_open" { + sc.QuoteOpen[r[0]] = true + } else { + sc.QuoteClose[r[0]] = true + } + case "window": + n, err := strconv.Atoi(val) + if err != nil || n <= 0 { + return nil, fmt.Errorf("line %d: window must be a positive integer (%q)", i+1, val) + } + sc.Window = n + default: + return nil, fmt.Errorf("line %d: unknown key %q (want cue|quote_open|quote_close|window)", i+1, key) + } + } + if len(sc.Cues) == 0 || len(sc.QuoteOpen) == 0 || len(sc.QuoteClose) == 0 { + return nil, fmt.Errorf("speech-cue needs at least one cue and both quote_open and quote_close (a half table can never attribute anything)") + } + if sc.Window == 0 { + sc.Window = 12 // engine default: a name+cue sits within a dozen runes of the quote in practice + } + return sc, nil +} + // parseTerminology reads terminology.txt into a TerminologySizing. Format: `keyvalue`, keys // kwic_per_term | kwic_width, each optional but positive when present. Fail-loud like parseHeading; an // empty file is refused too, because its bytes move Version() while changing nothing. diff --git a/backend/internal/membank/memory.go b/backend/internal/membank/memory.go index 3097f394..39466d31 100644 --- a/backend/internal/membank/memory.go +++ b/backend/internal/membank/memory.go @@ -170,6 +170,10 @@ type Bank struct { // equality but STABILITY: base stays fixed as mined rows are added; only enriched moves. enrichedVersion string baseVersion string + // voices/pairs are the pack-19 record types. They are bank CONTENT but never matchable SURFACES, so + // they sit beside the automaton rather than inside it: nothing here can put them in keyOwners. + voices []store.VoiceProfile + pairs []store.AddressPair } // PickedEntry is one selected record for a chunk with its firing key and disposition. @@ -233,13 +237,35 @@ type Selection struct { func (b *Bank) Version() string { return b.enrichedVersion } func (b *Bank) BaseVersion() string { return b.baseVersion } -// Materialize builds a Bank from the book's stored glossary rows (ORDER -// BY-stable — GlossaryForBook). Pure and deterministic. It computes memoryVersion as a +// BankInput is everything a book's bank is materialized and hashed from: the glossary rows plus the two +// pack-19 record types. It exists so the fold has ONE argument that can grow, and so the separation that +// matters is carried by the type system: only Rows ever reaches the matcher, so a voice profile cannot +// become a matchable term by anyone forgetting a filter. +type BankInput struct { + Rows []store.GlossaryEntry + Voices []store.VoiceProfile + Pairs []store.AddressPair + // InjectVoice reports whether the voice/address rows reach the WIRE on this run. It is the CONDITION + // of their fold: while they only feed the $0 flagger (which recomputes every run and self-heals), + // hashing them would re-bill a book for authoring a profile that changed no byte the model sees. + // The moment they are injected, they must fold — the D39.42 п.3 class, in the other direction. + InjectVoice bool +} + +// Materialize builds a Bank from the book's stored glossary rows alone — the pre-pack-19 form, kept for +// every caller that has no voice/address content. +func Materialize(rows []store.GlossaryEntry, gateOn bool) *Bank { + return MaterializeBank(BankInput{Rows: rows}, gateOn) +} + +// MaterializeBank builds a Bank from the book's stored bank content (ORDER BY-stable — GlossaryForBook / +// VoiceProfilesForBook / AddressPairsForBook). Pure and deterministic. It computes memoryVersion as a // content hash of the frozen APPROVED rows (D8: content-hash, not a version-counter — // drift-proof) plus the normalization + matcher algorithm versions, so ANY change to // the approved glossary OR to the deterministic machinery is a loud --resnapshot (F1). -func Materialize(rows []store.GlossaryEntry, gateOn bool) *Bank { - b := &Bank{keyOwners: map[string][]int{}} +func MaterializeBank(in BankInput, gateOn bool) *Bank { + rows := in.Rows + b := &Bank{keyOwners: map[string][]int{}, voices: in.Voices, pairs: in.Pairs} var allKeys []string seenKey := map[string]bool{} @@ -302,11 +328,18 @@ func Materialize(rows []store.GlossaryEntry, gateOn bool) *Bank { } sort.Strings(allKeys) // deterministic automaton construction b.ac = buildAC(allKeys) - b.enrichedVersion = ComputeVersion(rows, gateOn) // all approved incl mined (the edit wave) - b.baseVersion = ComputeVersionScoped(rows, gateOn, true) // excl Source:mined (the draft wave) + b.enrichedVersion = ComputeVersionScopedIn(in, gateOn, false) // all approved incl mined (the edit wave) + b.baseVersion = ComputeVersionScopedIn(in, gateOn, true) // excl Source:mined (the draft wave) return b } +// Voices returns the book's voice profiles (frozen for the job). Read-only: callers project them onto a +// chapter, never mutate them. +func (b *Bank) Voices() []store.VoiceProfile { return b.voices } + +// Pairs returns the book's address-register journal entries (frozen for the job). +func (b *Bank) Pairs() []store.AddressPair { return b.pairs } + // ComputeVersion is the F1 content-hash: the frozen rows (ORDER BY-stable) plus the // normalization and matcher algorithm versions. Approved-only per D8/§8 when the post-check // hard gate is OFF (auto/draft injected-content changes are caught at the per-chunk @@ -330,11 +363,29 @@ func ComputeVersion(rows []store.GlossaryEntry, gateOn bool) string { // pre-split fold (no extra field), so the enriched hash / existing snapshots do not move; the // excludeMined=true variant adds a domain separator so base and enriched never collide. func ComputeVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool) string { + return ComputeVersionScopedIn(BankInput{Rows: rows}, gateOn, excludeMined) +} + +// ComputeVersionScopedIn is ComputeVersionScoped over the whole bank input — the form that also folds the +// pack-19 voice/address rows, CONDITIONALLY (BankInput.InjectVoice). +// +// The condition is the money contract of pack-19. While those rows only feed the $0 flagger they change +// no byte the model sees, and a book must not re-pay a wave for authoring a profile; the flagger's +// counters live in retrieval_state, which is recomputed every run and self-heals. The moment they are +// injected they DO change the wire, and then not folding them would reopen exactly the class D39.42 п.3 +// closed — a wire change the snapshot cannot see, so projectRebill projects $0 for a real re-payment. +// A book with no voice rows, or a run with the injection off, hashes BYTE-IDENTICALLY to before this +// existed: nothing is written at all. +func ComputeVersionScopedIn(in BankInput, gateOn, excludeMined bool) string { + rows := in.Rows h := sha256.New() h.Write([]byte("tm-memory-v2\x00")) h.Write([]byte(text.NormVersion() + "\x00" + matchVersion + "\x00")) h.Write([]byte("gate:" + strconv.FormatBool(gateOn) + "\x00")) hasUnverified := false + // inScope collects the characters this fold's ROW scope actually contains, so the voice/address fold + // below can apply the SAME scope before deciding anything (see the loop at the end). + inScope := map[[2]string]bool{} if excludeMined { h.Write([]byte("base-excl-mined\x00")) // domain separator: base ≠ enriched even over identical rows } @@ -355,6 +406,7 @@ func ComputeVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool) if r.Status != "approved" { hasUnverified = true } + inScope[[2]string{r.Src, r.Sense}] = true // A fixed, length-prefixed field layout so no content can forge a boundary. writeField(h, r.Status) writeField(h, r.Src) @@ -385,6 +437,40 @@ func ComputeVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool) if hasUnverified { h.Write([]byte("editor-unverified-section-v1\x00")) } + // The pack-19 record types, appended AFTER everything above so a book without them is byte-identical + // to the pre-pack-19 hash by construction, not by argument. + // + // The two loops copy the order the row loop above uses and for the same reason: the SCOPE exclusion is + // applied FIRST, and only a row that survived it may set the tag condition. A profile whose character + // is not in this scope (its term is Source:mined and this is the base/draft fold) describes somebody + // the wave never sees — folding it would move the draft version when a mined-only character's profile + // is edited, which is precisely the base/enriched separation the mined filter exists to keep. + hasVoice := false + for _, v := range in.Voices { + if !in.InjectVoice || !inScope[[2]string{v.Src, v.Sense}] { + continue + } + hasVoice = true + for _, f := range []string{v.Src, v.Sense, v.Register, v.SelfRef, v.AddressDefault, + v.LexiconMarkers, v.NGLexicon, v.Exemplars, v.Brightness, + strconv.Itoa(v.SinceCh), strconv.Itoa(v.UntilCh)} { + writeField(h, f) + } + } + for _, p := range in.Pairs { + if !in.InjectVoice || + !inScope[[2]string{p.SpeakerSrc, p.SpeakerSense}] || !inScope[[2]string{p.AddresseeSrc, p.AddresseeSense}] { + continue + } + hasVoice = true + for _, f := range []string{p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense, + p.Register, p.Form, p.Closeness, strconv.Itoa(p.SinceCh), strconv.Itoa(p.UntilCh)} { + writeField(h, f) + } + } + if hasVoice { + h.Write([]byte("voice-address-v1\x00")) + } return hex.EncodeToString(h.Sum(nil)) } @@ -493,6 +579,12 @@ func (b *Bank) Select(chunk string, chapter int, stickyPrev map[string]Injection continue } if spoilerBlocked(e, chapter) { + // RECORDED, not dropped (pack-19): a window-blocked carry is a spoiler reject like any other, + // and a silent drop makes n_spoiler_blocked under-count. Unreachable through the production + // driver — precomputeSticky resets the window at every chapter boundary, so a carry always + // arrives at the SAME chapter it fired in and cannot have become blocked — but Select is + // exported and the invariant that protects it lives in another package. + sel.Rejected = append(sel.Rejected, PickedEntry{entry: e, Sticky: true, Disp: Reject}) continue } hits[e.id] = PickedEntry{entry: e, Sticky: true, Disp: carryDisp} diff --git a/backend/internal/membank/mempostcheck.go b/backend/internal/membank/mempostcheck.go index 851ec75f..1b26e9ae 100644 --- a/backend/internal/membank/mempostcheck.go +++ b/backend/internal/membank/mempostcheck.go @@ -172,3 +172,53 @@ func containsWholeWord(hay, form []rune) bool { } return false } + +// SpoilerLeak is one rendering the spoiler window REJECTED for this chapter that appeared in the output +// anyway — the self-inflicted spoiler the windows exist to prevent (D21 п.3 / research/15 §2.2). +type SpoilerLeak struct { + Src string `json:"src"` + Dst string `json:"dst"` + // Since/Until are the window that excluded the row, so the operator sees at a glance whether the + // leak is a premature reveal (before since_ch) or a stale one (after until_ch). + SinceCh int `json:"since_ch"` + UntilCh int `json:"until_ch"` +} + +// SpoilerLeaks checks the chunk's REJECTED records against the model output: the row's key fired here, +// the window said the chapter must not know this rendering yet, and the rendering is in the output all +// the same. It is the reveal half of D21 п.3, built on the machinery that already exists — the selection +// records its rejects, and dstFormPresent already answers "is this rendering here" decl-aware — so it +// needs no schema and no second matcher. +// +// SCOPE, ratified (D39.55): "a leak on a FIRED key". Two classes are therefore OUT, and out on purpose +// rather than by oversight: +// +// - a STICKY carry rejected by the window (Sticky==true). Its key did NOT fire in this chunk, so the +// model was not looking at the entity here; counting it would silently widen the ratified scope on +// the back of the pack-19 fix that put those carries into Rejected at all; +// - a SOURCE-ANCHORED reveal — an identity twist whose source surface never occurs (the text says +// "the stranger", not the name). Nothing fires, so nothing is rejected, so there is nothing to +// check. Closing that class needs a target-side index of post-reveal renderings, which is a +// different mechanism and not this pack's. +// +// Pure and deterministic (rejected order, which Select fixes). Observability only — never a disposition. +func (b *Bank) SpoilerLeaks(rejected []PickedEntry, output string) []SpoilerLeak { + nout := []rune(text.NormalizeTargetForm(output)) + var out []SpoilerLeak + for _, p := range rejected { + if !p.valid() || p.Sticky { + continue + } + if strings.TrimSpace(p.entry.dst) == "" { + continue // nothing to leak + } + if !dstFormPresent(p.entry, nout) { + continue + } + out = append(out, SpoilerLeak{ + Src: p.entry.src, Dst: p.entry.dst, + SinceCh: p.entry.sinceCh, UntilCh: p.entry.untilCh, + }) + } + return out +} diff --git a/backend/internal/membank/memseed.go b/backend/internal/membank/memseed.go index de2e3674..52cfcb15 100644 --- a/backend/internal/membank/memseed.go +++ b/backend/internal/membank/memseed.go @@ -25,19 +25,21 @@ import ( // --- manual seed file (YAML) ---------------------------------------------------- -// LoadGlossarySeed parses a glossary seed YAML into store entries. A manual seed is +// LoadBankSeed parses a seed YAML into the bank's three record types. A manual seed is // curated, so an empty status defaults to "approved" (the author downgrades explicitly // with status: draft|auto). Validation is fail-loud: a term without src, or with an // unknown status, is a config error (a silently-dropped term is exactly the A-class -// hole this bank exists to close). -func LoadGlossarySeed(path string) ([]store.GlossaryEntry, error) { +// hole this bank exists to close). The voices:/addresses: sections follow the same +// contract — see loadVoiceSections. +func LoadBankSeed(path string) (BankSeed, error) { + var bs BankSeed raw, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("membank: read glossary seed %s: %w", path, err) + return bs, fmt.Errorf("membank: read glossary seed %s: %w", path, err) } var sf seed.File if err := yaml.Unmarshal(raw, &sf); err != nil { - return nil, fmt.Errorf("membank: parse glossary seed %s: %w", path, err) + return bs, fmt.Errorf("membank: parse glossary seed %s: %w", path, err) } var out []store.GlossaryEntry var problems []string @@ -87,7 +89,7 @@ func LoadGlossarySeed(path string) ([]store.GlossaryEntry, error) { if t.Decl != nil { b, mErr := json.Marshal(declInfo{Invariant: t.Decl.Invariant, Forms: t.Decl.Forms}) if mErr != nil { - return nil, mErr + return bs, mErr } decl = string(b) } @@ -149,9 +151,14 @@ func LoadGlossarySeed(path string) ([]store.GlossaryEntry, error) { } } if len(problems) > 0 { - return nil, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - ")) + return bs, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - ")) } - return out, nil + voices, pairs, verr := loadVoiceSections(path, &sf) + if verr != nil { + return BankSeed{}, verr + } + bs.Terms, bs.Voices, bs.Pairs = out, voices, pairs + return bs, nil } // ApprovedSharedKeyCollisions detects the alias-generalization of the D16.1 polysemy livelock diff --git a/backend/internal/membank/memvoice.go b/backend/internal/membank/memvoice.go new file mode 100644 index 00000000..5fb70b4e --- /dev/null +++ b/backend/internal/membank/memvoice.go @@ -0,0 +1,379 @@ +package membank + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "textmachine/backend/internal/seed" + "textmachine/backend/internal/store" + "textmachine/backend/internal/text" +) + +// memvoice.go: the two D21 bank record types pack-19 added — the per-character VOICE profile and the +// ordered ADDRESS pair — from seed file to store rows, plus the projection the flagger consumes. +// +// They are bank content and follow the bank's rules: curated only (no miner emits them), validated +// fail-loud at load (a silently inert profile is the class this bank exists to close), windowed on the +// SAME chapter axis as the spoiler window, and folded into the same memory_version. +// +// What they are NOT: matchable terms. A profile is keyed BY a character, it is not a surface to find in +// the text — so they never enter the Aho-Corasick automaton or the glossary injection. That separation +// is carried by the Go type system (BankInput.Rows is the only field Materialize matches over), not by a +// runtime filter that a later edit could forget. + +// registerInformal / registerFormal are the ABSTRACT T/V vocabulary. They are typological category +// names, not target words: which surfaces realise them is target data, so a target with no T/V +// distinction ships no surfaces and the whole mechanism stays inert. +const ( + registerInformal = "informal" + registerFormal = "formal" +) + +func validRegister(s string) bool { return s == registerInformal || s == registerFormal } + +// BankSeed is one parsed seed file: the terms plus the two pack-19 record types. +type BankSeed struct { + Terms []store.GlossaryEntry + Voices []store.VoiceProfile + Pairs []store.AddressPair +} + +// LoadGlossarySeed parses the TERMS of a seed file — the pre-pack-19 entry point, kept for every caller +// that has no use for the other two record types. +func LoadGlossarySeed(path string) ([]store.GlossaryEntry, error) { + s, err := LoadBankSeed(path) + if err != nil { + return nil, err + } + return s.Terms, nil +} + +// loadVoiceSections validates and materializes the voices:/addresses: sections of a parsed seed file. +// Fail-loud on anything that would be silently inert or would crash the UNIQUE constraint mid-run, which +// is the same contract the terms half keeps. +func loadVoiceSections(path string, sf *seed.File) ([]store.VoiceProfile, []store.AddressPair, error) { + var problems []string + var voices []store.VoiceProfile + voiceKeys := map[[4]string]bool{} + for i, v := range sf.Voices { + src, sense := strings.TrimSpace(v.Src), strings.TrimSpace(v.Sense) + if src == "" { + problems = append(problems, fmt.Sprintf("voice %d: src is required (it names the glossary term this profile belongs to)", i)) + continue + } + if d := strings.TrimSpace(v.AddressDefault); d != "" && !validRegister(d) { + problems = append(problems, fmt.Sprintf("voice %q: address_default must be %s|%s, got %q", src, registerInformal, registerFormal, d)) + continue + } + k := [4]string{src, sense, fmt.Sprint(v.SinceCh), fmt.Sprint(v.UntilCh)} + if voiceKeys[k] { + problems = append(problems, fmt.Sprintf("voice %q: duplicate (src, sense=%q, since_ch=%d, until_ch=%d) — a character has one profile per window", src, sense, v.SinceCh, v.UntilCh)) + continue + } + voiceKeys[k] = true + markers, err := cleanList(v.LexiconMarkers, "voice "+src+" lexicon_markers") + if err != nil { + problems = append(problems, err.Error()) + continue + } + ng, err := cleanList(v.NGLexicon, "voice "+src+" ng_lexicon") + if err != nil { + problems = append(problems, err.Error()) + continue + } + ex, err := cleanList(v.Exemplars, "voice "+src+" exemplars") + if err != nil { + problems = append(problems, err.Error()) + continue + } + // A profile that states NOTHING is an authoring slip: it costs a bank row and a hash, and directs + // nothing. Refused for the same reason an empty langpack table is (never silently empty). + if strings.TrimSpace(v.Register) == "" && strings.TrimSpace(v.SelfRef) == "" && + strings.TrimSpace(v.AddressDefault) == "" && len(markers) == 0 && len(ng) == 0 && len(ex) == 0 { + problems = append(problems, fmt.Sprintf("voice %q: the profile states nothing (needs at least one of register|self_ref|address_default|lexicon_markers|ng_lexicon|exemplars)", src)) + continue + } + voices = append(voices, store.VoiceProfile{ + Src: src, Sense: sense, + Register: strings.TrimSpace(v.Register), SelfRef: strings.TrimSpace(v.SelfRef), + AddressDefault: strings.TrimSpace(v.AddressDefault), + LexiconMarkers: encodeList(markers), NGLexicon: encodeList(ng), Exemplars: encodeList(ex), + Brightness: strings.TrimSpace(v.Brightness), SinceCh: v.SinceCh, UntilCh: v.UntilCh, + }) + } + // Two profiles of one character valid at the same chapter: the projection would have to pick, and it + // has no basis to. Same rule, same reason, as two renderings of one term in one chapter. + for i := range voices { + for j := 0; j < i; j++ { + a, b := voices[i], voices[j] + if a.Src == b.Src && a.Sense == b.Sense && windowsOverlap(a.SinceCh, a.UntilCh, b.SinceCh, b.UntilCh) { + problems = append(problems, fmt.Sprintf("voice %q: windows [%d,%d] and [%d,%d] overlap — a character has ONE profile per chapter", + a.Src, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh)) + } + } + } + + var pairs []store.AddressPair + pairKeys := map[[6]string]bool{} + for i, p := range sf.Addresses { + sp, spSense := strings.TrimSpace(p.Speaker), strings.TrimSpace(p.SpeakerSense) + ad, adSense := strings.TrimSpace(p.Addressee), strings.TrimSpace(p.AddresseeSense) + if sp == "" || ad == "" { + problems = append(problems, fmt.Sprintf("address %d: speaker and addressee are both required", i)) + continue + } + if sp == ad && spSense == adSense { + problems = append(problems, fmt.Sprintf("address %q: speaker and addressee are the same character — a register is a fact about a PAIR", sp)) + continue + } + if !validRegister(strings.TrimSpace(p.Register)) { + problems = append(problems, fmt.Sprintf("address %q→%q: register must be %s|%s, got %q (the value is a typological category, never a target word)", + sp, ad, registerInformal, registerFormal, p.Register)) + continue + } + k := [6]string{sp, spSense, ad, adSense, fmt.Sprint(p.SinceCh), fmt.Sprint(p.UntilCh)} + if pairKeys[k] { + problems = append(problems, fmt.Sprintf("address %q→%q: duplicate (speaker, addressee, since_ch=%d, until_ch=%d)", sp, ad, p.SinceCh, p.UntilCh)) + continue + } + pairKeys[k] = true + pairs = append(pairs, store.AddressPair{ + SpeakerSrc: sp, SpeakerSense: spSense, AddresseeSrc: ad, AddresseeSense: adSense, + Register: strings.TrimSpace(p.Register), Form: strings.TrimSpace(p.Form), + Closeness: strings.TrimSpace(p.Closeness), SinceCh: p.SinceCh, UntilCh: p.UntilCh, + }) + } + // Contradictory registers for one ordered pair at one chapter. A ты↔вы switch is legitimate — as a + // SECOND row with a non-overlapping window, which is what makes the table a journal. + for i := range pairs { + for j := 0; j < i; j++ { + a, b := pairs[i], pairs[j] + if a.SpeakerSrc != b.SpeakerSrc || a.SpeakerSense != b.SpeakerSense || + a.AddresseeSrc != b.AddresseeSrc || a.AddresseeSense != b.AddresseeSense { + continue + } + if a.Register != b.Register && windowsOverlap(a.SinceCh, a.UntilCh, b.SinceCh, b.UntilCh) { + problems = append(problems, fmt.Sprintf("address %q→%q: registers %q [%d,%d] and %q [%d,%d] overlap — one pair has ONE register per chapter (a switch is a new window, not an overlap)", + a.SpeakerSrc, a.AddresseeSrc, b.Register, b.SinceCh, b.UntilCh, a.Register, a.SinceCh, a.UntilCh)) + } + } + } + if len(problems) > 0 { + return nil, nil, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - ")) + } + return voices, pairs, nil +} + +// cleanList trims a seed string list and refuses an empty member: a blank exemplar or marker is an +// authoring slip that would inject an empty line and count toward a budget. +func cleanList(in []string, what string) ([]string, error) { + var out []string + for _, s := range in { + t := strings.TrimSpace(s) + if t == "" { + return nil, fmt.Errorf("%s: an entry is empty", what) + } + out = append(out, t) + } + return out, nil +} + +// encodeList renders a string list as the JSON stored in a column. Empty → "" (not "null"/"[]"), so an +// unstated field costs no bytes in the row and none in the hash. +func encodeList(in []string) string { + if len(in) == 0 { + return "" + } + b, err := json.Marshal(in) + if err != nil { // a []string cannot fail to marshal + return "" + } + return string(b) +} + +// decodeList reads a stored JSON list back. A malformed blob yields nil rather than an error: the column +// is written by encodeList alone, so a bad value is impossible without direct DB surgery, and the +// flagger's honest answer to "no data" is to check nothing. +func decodeList(s string) []string { + if strings.TrimSpace(s) == "" { + return nil + } + var out []string + if json.Unmarshal([]byte(s), &out) != nil { + return nil + } + return out +} + +// UnknownVoiceCharacters reports voice/address rows whose (src, sense) names no term in the bank. Such a +// row is silently inert — the character has no target surfaces, so nothing can ever attribute a reply to +// it — which is exactly the A-class hole the bank exists to close, so the caller fails the run loud. +// Checked over the FULL entry set (seed + ruby + mined + auto), because a character may legitimately be +// signed in a delta rather than the base seed. Deterministic (input order). +func UnknownVoiceCharacters(entries []store.GlossaryEntry, voices []store.VoiceProfile, pairs []store.AddressPair) []string { + known := map[[2]string]bool{} + for _, e := range entries { + known[[2]string{e.Src, e.Sense}] = true + } + var out []string + seen := map[string]bool{} + add := func(msg string) { + if !seen[msg] { + seen[msg] = true + out = append(out, msg) + } + } + for _, v := range voices { + if !known[[2]string{v.Src, v.Sense}] { + add(fmt.Sprintf("voice profile %q (sense %q) names no term in the bank", v.Src, v.Sense)) + } + } + for _, p := range pairs { + if !known[[2]string{p.SpeakerSrc, p.SpeakerSense}] { + add(fmt.Sprintf("address speaker %q (sense %q) names no term in the bank", p.SpeakerSrc, p.SpeakerSense)) + } + if !known[[2]string{p.AddresseeSrc, p.AddresseeSense}] { + add(fmt.Sprintf("address addressee %q (sense %q) names no term in the bank", p.AddresseeSrc, p.AddresseeSense)) + } + } + return out +} + +// VoiceWindowGaps reports a character whose profiles leave a chapter UNCOVERED between two windows +// (…until_ch=10 then since_ch=20: chapters 11–19 have no voice at all). Not an error — a deliberate gap +// is legitimate — but it is invisible in a seed file and is far more often a typo, so the caller logs it. +// The optional lint D39.55 asked for, applied where the data actually is. +func VoiceWindowGaps(voices []store.VoiceProfile) []string { + byChar := map[[2]string][]store.VoiceProfile{} + var order [][2]string + for _, v := range voices { + k := [2]string{v.Src, v.Sense} + if _, ok := byChar[k]; !ok { + order = append(order, k) + } + byChar[k] = append(byChar[k], v) + } + var out []string + for _, k := range order { + win := byChar[k] + if len(win) < 2 { + continue + } + sort.Slice(win, func(i, j int) bool { return win[i].SinceCh < win[j].SinceCh }) + for i := 1; i < len(win); i++ { + prevUntil := win[i-1].UntilCh + if prevUntil == 0 { // open-ended: nothing after it can be a gap + continue + } + if win[i].SinceCh > prevUntil+1 { + out = append(out, fmt.Sprintf("voice %q: chapters %d–%d have no profile (window ends at %d, next starts at %d)", + k[0], prevUntil+1, win[i].SinceCh-1, prevUntil, win[i].SinceCh)) + } + } + } + return out +} + +// CharacterView is one character as an OUTPUT-side checker needs to see it: an opaque identity, the +// normalized target surfaces that name it, and the two profile fields the deterministic checks read. It +// is deliberately free of store and checker types — membank owns the projection, the driver owns the +// adaptation — so neither package has to know the other's vocabulary. +type CharacterView struct { + Key string // src\x1fsense — compared, never rendered into a message + Name string // the approved rendering, for a human-readable detail line + Forms []string + SelfRef string + NG []string +} + +// AddressView is one ordered register record projected onto a chapter. +type AddressView struct { + Speaker, Addressee, Register string // Speaker/Addressee are CharacterView.Key +} + +// VoiceProjection resolves the bank's voice/address content ONTO one chapter: the profiles and register +// records whose window contains it, joined to the terms that give each character its target surfaces. +// +// It is a PROJECTION, never stored state — which is what keeps address_pairs a journal with one source +// of truth (D21 п.2) instead of a second store that could disagree with it. A character whose term is +// spoiler-blocked at this chapter is dropped entirely: it cannot be matched in the output, so a profile +// for it could only produce noise. Pure and deterministic (bank order). +func (b *Bank) VoiceProjection(chapter int) ([]CharacterView, []AddressView) { + if b == nil { + return nil, nil + } + // The character's surfaces come from the term row valid at this chapter (the same window rule the + // injection uses), so a pre/post-reveal handoff projects the rendering the chapter is allowed to see. + forms := func(src, sense string) (view CharacterView, ok bool) { + for i := range b.entries { + e := &b.entries[i] + if e.src != src || e.sense != sense || spoilerBlocked(e, chapter) { + continue + } + base := text.NormalizeTargetForm(e.dst) + if base == "" { + continue + } + view.Key = src + "\x1f" + sense + view.Name = e.dst + view.Forms = append([]string{base}, e.declForms...) + return view, true + } + return view, false + } + byKey := map[string]*CharacterView{} + var chars []CharacterView + order := []string{} + need := func(src, sense string) *CharacterView { + key := src + "\x1f" + sense + if v, ok := byKey[key]; ok { + return v + } + v, ok := forms(src, sense) + if !ok { + byKey[key] = nil + return nil + } + byKey[key] = &v + order = append(order, key) + return &v + } + for _, vp := range b.voices { + if vp.SinceCh > 0 && chapter < vp.SinceCh { + continue + } + if vp.UntilCh > 0 && chapter > vp.UntilCh { + continue + } + v := need(vp.Src, vp.Sense) + if v == nil { + continue + } + v.SelfRef = text.NormalizeTargetForm(vp.SelfRef) + for _, w := range decodeList(vp.NGLexicon) { + if n := text.NormalizeTargetForm(w); n != "" { + v.NG = append(v.NG, n) + } + } + } + var addrs []AddressView + for _, p := range b.pairs { + if p.SinceCh > 0 && chapter < p.SinceCh { + continue + } + if p.UntilCh > 0 && chapter > p.UntilCh { + continue + } + sp, ad := need(p.SpeakerSrc, p.SpeakerSense), need(p.AddresseeSrc, p.AddresseeSense) + if sp == nil || ad == nil { + continue + } + addrs = append(addrs, AddressView{Speaker: sp.Key, Addressee: ad.Key, Register: p.Register}) + } + for _, k := range order { + chars = append(chars, *byKey[k]) + } + return chars, addrs +} diff --git a/backend/internal/membank/memvoice_test.go b/backend/internal/membank/memvoice_test.go new file mode 100644 index 00000000..6ba31f68 --- /dev/null +++ b/backend/internal/membank/memvoice_test.go @@ -0,0 +1,269 @@ +package membank + +import ( + "strings" + "testing" + + "textmachine/backend/internal/lang" + "textmachine/backend/internal/store" +) + +// memvoice_test.go pins the pack-19 record types on the axes that cost money if they are wrong: the +// CONDITIONAL fold (a book must not re-pay for authoring a profile, and must re-pay the moment one +// reaches the wire), the scope order the fold shares with the mined filter, and the structural promise +// that a voice profile is never a matchable term. + +func term(src, dst string) store.GlossaryEntry { + return store.GlossaryEntry{Src: src, Dst: dst, Status: "approved", Source: "seed"} +} + +func profile(src string) store.VoiceProfile { + return store.VoiceProfile{Src: src, Register: "ледяная краткость", SelfRef: "ваша служанка"} +} + +// The load-bearing money property: while the injection is off, voice/address content is invisible to the +// hash, so a book that gains a whole cast of profiles keeps its snapshot and re-pays nothing. +func TestVoiceRowsDoNotMoveTheBankVersionWhileUninjected(t *testing.T) { + rows := []store.GlossaryEntry{term("方源", "Фан Юань")} + bare := ComputeVersionScopedIn(BankInput{Rows: rows}, false, false) + withVoice := ComputeVersionScopedIn(BankInput{ + Rows: rows, + Voices: []store.VoiceProfile{profile("方源")}, + Pairs: []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "方源", Register: "formal"}}, + }, false, false) + if bare != withVoice { + t.Fatalf("an UNINJECTED profile moved the bank version — every book with one would re-pay a wave\n bare %s\n with %s", bare, withVoice) + } + // And the pre-pack-19 entry point must reproduce the same bytes, so no existing book moves at all. + if legacy := ComputeVersion(rows, false); legacy != bare { + t.Fatalf("the glossary-only fold changed shape: %s vs %s", legacy, bare) + } +} + +// The other half of the same contract: once injected, the rows are wire bytes, so any edit MUST be a +// loud --resnapshot. Not folding them then would reopen the class D39.42 п.3 closed. +func TestInjectedVoiceRowsMoveTheBankVersion(t *testing.T) { + rows := []store.GlossaryEntry{term("方源", "Фан Юань")} + off := ComputeVersionScopedIn(BankInput{Rows: rows, Voices: []store.VoiceProfile{profile("方源")}}, false, false) + on := ComputeVersionScopedIn(BankInput{Rows: rows, Voices: []store.VoiceProfile{profile("方源")}, InjectVoice: true}, false, false) + if off == on { + t.Fatal("turning the injection on left the bank version unchanged — the flip would be a silent re-payment") + } + edited := profile("方源") + edited.SelfRef = "я" + after := ComputeVersionScopedIn(BankInput{Rows: rows, Voices: []store.VoiceProfile{edited}, InjectVoice: true}, false, false) + if after == on { + t.Fatal("editing an INJECTED profile left the bank version unchanged — a wire change the snapshot cannot see") + } + // A pair edit is the same class. + p1 := BankInput{Rows: rows, Pairs: []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "formal"}}, InjectVoice: true} + p2 := p1 + p2.Pairs = []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "informal"}} + rows2 := []store.GlossaryEntry{term("方源", "Фан Юань"), term("白凝冰", "Бай Нинбин")} + p1.Rows, p2.Rows = rows2, rows2 + if ComputeVersionScopedIn(p1, false, false) == ComputeVersionScopedIn(p2, false, false) { + t.Fatal("flipping a pair's register left the bank version unchanged") + } +} + +// The scope order (D39.55 «ровно так» (г)): the exclusion runs BEFORE the tag condition, so a profile +// whose character is mined-only folds into the ENRICHED version and NOT into the base — otherwise +// editing a mined-only character's profile would move the DRAFT wave, which never sees that character. +func TestVoiceFoldRespectsTheMinedScopeBeforeCountingIt(t *testing.T) { + mined := term("蛊", "Гу") + mined.Source = "mined" + rows := []store.GlossaryEntry{term("方源", "Фан Юань"), mined} + withMinedProfile := BankInput{Rows: rows, Voices: []store.VoiceProfile{profile("蛊")}, InjectVoice: true} + noProfile := BankInput{Rows: rows, InjectVoice: true} + + if got, want := ComputeVersionScopedIn(withMinedProfile, false, true), ComputeVersionScopedIn(noProfile, false, true); got != want { + t.Fatalf("a mined-only character's profile folded into the BASE version — the draft wave would re-pay for a character it never sees\n got %s\n want %s", got, want) + } + if ComputeVersionScopedIn(withMinedProfile, false, false) == ComputeVersionScopedIn(noProfile, false, false) { + t.Fatal("a mined character's profile did NOT fold into the enriched version — the edit wave would not re-pay for a wire change") + } +} + +// (в) A voice profile is bank content, never a matchable surface: nothing it carries may reach the +// automaton or either injection block. The type system makes it structural; this pins the observable. +func TestVoiceProfileNeverReachesTheMatcherOrTheInjection(t *testing.T) { + const canary = "СЕКРЕТПРОФИЛЬ" + vp := store.VoiceProfile{Src: "方源", Register: canary, SelfRef: canary, Exemplars: `["` + canary + `"]`} + b := MaterializeBank(BankInput{ + Rows: []store.GlossaryEntry{term("方源", "Фан Юань")}, + Voices: []store.VoiceProfile{vp}, + Pairs: []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "formal", Form: canary}}, + }, false) + sel := b.Select("方源 стоял тут.", 1, nil, 0) + if len(sel.Injected) != 1 { + t.Fatalf("the TERM must still fire normally, got %d injected", len(sel.Injected)) + } + tx := lang.InjectionTextsFor("ru") + for name, block := range map[string]string{ + "draft": RenderGlossaryBlock(sel.Injected, tx), + "editor": RenderEditorConstraintBlock(sel.Injected, tx), + } { + if strings.Contains(block, canary) { + t.Fatalf("%s injection leaked voice-profile content: %q", name, block) + } + } + // And the profile's own src must not have become a second, separate match. + if n := len(b.Select(canary, 1, nil, 0).Injected); n != 0 { + t.Fatalf("profile content became matchable: %d hits on the canary", n) + } +} + +// (ж) A sticky carry the window rejects is RECORDED, not dropped: n_spoiler_blocked must not under-count. +func TestStickyCarryBlockedByWindowIsRecordedAsRejected(t *testing.T) { + late := term("真相", "правда") + late.SinceCh = 5 + b := Materialize([]store.GlossaryEntry{late}, false) + // The carry arrives at chapter 1, where the window blocks it. Reachable only through the exported + // API — precomputeSticky resets per chapter — which is exactly why the guard belongs here. + sel := b.Select("ничего не совпадает", 1, map[string]InjectionDisposition{ + b.entries[0].id: Confirmed, + }, 0) + if len(sel.Injected) != 0 { + t.Fatalf("a blocked carry must not inject, got %d", len(sel.Injected)) + } + if len(sel.Rejected) != 1 || !sel.Rejected[0].Sticky { + t.Fatalf("a window-blocked sticky carry must be recorded as a sticky reject, got %+v", sel.Rejected) + } +} + +func TestSpoilerLeakFlagger(t *testing.T) { + late := store.GlossaryEntry{Src: "真相", Dst: "её брат", Status: "approved", Source: "seed", SinceCh: 5, + Decl: `{"forms":["её брата"]}`} + b := Materialize([]store.GlossaryEntry{late}, false) + sel := b.Select("真相 出现了。", 1, nil, 0) // the key FIRES, the window rejects it + if len(sel.Rejected) != 1 { + t.Fatalf("setup: want one rejected row, got %d", len(sel.Rejected)) + } + if n := len(b.SpoilerLeaks(sel.Rejected, "Он молчал о прошлом.")); n != 0 { + t.Fatalf("no leak expected, got %d", n) + } + leaks := b.SpoilerLeaks(sel.Rejected, "Это был её брат.") + if len(leaks) != 1 || leaks[0].Src != "真相" || leaks[0].SinceCh != 5 { + t.Fatalf("the rejected rendering reached the output and must be flagged, got %+v", leaks) + } + if n := len(b.SpoilerLeaks(sel.Rejected, "Он ждал её брата.")); n != 1 { + t.Fatalf("a DECLINED form of the rejected rendering is the same leak, got %d", n) + } + // The ratified scope is "a leak on a FIRED key": a sticky reject did not fire here, so it is out — + // otherwise the (ж) fix above would silently widen what this flagger claims to measure. + sticky := []PickedEntry{{entry: sel.Rejected[0].entry, Sticky: true, Disp: Reject}} + if n := len(b.SpoilerLeaks(sticky, "Это был её брат.")); n != 0 { + t.Fatalf("a STICKY reject is outside the ratified scope, got %d", n) + } +} + +func TestVoiceProjectionRespectsWindows(t *testing.T) { + rows := []store.GlossaryEntry{term("方源", "Фан Юань"), term("白凝冰", "Бай Нинбин")} + early := profile("方源") + early.UntilCh = 10 + late := profile("方源") + late.SinceCh = 11 + late.SelfRef = "я" + b := MaterializeBank(BankInput{ + Rows: rows, + Voices: []store.VoiceProfile{early, late}, + Pairs: []store.AddressPair{ + {SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "formal", UntilCh: 10}, + {SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "informal", SinceCh: 11}, + }, + }, false) + chars, addrs := b.VoiceProjection(3) + if len(chars) == 0 || chars[0].SelfRef != "ваша служанка" { + t.Fatalf("chapter 3 must project the EARLY profile, got %+v", chars) + } + if len(addrs) != 1 || addrs[0].Register != "formal" { + t.Fatalf("chapter 3 must project the formal register, got %+v", addrs) + } + _, addrs = b.VoiceProjection(20) + if len(addrs) != 1 || addrs[0].Register != "informal" { + t.Fatalf("chapter 20 must project the switched register, got %+v", addrs) + } + // A character whose term is spoiler-blocked cannot be matched in the output, so it is not projected. + hidden := term("影", "Тень") + hidden.SinceCh = 30 + b2 := MaterializeBank(BankInput{Rows: []store.GlossaryEntry{hidden}, Voices: []store.VoiceProfile{profile("影")}}, false) + if chars, _ := b2.VoiceProjection(1); len(chars) != 0 { + t.Fatalf("a spoiler-blocked character must not project, got %+v", chars) + } +} + +func TestBankSeedLoadsVoiceSections(t *testing.T) { + bs, err := LoadBankSeed(writeSeed(t, ` +terms: + - src: 方源 + dst: Фан Юань + - src: 白凝冰 + dst: Бай Нинбин +voices: + - src: 白凝冰 + register: ледяная краткость + self_ref: ваша служанка + ng_lexicon: [ладно, окей] + exemplars: ["Слушаюсь."] +addresses: + - speaker: 白凝冰 + addressee: 方源 + register: formal + since_ch: 1 + until_ch: 40 +`)) + if err != nil { + t.Fatal(err) + } + if len(bs.Voices) != 1 || bs.Voices[0].SelfRef != "ваша служанка" || bs.Voices[0].NGLexicon != `["ладно","окей"]` { + t.Fatalf("voice section not materialized: %+v", bs.Voices) + } + if len(bs.Pairs) != 1 || bs.Pairs[0].Register != "formal" || bs.Pairs[0].UntilCh != 40 { + t.Fatalf("address section not materialized: %+v", bs.Pairs) + } +} + +func TestBankSeedRefusesMalformedVoiceSections(t *testing.T) { + cases := map[string]string{ + "empty profile": "voices:\n - src: 方源\n", + "bad register": "addresses:\n - speaker: A\n addressee: B\n register: ты\n", + "self-addressed pair": "addresses:\n - speaker: A\n addressee: A\n register: formal\n", + "overlapping windows": "voices:\n - src: A\n self_ref: x\n until_ch: 10\n - src: A\n self_ref: y\n since_ch: 5\n", + "contradictory pair": "addresses:\n - speaker: A\n addressee: B\n register: formal\n until_ch: 10\n - speaker: A\n addressee: B\n register: informal\n since_ch: 5\n", + "empty exemplar": "voices:\n - src: A\n exemplars: [\"\"]\n", + "missing addressee": "addresses:\n - speaker: A\n register: formal\n", + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + if _, err := LoadBankSeed(writeSeed(t, body)); err == nil { + t.Fatal("a malformed voice/address section must fail loud at load, before any billing") + } + }) + } + // A ты↔вы switch — the same pair, non-overlapping windows — is the legitimate journal shape. + if _, err := LoadBankSeed(writeSeed(t, "addresses:\n - speaker: A\n addressee: B\n register: formal\n until_ch: 10\n - speaker: A\n addressee: B\n register: informal\n since_ch: 11\n")); err != nil { + t.Fatalf("a register switch across non-overlapping windows is legitimate: %v", err) + } +} + +func TestUnknownVoiceCharactersAndWindowGaps(t *testing.T) { + rows := []store.GlossaryEntry{term("方源", "Фан Юань")} + bad := UnknownVoiceCharacters(rows, + []store.VoiceProfile{{Src: "нет-такого"}}, + []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "тоже-нет"}}) + if len(bad) != 2 { + t.Fatalf("both the unknown profile and the unknown addressee must be named, got %v", bad) + } + if n := len(UnknownVoiceCharacters(rows, []store.VoiceProfile{{Src: "方源"}}, nil)); n != 0 { + t.Fatalf("a known character must not be reported, got %d", n) + } + gaps := VoiceWindowGaps([]store.VoiceProfile{ + {Src: "A", UntilCh: 10}, {Src: "A", SinceCh: 20}, + }) + if len(gaps) != 1 || !strings.Contains(gaps[0], "11–19") { + t.Fatalf("the uncovered range must be named, got %v", gaps) + } + if n := len(VoiceWindowGaps([]store.VoiceProfile{{Src: "A", UntilCh: 10}, {Src: "A", SinceCh: 11}})); n != 0 { + t.Fatalf("adjacent windows are not a gap, got %d", n) + } +} diff --git a/backend/internal/pipeline/chunkrun.go b/backend/internal/pipeline/chunkrun.go index 1d273775..669118ab 100644 --- a/backend/internal/pipeline/chunkrun.go +++ b/backend/internal/pipeline/chunkrun.go @@ -103,7 +103,7 @@ var roleInjectionRenderers = map[string]injectionRenderer{ // selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED // records (what the model saw); spoiler/eviction are the dropped-and-logged totals; // post-check misses are recorded only when the translator actually produced text. -func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel membank.Selection, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult, bank bankFlags, bankProps string) error { +func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel membank.Selection, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult, bank bankFlags, bankProps string, voice checks.VoiceResult, leaks []membank.SpoilerLeak) error { rs := store.RetrievalState{ BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, SnapshotID: snapID, NStyleFlags: cheap.Total(), @@ -117,6 +117,7 @@ func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel memban rs.StyleDetail = string(b) } } + setVoiceState(&rs, voice, leaks) for _, p := range sel.Injected { if p.Sticky { rs.NSticky++ diff --git a/backend/internal/pipeline/quality.go b/backend/internal/pipeline/quality.go index db48bd8d..e9ac7221 100644 --- a/backend/internal/pipeline/quality.go +++ b/backend/internal/pipeline/quality.go @@ -113,6 +113,24 @@ type QualityReport struct { UnverifiedShown int `json:"unverified_shown,omitempty"` UnverifiedFollowed int `json:"unverified_followed,omitempty"` + // The pack-19 flaggers (D39.55). VoiceFlags is axes A-C — the T/V contradictions, flattened + // self-designations and forbidden lexemes the run found in ATTRIBUTED replies; VoiceReplies / + // VoiceAttributed are its denominators, without which the count cannot be read. VoicePairRegister is + // axis D, the registry check, reported apart from the count because its addressee comes from a + // heuristic rather than from attribution. SpoilerLeaks is the reveal half of D21 п.3: renderings the + // spoiler window rejected for their chapter that reached the shipped text anyway — the only one of + // the four that is a safety signal rather than a style measurement. None gates anything; all + // omitempty, so a book without voice content reports byte-identically to before. + VoiceFlags int `json:"voice_flags,omitempty"` + VoiceReplies int `json:"voice_replies,omitempty"` + VoiceAttributed int `json:"voice_attributed,omitempty"` + VoicePairRegister int `json:"voice_pair_register,omitempty"` + SpoilerLeaks int `json:"spoiler_leaks,omitempty"` + // VoiceCheckVersion names the rules that produced those counts. The voice gate is deliberately not + // snapshot-folded (config.VoiceGate), so the version travels with the numbers instead — the same + // mitigation the terminologist uses for the same trade-off. + VoiceCheckVersion string `json:"voice_check_version,omitempty"` + // EscalationHops / SpendByModel are the money-side content-label provenance (B6): how many fallback // CALLS the book actually paid for (the per-unit `escalated` boolean cannot count them) and how the // spend splits across model slugs — which is what answers "what did the label-routed endpoint cost" @@ -248,6 +266,16 @@ func (r *Runner) QualityReport() (*QualityReport, error) { rep.TrustGated += rs.NTrustGatedSuppress rep.UnverifiedShown += rs.NUnverifiedShown rep.UnverifiedFollowed += rs.NUnverifiedFollowed + rep.VoiceFlags += rs.NVoiceFlags + rep.SpoilerLeaks += rs.NSpoilerLeaks + if rs.VoiceDetail != "" { + var v checks.VoiceResult + if json.Unmarshal([]byte(rs.VoiceDetail), &v) == nil { + rep.VoiceReplies += v.Replies + rep.VoiceAttributed += v.Attributed + rep.VoicePairRegister += v.PairRegister + } + } if gateOn && rs.NPostcheckMiss > 0 { withheld[chunkKey{rs.Chapter, rs.ChunkIdx}] = true } @@ -405,6 +433,9 @@ func (r *Runner) QualityReport() (*QualityReport, error) { return nil, merr } rep.SpendByModel = byModel + if r.Pipeline.Gates.Voice.Enabled { + rep.VoiceCheckVersion = checks.VoiceCheckVersion + } if len(r.Book.ContentLabels) > 0 { rep.ContentLabels = r.Book.ContentLabels rep.Routing = r.contentRoutingRows() diff --git a/backend/internal/pipeline/render.go b/backend/internal/pipeline/render.go index 10b3c482..41fa75b5 100644 --- a/backend/internal/pipeline/render.go +++ b/backend/internal/pipeline/render.go @@ -85,7 +85,7 @@ type PromptTemplate struct { System string FewShot string User string - SHA256 string // hash of the raw file — part of the snapshot + SHA256 string // hash of the CANONICAL (comment-stripped) file — part of the snapshot } // commentOpen/commentClose delimit an editorial comment in a prompt file — a note to whoever diff --git a/backend/internal/pipeline/runner.go b/backend/internal/pipeline/runner.go index 94b5e75d..bd5d4798 100644 --- a/backend/internal/pipeline/runner.go +++ b/backend/internal/pipeline/runner.go @@ -268,7 +268,7 @@ func (r *Runner) loadLangPack() error { // embedded target data (target-general lists, keyed by target lang). Built here so it exists even for a // no-langpack book (a nil pack → inert pair checkers; the target lists still load). r.pack is set below. defer func() { - r.checkers = checks.CompileCheckers(checks.DCCheckerData(r.pack), lang.TargetChecksFor(r.Book.TargetLang)) + r.checkers = checks.CompileCheckersFor(r.pack, lang.TargetChecksFor(r.Book.TargetLang)) }() if r.Book.LangpackRoot == "" { return nil // no langpack declared → nil pack, miner inert diff --git a/backend/internal/pipeline/seeding.go b/backend/internal/pipeline/seeding.go index 11672af8..bba65610 100644 --- a/backend/internal/pipeline/seeding.go +++ b/backend/internal/pipeline/seeding.go @@ -22,12 +22,15 @@ import ( // approved dst-collisions are logged (not fatal — some collisions are legitimate). func (r *Runner) seedGlossary(ctx context.Context) error { var entries []store.GlossaryEntry + var voices []store.VoiceProfile + var pairs []store.AddressPair if r.Book.GlossarySeed != "" { - seed, err := membank.LoadGlossarySeed(r.Book.GlossarySeed) + bank, err := membank.LoadBankSeed(r.Book.GlossarySeed) if err != nil { return err } - entries = append(entries, seed...) + entries = append(entries, bank.Terms...) + voices, pairs = bank.Voices, bank.Pairs } manualSrcs := map[string]bool{} for _, e := range entries { @@ -96,14 +99,40 @@ func (r *Runner) seedGlossary(ctx context.Context) error { if cols := membank.ApprovedSharedKeyCollisions(entries); len(cols) > 0 { return fmt.Errorf("pipeline: glossary shared-key collisions (A2 / D16.1 livelock class):\n - %s", strings.Join(cols, "\n - ")) } - if err := r.Store.ReplaceGlossary(r.Book.BookID, entries); err != nil { - return fmt.Errorf("pipeline: replace glossary for %s (%d entries): %w", r.Book.BookID, len(entries), err) + // A voice/address row naming a character the bank does not have is SILENTLY inert — nothing can ever + // attribute a reply to it — which is the A-class hole this bank exists to close, so it stops the run. + // Checked over the FULL entry set (seed + ruby + mined + auto), because a character may legitimately be + // signed in a delta rather than in the base seed. + if unknown := membank.UnknownVoiceCharacters(entries, voices, pairs); len(unknown) > 0 { + return fmt.Errorf("pipeline: voice/address rows name characters absent from the bank (a profile for a term that does not exist can never fire):\n - %s", strings.Join(unknown, "\n - ")) + } + if err := r.Store.ReplaceBank(r.Book.BookID, entries, voices, pairs); err != nil { + return fmt.Errorf("pipeline: replace bank for %s (%d terms, %d voices, %d pairs): %w", r.Book.BookID, len(entries), len(voices), len(pairs), err) } rows, err := r.Store.GlossaryForBook(r.Book.BookID) if err != nil { return fmt.Errorf("pipeline: read glossary for %s: %w", r.Book.BookID, err) } - r.memory = membank.Materialize(rows, r.Pipeline.Gates.Glossary.PostcheckGate) + storedVoices, err := r.Store.VoiceProfilesForBook(r.Book.BookID) + if err != nil { + return fmt.Errorf("pipeline: read voice profiles for %s: %w", r.Book.BookID, err) + } + storedPairs, err := r.Store.AddressPairsForBook(r.Book.BookID) + if err != nil { + return fmt.Errorf("pipeline: read address pairs for %s: %w", r.Book.BookID, err) + } + // A chapter range no profile covers is legitimate but is far more often a typo, and it is invisible in + // a seed file — so it is logged, never fatal (the optional lint of D39.55). + if gaps := membank.VoiceWindowGaps(storedVoices); len(gaps) > 0 { + r.Log.WarnContext(ctx, "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)", + "book", r.Book.BookID, "gaps", strings.Join(gaps, "; ")) + } + // InjectVoice is FALSE and has no config knob: pack-19 builds the schema and the flagger, and D21 п.2 + // holds the injection conditional until the polygon experiment. It is the fold's condition, so while + // it is false a book with voice rows hashes exactly as it did without them and nobody re-pays for + // authoring a profile. Wiring the injection means setting it and accepting a full --resnapshot. + bankIn := membank.BankInput{Rows: rows, Voices: storedVoices, Pairs: storedPairs} + r.memory = membank.MaterializeBank(bankIn, r.Pipeline.Gates.Glossary.PostcheckGate) // The DRAFT wave selects over a BASE-scoped bank (Source:mined excluded) so its injection is // byte-identical across a bank-mining enrichment — matching the draft-wave snapshot (baseMemoryVersion), // which keeps «one re-payment» honest at the WIRE level, not only the version-hash level. Only the @@ -119,7 +148,9 @@ func (r *Runner) seedGlossary(ctx context.Context) error { baseRows = append(baseRows, row) } if hasMined { - r.baseMemory = membank.Materialize(baseRows, r.Pipeline.Gates.Glossary.PostcheckGate) + baseIn := bankIn + baseIn.Rows = baseRows + r.baseMemory = membank.MaterializeBank(baseIn, r.Pipeline.Gates.Glossary.PostcheckGate) } else { r.baseMemory = r.memory } @@ -137,7 +168,8 @@ func (r *Runner) seedGlossary(ctx context.Context) error { "book", r.Book.BookID, "conflicts", strings.Join(cols, "; ")) } r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID, - "entries", len(rows), "memory_version", r.memory.Version()[:12]) + "entries", len(rows), "voices", len(storedVoices), "address_pairs", len(storedPairs), + "memory_version", r.memory.Version()[:12]) return nil } diff --git a/backend/internal/pipeline/voicerun.go b/backend/internal/pipeline/voicerun.go new file mode 100644 index 00000000..e9560323 --- /dev/null +++ b/backend/internal/pipeline/voicerun.go @@ -0,0 +1,82 @@ +package pipeline + +import ( + "encoding/json" + + "textmachine/backend/internal/checks" + "textmachine/backend/internal/membank" + "textmachine/backend/internal/store" +) + +// voicerun.go: the driver half of the pack-19 flaggers — the adapter that turns the bank's chapter +// projection into the checker's plain-value registry, and the two $0 calls the final wave makes. +// +// Both are OBSERVABILITY: neither can flag a chunk, neither touches the wire, and both are recomputed +// from stored text on every run, so a resume re-derives them without re-billing anything. + +// voiceRegistry adapts membank's chapter projection to the checker vocabulary. Two packages, two value +// types, one trivial mapping — which is what keeps checks free of store types and membank free of +// checker types. Empty when the book has no voice/address content (the flagger is then inert). +func voiceRegistry(mem *membank.Bank, chapter int) checks.VoiceRegistry { + if mem == nil { + return checks.VoiceRegistry{} + } + chars, addrs := mem.VoiceProjection(chapter) + if len(chars) == 0 { + return checks.VoiceRegistry{} + } + reg := checks.VoiceRegistry{Chars: make([]checks.VoiceCharacter, 0, len(chars))} + for _, c := range chars { + reg.Chars = append(reg.Chars, checks.VoiceCharacter{ + Key: c.Key, Name: c.Name, Forms: c.Forms, SelfRef: c.SelfRef, NG: c.NG, + }) + } + for _, a := range addrs { + reg.Pairs = append(reg.Pairs, checks.VoiceAddressPair{ + Speaker: a.Speaker, Addressee: a.Addressee, Register: a.Register, + }) + } + return reg +} + +// runVoiceChecks runs the voice flagger over one unit's shipped text, when the gate is on. A book with +// the gate off, or with no voice content, gets a zero result and writes zeros — byte-identical to a run +// before this existed. +func (r *Runner) runVoiceChecks(mem *membank.Bank, chapter int, source, final string) checks.VoiceResult { + if !r.Pipeline.Gates.Voice.Enabled || final == "" { + return checks.VoiceResult{} + } + return checks.RunVoiceChecks(source, final, voiceRegistry(mem, chapter), r.checkers) +} + +// spoilerLeaks checks the chapter's REJECTED bank rows against the shipped text: a rendering the spoiler +// window excluded that reached the reader anyway. Unlike the voice flagger it is NOT gated — the spoiler +// window is a standing safety mechanism (C1), not an opt-in measurement, and a leak is the one thing the +// window exists to prevent. $0 and observability-only, like every sibling on the retrieval-state row. +func (r *Runner) spoilerLeaks(mem *membank.Bank, sel membank.Selection, final string) []membank.SpoilerLeak { + if mem == nil || final == "" || len(sel.Rejected) == 0 { + return nil + } + return mem.SpoilerLeaks(sel.Rejected, final) +} + +// setVoiceState writes the two pack-19 flagger results onto a retrieval-state row. Zero results write +// zeros and an empty detail, so a gate-off book's row is byte-identical to what it was before these +// columns existed. Shared by the draft-chunk and edit-unit paths so the two cannot drift. +func setVoiceState(rs *store.RetrievalState, voice checks.VoiceResult, leaks []membank.SpoilerLeak) { + rs.NVoiceFlags, rs.VoiceDetail = voice.Total(), "" + // The detail carries the WHOLE result, not only the flagged axes: the denominators (replies, + // attributed, source replies) are what make the count readable, and the axis-D indicator is reported + // beside them precisely because it is excluded from the count. + if voice.Total() > 0 || voice.PairRegister > 0 || voice.Replies > 0 { + if b, err := json.Marshal(voice); err == nil { + rs.VoiceDetail = string(b) + } + } + rs.NSpoilerLeaks, rs.SpoilerLeakDetail = len(leaks), "" + if len(leaks) > 0 { + if b, err := json.Marshal(leaks); err == nil { + rs.SpoilerLeakDetail = string(b) + } + } +} diff --git a/backend/internal/pipeline/voicerun_test.go b/backend/internal/pipeline/voicerun_test.go new file mode 100644 index 00000000..d296fb6a --- /dev/null +++ b/backend/internal/pipeline/voicerun_test.go @@ -0,0 +1,200 @@ +package pipeline + +import ( + "context" + "strings" + "testing" +) + +// voicerun_test.go: the pack-19 flaggers THROUGH THE REAL DRIVER — a seed with voice/address sections, a +// mock editor whose output carries the defect, and the assertion that the count reaches retrieval_state +// and the quality report WITHOUT flagging the unit. + +const voiceGateBlock = "\ngates:\n voice:\n enabled: true\n" + +// voiceSeed carries both new sections plus the terms they name. 白凝冰 owes 方源 the formal register and +// calls herself «ваша служанка» — the worked example of research/15. +const voiceSeed = ` +terms: + - src: 方源 + dst: Фан Юань + decl: { forms: [Фан Юаня, Фан Юаню] } + - src: 白凝冰 + dst: Бай Нинбин +voices: + - src: 白凝冰 + register: ледяная краткость + self_ref: ваша служанка + ng_lexicon: [ладно] +addresses: + - speaker: 白凝冰 + addressee: 方源 + register: formal +` + +const voiceSource = "白凝冰对方源说话。" + +func TestVoiceFlaggerReachesTheStoreAndTheReport(t *testing.T) { + srv := newJSONProvider(&reqRec{}, func(body string) (string, string) { + if isEditBody(body) { + // One reply, attributed, that flattens the self-designation AND says a forbidden word. + return "— Ладно, я всё сделаю, — сказала Бай Нинбин.", "stop" + } + return "ЧЕРНОВИК", "stop" + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{ + source: voiceSource, glossarySeed: voiceSeed, gatesYAML: voiceGateBlock, + }) + r := newRunner(t, bookPath) + defer r.Close() + res, err := r.TranslateBook(context.Background()) + if err != nil { + t.Fatal(err) + } + if res.Flagged != 0 { + t.Fatalf("the voice flagger is observability — it must never flag a unit, got flagged=%d", res.Flagged) + } + rs, err := r.Store.GetRetrievalState("test-book", 1, 0) + if err != nil || rs == nil { + t.Fatalf("no retrieval state persisted: %v", err) + } + if rs.NVoiceFlags == 0 { + t.Fatalf("the voice flags did not reach retrieval_state: %+v", rs) + } + if !strings.Contains(rs.VoiceDetail, "Бай Нинбин") { + t.Fatalf("the detail must name the character: %q", rs.VoiceDetail) + } + rep, err := r.QualityReport() + if err != nil { + t.Fatal(err) + } + if rep.VoiceFlags != rs.NVoiceFlags || rep.VoiceReplies == 0 || rep.VoiceAttributed == 0 { + t.Fatalf("the report must aggregate the flags AND their denominators: %+v", rep) + } + if rep.VoiceCheckVersion == "" { + t.Fatal("the gate is not snapshot-folded, so the report MUST carry the rule version the counts came from") + } +} + +// The gate is off by default, and off means zero — not "computed and hidden". +func TestVoiceFlaggerIsSilentWhenTheGateIsOff(t *testing.T) { + srv := newJSONProvider(&reqRec{}, func(body string) (string, string) { + if isEditBody(body) { + return "— Ладно, я всё сделаю, — сказала Бай Нинбин.", "stop" + } + return "ЧЕРНОВИК", "stop" + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: voiceSource, glossarySeed: voiceSeed}) + r := newRunner(t, bookPath) + defer r.Close() + if _, err := r.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + rs, err := r.Store.GetRetrievalState("test-book", 1, 0) + if err != nil || rs == nil { + t.Fatalf("no retrieval state persisted: %v", err) + } + if rs.NVoiceFlags != 0 || rs.VoiceDetail != "" { + t.Fatalf("gate off must write nothing at all: %+v", rs) + } + rep, err := r.QualityReport() + if err != nil { + t.Fatal(err) + } + if rep.VoiceFlags != 0 || rep.VoiceCheckVersion != "" { + t.Fatalf("gate off must report nothing: %+v", rep) + } +} + +// The reveal half of D21 п.3: a rendering the spoiler window rejected for this chapter that reached the +// shipped text. NOT gated by gates.voice — the window is a standing safety mechanism. +func TestSpoilerLeakSurfacesUngated(t *testing.T) { + const seed = ` +terms: + - src: 真相 + dst: её брат + since_ch: 5 +` + srv := newJSONProvider(&reqRec{}, func(body string) (string, string) { + if isEditBody(body) { + return "Это был её брат.", "stop" // the post-reveal rendering, in chapter 1 + } + return "ЧЕРНОВИК", "stop" + }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "真相 出现了。", glossarySeed: seed}) + r := newRunner(t, bookPath) + defer r.Close() + res, err := r.TranslateBook(context.Background()) + if err != nil { + t.Fatal(err) + } + if res.Flagged != 0 { + t.Fatalf("a leak is observability, not a disposition, got flagged=%d", res.Flagged) + } + rs, err := r.Store.GetRetrievalState("test-book", 1, 0) + if err != nil || rs == nil { + t.Fatalf("no retrieval state persisted: %v", err) + } + if rs.NSpoilerLeaks != 1 || !strings.Contains(rs.SpoilerLeakDetail, "真相") { + t.Fatalf("the spoiler leak did not reach retrieval_state: %+v", rs) + } + rep, err := r.QualityReport() + if err != nil { + t.Fatal(err) + } + if rep.SpoilerLeaks != 1 { + t.Fatalf("the report must aggregate the leak: %+v", rep) + } +} + +// A voice/address row naming a character the bank does not hold is silently inert — so the run must stop +// at seeding, before any provider call, rather than translate a book whose profiles can never fire. +func TestSeedRefusesAVoiceProfileForAnUnknownCharacter(t *testing.T) { + const seed = ` +terms: + - src: 方源 + dst: Фан Юань +voices: + - src: 白凝冰 + self_ref: ваша служанка +` + srv := newJSONProvider(&reqRec{}, func(string) (string, string) { return "ЧЕРНОВИК", "stop" }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: voiceSource, glossarySeed: seed}) + r := newRunner(t, bookPath) + defer r.Close() + _, err := r.TranslateBook(context.Background()) + if err == nil { + t.Fatal("a profile for a term that does not exist must stop the run") + } + if !strings.Contains(err.Error(), "白凝冰") { + t.Fatalf("the error must name the character: %v", err) + } +} + +// The voice content is bank content: it is replaced in the SAME transaction as the glossary and read +// back for the projection, so a re-seed converges every column exactly as the terms do. +func TestVoiceContentRoundTripsThroughTheBankReplace(t *testing.T) { + srv := newJSONProvider(&reqRec{}, func(string) (string, string) { return "ЧЕРНОВИК", "stop" }) + defer srv.Close() + bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: voiceSource, glossarySeed: voiceSeed}) + r := newRunner(t, bookPath) + defer r.Close() + if _, err := r.TranslateBook(context.Background()); err != nil { + t.Fatal(err) + } + voices, err := r.Store.VoiceProfilesForBook("test-book") + if err != nil || len(voices) != 1 { + t.Fatalf("voice profiles not persisted: %v %+v", err, voices) + } + if voices[0].SelfRef != "ваша служанка" || voices[0].NGLexicon != `["ладно"]` { + t.Fatalf("voice profile columns did not round-trip: %+v", voices[0]) + } + pairs, err := r.Store.AddressPairsForBook("test-book") + if err != nil || len(pairs) != 1 || pairs[0].Register != "formal" { + t.Fatalf("address pairs not persisted: %v %+v", err, pairs) + } +} diff --git a/backend/internal/pipeline/waverun.go b/backend/internal/pipeline/waverun.go index aaeec649..cb8ae58b 100644 --- a/backend/internal/pipeline/waverun.go +++ b/backend/internal/pipeline/waverun.go @@ -386,8 +386,14 @@ func (r *Runner) runDraftChunk(ctx context.Context, draftSnapshot string, ch chu if isFinalWave && !seq.flagged && seq.finalText != "" && isRuTarget(r.Book.TargetLang) { cheap = checks.RunCheapGates(ch.Text, seq.finalText, seq.finalText, r.cheapGateConfig()) } + var voice checks.VoiceResult + var leaks []membank.SpoilerLeak + if isFinalWave && !seq.flagged && seq.finalText != "" { + voice = r.runVoiceChecks(r.baseMemory, ch.Chapter, ch.Text, seq.finalText) + leaks = r.spoilerLeaks(r.baseMemory, memSel, seq.finalText) + } if r.baseMemory != nil { - if err := r.persistRetrievalState(draftSnapshot, ch, memSel, postMisses, outputChecked, cheap, seq.bankTelem, seq.bankProps); err != nil { + if err := r.persistRetrievalState(draftSnapshot, ch, memSel, postMisses, outputChecked, cheap, seq.bankTelem, seq.bankProps, voice, leaks); err != nil { return seq, err } } @@ -498,8 +504,24 @@ func (r *Runner) runEditUnit(ctx context.Context, editSnapshot string, unit edit "chapter", unit.Chapter, "leader_chunk", unit.FirstChunkIdx, "style_flags", cheap.Total()) } } + // The pack-19 flaggers, on the same shipped text and under the same "clean unit" condition as the + // cheap gates: a flagged unit ships nothing, so measuring its prose would report defects nobody reads. + var voice checks.VoiceResult + var leaks []membank.SpoilerLeak + if !flagged && finalText != "" { + voice = r.runVoiceChecks(r.memory, unit.Chapter, leader.Text, finalText) + if voice.Total() > 0 { + r.Log.InfoContext(ctx, "voice flagger fired on the edit unit (observability, not a gate)", + "chapter", unit.Chapter, "leader_chunk", unit.FirstChunkIdx, + "voice_flags", voice.Total(), "attributed_replies", voice.Attributed, "version", checks.VoiceCheckVersion) + } + if leaks = r.spoilerLeaks(r.memory, editSel, finalText); len(leaks) > 0 { + r.Log.WarnContext(ctx, "spoiler leak: a rendering this chapter must not know yet reached the output", + "chapter", unit.Chapter, "leader_chunk", unit.FirstChunkIdx, "leaks", len(leaks), "first", leaks[0].Src) + } + } if r.memory != nil { - if err := r.mergeUnitRetrievalState(unit.Chapter, unit.FirstChunkIdx, postMisses, outputChecked, cheap); err != nil { + if err := r.mergeUnitRetrievalState(unit.Chapter, unit.FirstChunkIdx, postMisses, outputChecked, cheap, voice, leaks); err != nil { return nil, err } } @@ -553,7 +575,7 @@ func (r *Runner) recordSkippedStages(ctx context.Context, staged []wavedStage, s // telemetry. A read-modify-write (single-writer safe; each unit owns a distinct leader chunk_idx) preserves // the draft fields and adds the unit-level post-check/style fields, so the read-models aggregate injection // per draft chunk and post-check/style per unit. A missing row (r.memory nil path) is a no-op. -func (r *Runner) mergeUnitRetrievalState(chapter, firstChunkIdx int, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult) error { +func (r *Runner) mergeUnitRetrievalState(chapter, firstChunkIdx int, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult, voice checks.VoiceResult, leaks []membank.SpoilerLeak) error { rs, err := r.Store.GetRetrievalState(r.Book.BookID, chapter, firstChunkIdx) if err != nil { return fmt.Errorf("pipeline: read leader retrieval_state ch%d/chunk%d: %w", chapter, firstChunkIdx, err) @@ -568,6 +590,7 @@ func (r *Runner) mergeUnitRetrievalState(chapter, firstChunkIdx int, misses memb rs.StyleDetail = string(b) } } + setVoiceState(rs, voice, leaks) rs.NPostcheckMiss = 0 rs.PostcheckDetail = "" if outputChecked { diff --git a/backend/internal/seed/seed.go b/backend/internal/seed/seed.go index 4132b845..44dfc071 100644 --- a/backend/internal/seed/seed.go +++ b/backend/internal/seed/seed.go @@ -17,6 +17,45 @@ package seed // File is a whole seed YAML document. type File struct { Terms []Term `yaml:"terms"` + // Voices and Addresses are the two D21 record types added by pack-19. They are OWNER-CURATED only — + // no miner emits them — so they have no counterpart on the mined-delta side of this schema. + Voices []Voice `yaml:"voices"` + Addresses []Address `yaml:"addresses"` +} + +// Voice is one character's voice profile for a chapter window (research/15 §1.5). Src/Sense name the +// glossary term this profile belongs to — the same stable key the bank uses everywhere else. +type Voice struct { + Src string `yaml:"src"` + Sense string `yaml:"sense"` + // Register is one line of register+tone; SelfRef is the character's own self-designation (the field + // D7 reserved as first_person); AddressDefault is the T/V default outside the pair registry. + Register string `yaml:"register"` + SelfRef string `yaml:"self_ref"` + AddressDefault string `yaml:"address_default"` // informal|formal + LexiconMarkers []string `yaml:"lexicon_markers"` + NGLexicon []string `yaml:"ng_lexicon"` + Exemplars []string `yaml:"exemplars"` + Brightness string `yaml:"brightness"` + SinceCh int `yaml:"since_ch"` + UntilCh int `yaml:"until_ch"` +} + +// Address is one entry of the ты/вы transition journal: the register the SPEAKER uses toward the +// ADDRESSEE over a chapter window. A register change is a new row with a non-overlapping window, never +// an edit of this one — the switch is a story event. +type Address struct { + Speaker string `yaml:"speaker"` + SpeakerSense string `yaml:"speaker_sense"` + Addressee string `yaml:"addressee"` + AddresseeSense string `yaml:"addressee_sense"` + // Register is ABSTRACT (informal|formal): which target surfaces realise it is target data, so a + // target language without a T/V distinction activates nothing. + Register string `yaml:"register"` + Form string `yaml:"form"` + Closeness string `yaml:"closeness"` + SinceCh int `yaml:"since_ch"` + UntilCh int `yaml:"until_ch"` } // Term is one seeded term: the source surface, its rendering and the metadata the bank diff --git a/backend/internal/store/glossary.go b/backend/internal/store/glossary.go index 97bbfa7e..80d93d76 100644 --- a/backend/internal/store/glossary.go +++ b/backend/internal/store/glossary.go @@ -98,15 +98,34 @@ type RetrievalState struct { // row is a candidate the model is entitled to reject. NUnverifiedShown int NUnverifiedFollowed int + // NVoiceFlags / VoiceDetail are the deterministic voice flagger (pack-19, gates.voice): T/V + // contradictions, flattened self-designations and forbidden lexemes in attributed replies. Zero for + // every gate-off unit. NSpoilerLeaks / SpoilerLeakDetail are the reveal half of D21 п.3: a rendering + // the spoiler window REJECTED for this chapter that reached the output anyway. Both are observability + // (never a disposition) and re-derived each run, like every sibling on this row. + NVoiceFlags int + VoiceDetail string // JSON + NSpoilerLeaks int + SpoilerLeakDetail string // JSON } -// ReplaceGlossary replaces a book's ENTIRE glossary (rows + aliases) with the given -// entries in ONE transaction, and appends a B1 revision record for every term whose -// approved rendering (dst) changed since the previous set. Scoped to book_id, so other -// books are untouched. Each entry's BookID must equal bookID. term_id linkage is -// established within the tx: a glossary row is inserted, its rowid taken, then its -// aliases inserted against that id (ids are fresh each replace — never hashed). +// ReplaceGlossary replaces a book's glossary with no voice/address rows — the pre-pack-19 form, kept +// because most callers (and every glossary-only test) have nothing else to write. func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error { + return s.ReplaceBank(bookID, entries, nil, nil) +} + +// ReplaceBank replaces a book's ENTIRE bank — glossary rows + aliases + voice profiles + address pairs — +// with the given content in ONE transaction, and appends a B1 revision record for every term whose +// approved rendering (dst) changed since the previous set. Scoped to book_id, so other books are +// untouched. Each entry's BookID must equal bookID. term_id linkage is established within the tx: a +// glossary row is inserted, its rowid taken, then its aliases inserted against that id (ids are fresh +// each replace — never hashed). +// +// The three record types share ONE transaction on purpose: they fold into one memory_version, so a +// half-applied replace would make the materialized bank disagree with the hash the snapshot pins it by. +// Voice/address rows carry no id linkage at all — they reference a character by the stable (src, sense). +func (s *Store) ReplaceBank(bookID string, entries []GlossaryEntry, voices []VoiceProfile, pairs []AddressPair) error { ctx, cancel := opContext() defer cancel() tx, err := s.w.BeginTx(ctx, nil) @@ -136,11 +155,10 @@ func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error { } rows.Close() - if _, err := tx.ExecContext(ctx, `DELETE FROM glossary_aliases WHERE book_id = ?`, bookID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM glossary WHERE book_id = ?`, bookID); err != nil { - return err + for _, table := range []string{"glossary_aliases", "glossary", "voice_profiles", "address_pairs"} { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE book_id = ?`, bookID); err != nil { + return err + } } for _, e := range entries { @@ -179,6 +197,28 @@ func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error { } } } + for _, v := range voices { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO voice_profiles ( + book_id, src, sense, register, self_ref, address_default, + lexicon_markers, ng_lexicon, exemplars, brightness, since_ch, until_ch + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`, + bookID, v.Src, v.Sense, v.Register, v.SelfRef, v.AddressDefault, + v.LexiconMarkers, v.NGLexicon, v.Exemplars, v.Brightness, v.SinceCh, v.UntilCh); err != nil { + return err + } + } + for _, p := range pairs { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO address_pairs ( + book_id, speaker_src, speaker_sense, addressee_src, addressee_sense, + register, form, closeness, since_ch, until_ch + ) VALUES (?,?,?,?,?,?,?,?,?,?)`, + bookID, p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense, + p.Register, p.Form, p.Closeness, p.SinceCh, p.UntilCh); err != nil { + return err + } + } return tx.Commit() } @@ -265,8 +305,9 @@ func (s *Store) UpsertRetrievalState(rs RetrievalState) error { embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids, n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail, n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail, - n_unverified_shown, n_unverified_followed, updated_at - ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now')) + n_unverified_shown, n_unverified_followed, + n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail, updated_at + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now')) ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET snapshot_id = excluded.snapshot_id, n_exact_hits = excluded.n_exact_hits, @@ -288,13 +329,18 @@ func (s *Store) UpsertRetrievalState(rs RetrievalState) error { banknote_truncated = excluded.banknote_truncated, n_unverified_shown = excluded.n_unverified_shown, n_unverified_followed = excluded.n_unverified_followed, + n_voice_flags = excluded.n_voice_flags, + voice_detail = excluded.voice_detail, + n_spoiler_leaks = excluded.n_spoiler_leaks, + spoiler_leak_detail = excluded.spoiler_leak_detail, updated_at = excluded.updated_at`, rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID, rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted, rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs, rs.NStyleFlags, rs.StyleDetail, rs.NTrustGatedSuppress, rs.TrustGateDetail, rs.NBanknoteLines, rs.BanknoteParseFail, rs.BanknoteTruncated, rs.BanknoteDetail, - rs.NUnverifiedShown, rs.NUnverifiedFollowed) + rs.NUnverifiedShown, rs.NUnverifiedFollowed, + rs.NVoiceFlags, rs.VoiceDetail, rs.NSpoilerLeaks, rs.SpoilerLeakDetail) return err } @@ -308,14 +354,16 @@ func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*Retrie n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids, n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail, n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail, - n_unverified_shown, n_unverified_followed + n_unverified_shown, n_unverified_followed, + n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`, bookID, chapter, chunkIdx).Scan( &rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs, &rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail, &rs.NBanknoteLines, &rs.BanknoteParseFail, &rs.BanknoteTruncated, &rs.BanknoteDetail, - &rs.NUnverifiedShown, &rs.NUnverifiedFollowed) + &rs.NUnverifiedShown, &rs.NUnverifiedFollowed, + &rs.NVoiceFlags, &rs.VoiceDetail, &rs.NSpoilerLeaks, &rs.SpoilerLeakDetail) if errors.Is(err, sql.ErrNoRows) { return nil, nil } @@ -333,7 +381,8 @@ func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error) n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids, n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail, n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail, - n_unverified_shown, n_unverified_followed + n_unverified_shown, n_unverified_followed, + n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail FROM retrieval_state WHERE book_id = ? ORDER BY chapter, chunk_idx`, func(rows *sql.Rows) (RetrievalState, error) { @@ -343,7 +392,8 @@ func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error) &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs, &rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail, &rs.NBanknoteLines, &rs.BanknoteParseFail, &rs.BanknoteTruncated, &rs.BanknoteDetail, - &rs.NUnverifiedShown, &rs.NUnverifiedFollowed) + &rs.NUnverifiedShown, &rs.NUnverifiedFollowed, + &rs.NVoiceFlags, &rs.VoiceDetail, &rs.NSpoilerLeaks, &rs.SpoilerLeakDetail) return rs, err }, bookID) } diff --git a/backend/internal/store/migrate.go b/backend/internal/store/migrate.go index 6ffa246d..c2f64650 100644 --- a/backend/internal/store/migrate.go +++ b/backend/internal/store/migrate.go @@ -181,7 +181,7 @@ var migrations = []string{ type TEXT NOT NULL DEFAULT '', -- name|term|title|… alias-graph node type (B3) sense TEXT NOT NULL DEFAULT '', -- A3 polysemy disambiguator (part of the uniqueness key) gender TEXT NOT NULL DEFAULT '', -- male|female|hidden|'' (C2; hidden = gender-avoidant rendering until reveal) - speech TEXT NOT NULL DEFAULT '', -- ты/вы register note (series-bible-lite; the transition JOURNAL is deferred) + speech TEXT NOT NULL DEFAULT '', -- DEPRECATED (pack-19): superseded by the address_pairs journal (v13); never read, never hashed decl TEXT NOT NULL DEFAULT '', -- JSON {"invariant":bool,"forms":[...]}: the dst declension forms the post-check accepts (E1/E2 safety, decl = grammaticality mechanism, not a market feature) translit_policy TEXT NOT NULL DEFAULT '', -- B5 western-name-via-katakana (field only in v1; validator = Phase 2) first_person TEXT NOT NULL DEFAULT '', -- D7 first_person field (field only) @@ -344,37 +344,101 @@ var migrations = []string{ ALTER TABLE retrieval_state ADD COLUMN n_unverified_shown INTEGER NOT NULL DEFAULT 0; ALTER TABLE retrieval_state ADD COLUMN n_unverified_followed INTEGER NOT NULL DEFAULT 0; `, + // v13 (pack-19 / D39.55): the two D21 record types the bank was missing — the per-character VOICE + // profile and the ordered ADDRESS pair. They are TABLES rather than glossary columns for one hard + // reason: a character's term row and its voice profile share (src, sense, since_ch, until_ch), which + // is the glossary UNIQUE key, so the profile could not coexist with the term it describes without + // rebuilding that constraint — forbidden by the append-only discipline above. + // + // They are nevertheless BANK CONTENT, not a store beside the bank: their rows are replaced in the SAME + // transaction as the glossary and fold into the SAME memory_version (conditionally — see + // membank.ComputeVersionScopedIn). Giving them a snapshot key of their own would classify a signature + // as moveOther and cost a whole edit wave instead of the touched units (pipeline/repin.go). + // + // address_pairs is the MATERIALIZATION of the ты/вы transition journal (D7-Amendment-1, D21 п.2), not a + // second source of truth: a row IS a journal entry (register valid from since_ch to until_ch) and the + // "current register at chapter N" is the projection membank computes, never a stored state. A ты↔вы + // switch is a second row with a non-overlapping window, exactly like a spoiler handoff. + // + // Both reference characters by the STABLE key (src, sense) rather than glossary.id: that id is a fresh + // autoincrement on every replace (see v5), so an id link would silently re-point after any re-seed. + ` + CREATE TABLE IF NOT EXISTS voice_profiles ( + id INTEGER PRIMARY KEY, + book_id TEXT NOT NULL, + src TEXT NOT NULL, -- character identity, half 1 (the glossary term's src) + sense TEXT NOT NULL DEFAULT '', -- character identity, half 2 + register TEXT NOT NULL DEFAULT '', -- one line: register + tone + self_ref TEXT NOT NULL DEFAULT '', -- the character's own self-designation (D7 first_person) + address_default TEXT NOT NULL DEFAULT '', -- informal|formal|'' — the T/V default outside the pair registry + lexicon_markers TEXT NOT NULL DEFAULT '', -- JSON []string: characteristic words + ng_lexicon TEXT NOT NULL DEFAULT '', -- JSON []string: words this character never says + exemplars TEXT NOT NULL DEFAULT '', -- JSON []string: 3-5 approved target replies + brightness TEXT NOT NULL DEFAULT '', -- injection priority hint (D21 п.1в: hypothesis, not a rule) + since_ch INTEGER NOT NULL DEFAULT 0, -- manner-change window, same axis as the glossary spoiler window + until_ch INTEGER NOT NULL DEFAULT 0, + UNIQUE (book_id, src, sense, since_ch, until_ch) + ); + CREATE INDEX IF NOT EXISTS voice_profiles_book_idx ON voice_profiles (book_id); + + CREATE TABLE IF NOT EXISTS address_pairs ( + id INTEGER PRIMARY KEY, + book_id TEXT NOT NULL, + speaker_src TEXT NOT NULL, + speaker_sense TEXT NOT NULL DEFAULT '', + addressee_src TEXT NOT NULL, + addressee_sense TEXT NOT NULL DEFAULT '', + register TEXT NOT NULL, -- informal|formal — ABSTRACT; the surfaces are target data + form TEXT NOT NULL DEFAULT '', -- the address form («молодой господин», a title) + closeness TEXT NOT NULL DEFAULT '', -- one word of hierarchy/closeness (why the register is what it is) + since_ch INTEGER NOT NULL DEFAULT 0, -- a ты<->вы switch is a STORY event: a second row, new window + until_ch INTEGER NOT NULL DEFAULT 0, + UNIQUE (book_id, speaker_src, speaker_sense, addressee_src, addressee_sense, since_ch, until_ch) + ); + CREATE INDEX IF NOT EXISTS address_pairs_book_idx ON address_pairs (book_id); + `, + // v14 (pack-19): the per-chunk observability of the two new $0 flaggers, riding the same + // recomputed-every-run row as the glossary post-check and the cheap style gates. + // + // • n_voice_flags / voice_detail — the deterministic T/V + voice-marker flagger (gates.voice); + // • n_spoiler_leaks / spoiler_leak_detail — a rendering the spoiler window REJECTED for this + // chapter that appeared in the output anyway (the reveal half of D21 п.3). + // + // Neither is ever a disposition, and both are re-derived deterministically each run (self-healing on + // resume), so they are observability exactly like their siblings. Additive with defaults ⇒ every prior + // row and the WIRE are unchanged. + ` + ALTER TABLE retrieval_state ADD COLUMN n_voice_flags INTEGER NOT NULL DEFAULT 0; + ALTER TABLE retrieval_state ADD COLUMN voice_detail TEXT NOT NULL DEFAULT ''; + ALTER TABLE retrieval_state ADD COLUMN n_spoiler_leaks INTEGER NOT NULL DEFAULT 0; + ALTER TABLE retrieval_state ADD COLUMN spoiler_leak_detail TEXT NOT NULL DEFAULT ''; + `, } -// DESIGN NOTE (D21.10 — reserved memory-bank v2 record types; Phase 2, NOT a migration, NOT code). +// DESIGN NOTE (D21.10 → BUILT by pack-19 / D39.55; kept as the record of what each type is FOR). // -// research/15 (D21) ratified three future memory MECHANISMS whose storage will extend the bank. This -// note reserves their shape so a later Phase 2 migration lands cleanly against the existing determinism/ -// snapshot discipline. It is DELIBERATELY not appended to `migrations` above: the mechanisms are Phase 2 -// (D21.1–3), gated on the pilot and on D15.2 landing — reserving a column now would be dead schema. +// research/15 (D21) ratified three memory MECHANISMS extending the bank. Two are now schema (v13); the +// third turned out to need none. // -// • voice_profile (D21.1) — per-character voice exemplars + descriptors, injected DRAFT-side -// (cap ≤300 tok/chunk; injection priority CONFIRMED-glossary > address_pair > voice_profile > -// reveal-aliases). FROZEN per-job until D15.2 lands (self-populating exemplars = mid-run memory -// append = book re-pay under D8/D15.1). Likely a `voice_profiles` table keyed (book_id, term_id) -// with exemplar/descriptor columns + a since_ch/until_ch window mirroring the glossary spoiler -// axis. Injected content rides content_hash; any live verdict rides verdictSnapshotID (D15.2). +// • voice_profile (D21.1) — BUILT as `voice_profiles` (v13), keyed by the STABLE (src, sense) rather +// than the reserved (book_id, term_id): glossary.id is a fresh autoincrement per replace. Still +// FROZEN per-job (self-populating exemplars = mid-run memory append = book re-pay under D8/D15.1); +// the condition for lifting it — D15.2 landing — has NOT happened (verdictSnapshotID/guard_hash do +// not exist in this engine, only in comments). // -// • address_pair (D21.2) — the ты/вы register for an ORDERED character pair. It is the -// MATERIALIZATION of the D7-Amendment-1 ты/вы transition JOURNAL, NOT a second store: the single -// source of truth is the journal (a ты↔вы switch is a STORY event), and address_pair is only its -// resolved current-state PROJECTION (recomputable, self-healing). Deterministic flagger -// unconditionally; injection only after the hard-pairs probe (lift on easy pairs = 0 — a -// flagger-only outcome is legitimate). Likely `address_pairs` (book_id, term_id_a, term_id_b, -// register, since_ch) DERIVED from a `speech_transitions` journal — must not duplicate the -// journal's authority (glossary.speech today is a static note; the journal supersedes it in Phase 2). +// • address_pair (D21.2) — BUILT as `address_pairs` (v13). The reserved shape guessed a derived +// projection over a separate `speech_transitions` journal; the built one collapses the two, because a +// windowed row IS a journal entry and the "register at chapter N" projection is computed, never +// stored. So there is still exactly one source of truth, with one table instead of two. +// `glossary.speech` (the static v5 note) is superseded and DEPRECATED — see its column comment. // -// • reveal_ch + pre/post-reveal aliases (D21.3) — a reveal window (`reveal_ch`) plus alias sets -// valid BEFORE vs AFTER the reveal (gender/identity twist: a post-reveal dst must not leak before -// reveal_ch — same self-inflicted-spoiler logic as since_ch/until_ch windows). Likely a `reveal_ch` -// column on glossary + alias rows tagged phase=pre|post, so injection selects by current_ch vs reveal_ch. -// -// None of these is a column today; Phase 2 implements them behind their own migration + tests. +// • reveal_ch + pre/post-reveal aliases (D21.3) — NOT built, and deliberately: the pre/post rendering +// pair is already expressible as two glossary rows with NON-OVERLAPPING windows (the UNIQUE key +// admits them, membank.windowsOverlap permits them, spoilerBlocked picks exactly one), and aliases +// inherit their row's window, so a phase tag would be a third way to say what two already say. What +// was genuinely missing is the LEAK CHECK — a rejected rendering appearing in the output anyway — +// which is membank.SpoilerLeaks over the selection's Rejected set, no schema at all. Scope amended +// by D39.55 (form, not intent). // migrate runs all pending migrations on the write pool, one transaction per // step, recording each in schema_version. diff --git a/backend/internal/store/voice.go b/backend/internal/store/voice.go new file mode 100644 index 00000000..f2b28661 --- /dev/null +++ b/backend/internal/store/voice.go @@ -0,0 +1,74 @@ +package store + +import "database/sql" + +// voice.go: durable storage for the two D21 bank record types added by pack-19 (schema v13) — the +// per-character VOICE profile and the ordered ADDRESS pair. Dumb storage like glossary.go: this layer +// replaces and reads back, the pipeline owns validation, projection and matching. +// +// Both are BANK CONTENT: they are replaced in the same transaction as the glossary (ReplaceBank) and +// fold into the same memory_version, so a signature on either is a bank-only snapshot move and stays +// re-pinnable per unit (pipeline/repin.go) instead of re-buying an edit wave. + +// VoiceProfile is one character's voice record for a chapter window (research/15 §1.5). The character is +// identified by the STABLE (Src, Sense) key of its glossary term — never by glossary.id, which is a fresh +// autoincrement on every replace. The list-valued columns are stored as JSON; membank owns the encoding. +type VoiceProfile struct { + Src string + Sense string + Register string + SelfRef string + AddressDefault string // informal|formal|"" — the T/V default outside the pair registry + LexiconMarkers string // JSON []string + NGLexicon string // JSON []string + Exemplars string // JSON []string + Brightness string + SinceCh int + UntilCh int +} + +// AddressPair is one ORDERED (speaker → addressee) register record for a chapter window — one entry of +// the ты/вы transition journal (D7-Amendment-1). Register is ABSTRACT (informal|formal); which target +// surfaces realise it is target data, so a target without a T/V distinction simply matches nothing. +type AddressPair struct { + SpeakerSrc string + SpeakerSense string + AddresseeSrc string + AddresseeSense string + Register string + Form string + Closeness string + SinceCh int + UntilCh int +} + +// VoiceProfilesForBook returns a book's voice profiles in a DETERMINISTIC order — the stable +// materialization the bank version hashes over. +func (s *Store) VoiceProfilesForBook(bookID string) ([]VoiceProfile, error) { + return queryAll(s.r, ` + SELECT src, sense, register, self_ref, address_default, + lexicon_markers, ng_lexicon, exemplars, brightness, since_ch, until_ch + FROM voice_profiles WHERE book_id = ? + ORDER BY src, sense, since_ch, until_ch`, + func(rows *sql.Rows) (VoiceProfile, error) { + var v VoiceProfile + err := rows.Scan(&v.Src, &v.Sense, &v.Register, &v.SelfRef, &v.AddressDefault, + &v.LexiconMarkers, &v.NGLexicon, &v.Exemplars, &v.Brightness, &v.SinceCh, &v.UntilCh) + return v, err + }, bookID) +} + +// AddressPairsForBook returns a book's address-register journal in a DETERMINISTIC order. +func (s *Store) AddressPairsForBook(bookID string) ([]AddressPair, error) { + return queryAll(s.r, ` + SELECT speaker_src, speaker_sense, addressee_src, addressee_sense, + register, form, closeness, since_ch, until_ch + FROM address_pairs WHERE book_id = ? + ORDER BY speaker_src, speaker_sense, addressee_src, addressee_sense, since_ch, until_ch`, + func(rows *sql.Rows) (AddressPair, error) { + var p AddressPair + err := rows.Scan(&p.SpeakerSrc, &p.SpeakerSense, &p.AddresseeSrc, &p.AddresseeSense, + &p.Register, &p.Form, &p.Closeness, &p.SinceCh, &p.UntilCh) + return p, err + }, bookID) +} diff --git a/backend/internal/store/voice_test.go b/backend/internal/store/voice_test.go new file mode 100644 index 00000000..3e955443 --- /dev/null +++ b/backend/internal/store/voice_test.go @@ -0,0 +1,116 @@ +package store + +import ( + "database/sql" + "testing" +) + +// voice_test.go: the pack-19 schema, on the two axes that can only be checked by execution — that an +// EXISTING database upgrades onto it, and that the bank's three record types are replaced as one unit. + +// The migration that matters is not the fresh one (every test creates that) but the UPGRADE: a stand +// database already at an older version must gain the pack-19 tables and columns without losing a row. +func TestMigrationUpgradesAnOlderDatabase(t *testing.T) { + const olderVersion = 12 // the schema head before pack-19 (v13 tables + v14 retrieval-state columns) + if len(migrations) <= olderVersion { + t.Fatalf("this test assumes pack-19 appended to a %d-step schema, got %d", olderVersion, len(migrations)) + } + path := t.TempDir() + "/old.db" + // Build a database at the OLD head by applying only the old steps through a raw connection. + raw, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err := raw.Exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY)`); err != nil { + t.Fatal(err) + } + for i := 0; i < olderVersion; i++ { + if _, err := raw.Exec(migrations[i]); err != nil { + t.Fatalf("apply old migration %d: %v", i+1, err) + } + if _, err := raw.Exec(`INSERT INTO schema_version (version) VALUES (?)`, i+1); err != nil { + t.Fatal(err) + } + } + // A row written under the OLD schema must survive the upgrade untouched. + if _, err := raw.Exec(`INSERT INTO glossary (book_id, src, dst, status) VALUES ('b','旧','старое','approved')`); err != nil { + t.Fatal(err) + } + // Premise check: at the OLD head the pack-19 schema must genuinely be absent, or this test would + // prove nothing about upgrading. + if _, qerr := raw.Query(`SELECT n_voice_flags FROM retrieval_state`); qerr == nil { + t.Fatal("the old schema already has the pack-19 columns — the upgrade is not being exercised") + } + if _, qerr := raw.Query(`SELECT 1 FROM voice_profiles`); qerr == nil { + t.Fatal("the old schema already has voice_profiles — the upgrade is not being exercised") + } + if err := raw.Close(); err != nil { + t.Fatal(err) + } + + s, err := Open(path) // runs the pending migrations + if err != nil { + t.Fatalf("an existing database must upgrade onto the pack-19 schema: %v", err) + } + defer s.Close() + + rows, err := s.GlossaryForBook("b") + if err != nil || len(rows) != 1 || rows[0].Dst != "старое" { + t.Fatalf("the pre-existing row must survive the upgrade: %v %+v", err, rows) + } + // The new tables and columns are usable, not merely present. + if err := s.ReplaceBank("b", rows, + []VoiceProfile{{Src: "旧", SelfRef: "ваша служанка"}}, + []AddressPair{{SpeakerSrc: "旧", AddresseeSrc: "новое", Register: "formal"}}); err != nil { + t.Fatalf("write to the upgraded schema: %v", err) + } + if v, err := s.VoiceProfilesForBook("b"); err != nil || len(v) != 1 { + t.Fatalf("voice_profiles unusable after the upgrade: %v %+v", err, v) + } + if err := s.UpsertRetrievalState(RetrievalState{ + BookID: "b", Chapter: 1, ChunkIdx: 0, SnapshotID: "s", NVoiceFlags: 3, VoiceDetail: `{"replies":2}`, + NSpoilerLeaks: 1, SpoilerLeakDetail: `[{"src":"旧"}]`, + }); err != nil { + t.Fatalf("write the new retrieval-state columns after the upgrade: %v", err) + } + got, err := s.GetRetrievalState("b", 1, 0) + if err != nil || got == nil || got.NVoiceFlags != 3 || got.NSpoilerLeaks != 1 || got.VoiceDetail == "" { + t.Fatalf("the new columns did not round-trip after the upgrade: %v %+v", err, got) + } +} + +// The bank is replaced as ONE unit: a re-seed that drops a profile must drop it from the store too, or a +// stale row would keep feeding the flagger a character the seed no longer signs. +func TestReplaceBankIsAFullReplaceOfAllThreeRecordTypes(t *testing.T) { + s, _ := openTemp(t) + terms := []GlossaryEntry{{BookID: "b", Src: "方源", Dst: "Фан Юань", Status: "approved"}} + if err := s.ReplaceBank("b", terms, + []VoiceProfile{{Src: "方源", SelfRef: "первый"}, {Src: "白凝冰", SelfRef: "второй"}}, + []AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "formal"}}); err != nil { + t.Fatal(err) + } + if v, _ := s.VoiceProfilesForBook("b"); len(v) != 2 { + t.Fatalf("setup: want 2 profiles, got %d", len(v)) + } + // Re-seed with ONE profile and no pairs at all. + if err := s.ReplaceBank("b", terms, []VoiceProfile{{Src: "方源", SelfRef: "первый"}}, nil); err != nil { + t.Fatal(err) + } + v, err := s.VoiceProfilesForBook("b") + if err != nil || len(v) != 1 || v[0].Src != "方源" { + t.Fatalf("the dropped profile lingered: %v %+v", err, v) + } + if p, _ := s.AddressPairsForBook("b"); len(p) != 0 { + t.Fatalf("the dropped address pairs lingered: %+v", p) + } + // Another book's content is untouched by a replace scoped to this one. + if err := s.ReplaceBank("other", nil, []VoiceProfile{{Src: "X", SelfRef: "чужой"}}, nil); err != nil { + t.Fatal(err) + } + if err := s.ReplaceBank("b", terms, nil, nil); err != nil { + t.Fatal(err) + } + if o, _ := s.VoiceProfilesForBook("other"); len(o) != 1 { + t.Fatalf("a replace must be scoped to its book, got %+v", o) + } +} diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index a45dd5b4..c36d5463 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -1,7 +1,7 @@ # Журнал прогресса > **⟶ ТЕКУЩЕЕ СОСТОЯНИЕ** (на 2026-07-26: пак-20 закрыт ЦЕЛИКОМ D39.41–53 — терминолог · флаговый стоп · двухсекционная инъекция · точечная ре-редактура · языковой экран; жанровый словарь отменён D39.47; полигон-пакеты 6/7/8 закрыты). **Источник истины по РЕШЕНИЯМ — `architecture/05-decisions-log.md` (D1–D39.53); этот файл — ЖУРНАЛ.** -> - **Фазы/курс:** Ф0 ✅ · Ф1-инфра ✅ (D20–D28; golden = инвариант №8) · арка «качество-первым» D26→D38.5 закрыта · **арх-ресет D39 исполнен ЦЕЛИКОМ** (7 слоёв → программа D39.1–D39.10 → паки 11–16, D39.13–D39.24; статус слоёв — шапка-таблица `architecture/09-target-architecture.md`) · пере-прогон rerun2 прочитан (D39.20: операционка живьём, $0-резюм ×3 арма, $6.63<$15; планка ≤2 НЕ пройдена; анти-корреляция гладкость↔верность ×3 → mistral-редактор вон) · эксп ЗАКРЫТ (D39.22), итерация №2 отложена · **курс = разработка бэкенда: пак-17 «канал B вживую» ЗАКРЫТ ЦЕЛИКОМ (дизайн D39.26 · стройка D39.27 · дельта `allow` D39.28, 25.07); вокабуляр = ПАРА `violence`/`sexually-explicit`, остаток — ФАКТЫ ToS в зоне полигона (fail-closed до них)** (D39.25: в движке НЕТ понятия «18+» — generic content-labels × provider-capabilities; лейблы вне хешей · резолв до валидации · один хоп `chain[0]` · гранулярность = книга) → **КУРС АМЕНДИРОВАН ВЛАДЕЛЬЦЕМ 25.07 (D39.33): МАСШТАБ — ПОСЛЕДНИМ, после достройки алгоритмического идеала; доказательство эффективности крупных изменений — МИНИ-ПРОГОНЫ ~10 глав (детерминированные чекеры и скан остатка первыми, судья — только когда от него зависит решение).** **ОЧЕРЕДЬ (D39.40/51/53, всё до неё исполнено):** пак-19 «голос/состояние» → ХОЛОДНЫЙ мини-прогон (пустой сид, отдельная БД) → пак-21 «чекеры» → свип гипотез → добор идеала (4 вопроса границы D39.33 у владельца) → МАСШТАБ целой книги (7,78 млн симв., ~2284 раздела) → пилот Ф2.5. Хроника треков A/B, exp15/16 и стройки паков — архивы ниже + D-лог. +> - **Фазы/курс:** Ф0 ✅ · Ф1-инфра ✅ (D20–D28; golden = инвариант №8) · арка «качество-первым» D26→D38.5 закрыта · **арх-ресет D39 исполнен ЦЕЛИКОМ** (7 слоёв → программа D39.1–D39.10 → паки 11–16, D39.13–D39.24; статус слоёв — шапка-таблица `architecture/09-target-architecture.md`) · пере-прогон rerun2 прочитан (D39.20: операционка живьём, $0-резюм ×3 арма, $6.63<$15; планка ≤2 НЕ пройдена; анти-корреляция гладкость↔верность ×3 → mistral-редактор вон) · эксп ЗАКРЫТ (D39.22), итерация №2 отложена · **курс = разработка бэкенда: пак-17 «канал B вживую» ЗАКРЫТ ЦЕЛИКОМ (дизайн D39.26 · стройка D39.27 · дельта `allow` D39.28, 25.07); вокабуляр = ПАРА `violence`/`sexually-explicit`, остаток — ФАКТЫ ToS в зоне полигона (fail-closed до них)** (D39.25: в движке НЕТ понятия «18+» — generic content-labels × provider-capabilities; лейблы вне хешей · резолв до валидации · один хоп `chain[0]` · гранулярность = книга) → **КУРС АМЕНДИРОВАН ВЛАДЕЛЬЦЕМ 25.07 (D39.33): МАСШТАБ — ПОСЛЕДНИМ, после достройки алгоритмического идеала; доказательство эффективности крупных изменений — МИНИ-ПРОГОНЫ ~10 глав (детерминированные чекеры и скан остатка первыми, судья — только когда от него зависит решение).** **ОЧЕРЕДЬ (D39.40/51/53, актуализация 31.07):** пак-19 «голос/состояние» ЗАКРЫТ ЦЕЛИКОМ (D39.54–56, вкл. малую пачку 14а+14б) → ХОЛОДНЫЙ мини-прогон (пустой сид, отдельная БД; + первое измерение осей голоса `gates.voice`) → пак-21 «чекеры» → свип гипотез → добор идеала (4 вопроса границы D39.33 у владельца) → МАСШТАБ целой книги (7,78 млн симв., ~2284 раздела) → пилот Ф2.5. Хроника треков A/B, exp15/16 и стройки паков — архивы ниже + D-лог. > - **Стек:** draft deepseek-v4-flash (thinking ON, +банкнота `translator-banknote.md`) → **терминолог** (роль пар-пака, deepseek-v4-flash, батчи, языковой экран `target_script` — D39.42/45/51/53) → **editor deepseek-v4-pro БИЛИНГВ ИНТЕРИМ (D39.22; glm-5 резерв)**, промпт v3-discourse, инъекция двухсекционная (D39.45) → судья gemini-3.1-pro-preview (Ф2, полигон); канал B Mistral+grok; 7 ключей; ~$0.85/ранобэ (D30.4). > - **ЕДИНЫЙ БЭКЛОГ — секция «Бэклог» ниже (введён 26.07 по решению владельца: одна таблица вместо наслоений; ревью-пасс оркестратора сверил её с коммитами и обсуждённым, добавил 7 строк). Норма прежняя: каждая петля обязана иметь диспозицию (решено / отложено-с-записью / отклонено); ведёт оркестратор. Историческая развёртка петель до 26.07 — в git-истории этого файла и в D-логе.** > - Архивы хроники: `archive/PROGRESS-2026-07-04-10.md` (D31) · `archive/PROGRESS-2026-07-10-13.md` (D39.6-гигиена) · `archive/PROGRESS-2026-07-13-25.md` (слайс 25.07: стройка паков 11–16, rerun2). Записи ниже — живой хвост. @@ -27,9 +27,7 @@ | 12 | `postcheck_gate` в hard-gate — требует пере-замера recall (recall чекеров теперь измерен и низок) | владелец | когда-нибудь (после пака-21) | отдельное решение | D39.34(2), D24.4+D28.1, D39.39 | | 13 | Включение платной петли ремонта + per-class исходы ремонта (схемное решение принимается вместе) — вернуться при расширении классов детекторов и ненулевом остатке | владелец | когда-нибудь | отдельное решение | D39.38, D39.39(а), CURRENT-STATE | | 13а | Веса рубрики K1–K12 не выбраны (Q1: «по данным фазы B»; фаза B прошла на BWS без весов — выбор нужен к агрегатной приёмке качества книги) | владелец | когда-нибудь (к пилоту) | Ф2.5 / отдельное решение | D39.44 Q1, D39.46 | -| **— ПАК-19 (текущий шаг очереди) —** | | | | | | -| 14 | Пак-19 «голос/состояние» (D21, урезанный скоуп) — выдан, первый в очереди D39.53; малая пачка 14а+14б ЗАКРЫТА D39.54 | бэкенд | блокер-очереди | пак-19 | D39.38, D39.40, D39.53 | -| 15 | Нарративная гендер-аннотация (since_ch-смены + перспектива, не булево поле) — вход пака-19 | бэкенд | блокер-очереди | пак-19 | D39.34(8) | +| 13б | Шипить ли `configs/langpacks/zh/speech-cue.txt` (формат+загрузчик готовы; байты двигают `LangpackVersion` = перекупка ОБЕИХ волн всех zh-книг; пока не шипнут — `SourceReplies`=0, знаменатель не измерен) | владелец | скоро (к холодному мини-прогону) | отдельное решение | D39.56, PACK19_BUILD §7.2 | | **— ХОЛОДНЫЙ МИНИ-ПРОГОН (после пака-19) —** | | | | | | | 16 | Холодный старт с пустым банком — единственная неизмеренная цифра цены/покрытия | бэкенд/полигон | блокер-очереди | холодный мини-прогон | D39.50 п.7, D39.51, POLYGON_PREMEASURE §5 | | 17 | Совместный recall майнер∪банкнота против сида живьём (замер 1 пакета-8 оказался неизмерим — цензура черновиков) | полигон | блокер-очереди | холодный мини-прогон | D39.48, D39.51, POLYGON_PREMEASURE §1.2 | @@ -74,6 +72,7 @@ | 48а | Серийный шаринг банка — РЕАЛИЗАЦИЯ (экспорт подписанного глоссария как сид-YAML следующей книги серии; спойлер-окна плющатся в since_ch=0); контракт round-trip держится тестом, самой фичи нет | бэкенд | когда-нибудь (перед второй книгой серии) | отдельное решение | D39.42 задел (а), PACK20_BANK_BUILD З-а | | 48б | Импорт «оригинал + чужой перевод» как режим данных терминолога (target-агностичный вход есть контрактом; сам режим + провенанс источника + юр-сторона в юр-пакете) | бэкенд/владелец | когда-нибудь | отдельное решение + Ф2.5 (юр) | D39.42 задел (б), PACK20_BANK_BUILD З-б | | 49 | Этапы Б+В спеки D15.2 (content-addressed resume / `guard_hash` — D39.31 сознательно не строил) | бэкенд | когда-нибудь | отдельное решение | D39.34(4), D33 п.5 | +| 49а | ALTER-шаги миграций v8–v14 не идемпотентны вопреки шапке `migrate.go:9-11` (полу-применённая БД не сходится; счётчик версий скрывает) — находка критика полноты пака-19 | бэкенд | когда-нибудь | отдельное решение | D39.56, PACK19_BUILD §6.9 | | 50 | F3-остаток идемпотентности | бэкенд | когда-нибудь | отдельное решение | D39.34(4) | | 51 | `human_override` LOCK (D29 п.1г, в коде 0 вхождений) | бэкенд | когда-нибудь | отдельное решение | D39.34(4) | | 52 | Ф2-гейты вне идеала: морфо-гейты канцелярита + L1-лемматизация (python-сайдкар) · полный OpenCC · опциональный пре-перевод гейт | бэкенд | когда-нибудь | отдельное решение | D39.34(4) | @@ -99,6 +98,10 @@ | 69 | Gemini API «под-18» обязательство на конечный продукт (независимо от лейблов) | владелец | когда-нибудь | Ф3 / отдельное решение | D39.32, D39.34(7) | | 70 | Action-security gate перед выдачей tools/webfetch (D25 п.5) | бэкенд | когда-нибудь | Ф3 | D39.34(7) | | 71 | Планы research/22: epub-tag-rewrite · Q7-леджер | бэкенд | когда-нибудь | Ф3 | D39.34(7) | +## Оркестратор №9 — СТРОЙКА ПАКА-19 ПРИНЯТА И ЗАЛЕНДЕНА (D39.56), ПАК-19 ЗАКРЫТ ЦЕЛИКОМ, 31.07 + +Фаза 2 сдана и принята короткой приёмкой (норма 30.07): сьют `-race` перегнан — зелёный; `repin.go`/`snapshot.go`/golden не тронуты (запреты (д)/(и) держатся); мутации сессии 23/23. Построено: таблицы v13/v14 + условный фолд (ни байта при выключенной инъекции) · ты/вы-флаггер A–C + D-индикатор · флаггер спойлер-утечки · данные V5 (5 категорий `target-ru` + опциональный `speech-cue`). Инъекция голоса — на бумаге до полигон-эксперимента (D21 п.2). Ратифицированное отклонение: атрибуция целевая, не исходная. Бэклог: 14/15 закрыты; +13б (`speech-cue.txt` — владельцу, к холодному прогону), +49а (миграции неидемпотентны). Очередь: ХОЛОДНЫЙ мини-прогон — теперь несёт и первое измерение осей голоса (`gates.voice` ON, строка 24). Отчёт: `archive/reports/PACK19_BUILD_2026-07-30.md` (ревью-шапка). + ## Оркестратор №9 — ДИЗАЙН ПАКА-19 РАТИФИЦИРОВАН (D39.55), стройка ждёт «идём» владельца, 30.07 Фаза 1 принята: 82 клейма — 71 CONFIRMED / 9 PARTIAL / 2 REFUTED (полнота, не факты). Ратифицировано: типы записей банка + условный фолд в `memory_version` · CONFIRMED-only голос/обращения · свой гейт `gates.voice` без фолда · reveal сокращён до флаггера утечки (амендмент D21 п.3) · слот инъекции не выбран (амендмент D21 п.1г, решит полигон) · D21 п.8 model-bound. Девять новых «ровно так» фазы 2 — в D39.55. Отчёт: `archive/reports/PACK19_DESIGN_2026-07-30.md`. diff --git a/docs/architecture/05-decisions-log.md b/docs/architecture/05-decisions-log.md index 92cbb0d6..114c959a 100644 --- a/docs/architecture/05-decisions-log.md +++ b/docs/architecture/05-decisions-log.md @@ -1176,3 +1176,7 @@ API-529-долг закрыт: 8-осевой refute-by-default воркфлоу ## D39.55 — ДИЗАЙН ПАКА-19 «голос/состояние» РАТИФИЦИРОВАН С ПОПРАВКАМИ (фаза 1, $0, стройка после «идём» владельца): три сущности = ТИПЫ ЗАПИСЕЙ БАНКА с УСЛОВНЫМ фолдом в `memory_version` (наследование точечной ре-редактуры подтверждено исполнением) · голос/обращения CONFIRMED-only, порядок инъекции «обязывающее перед необязывающим» · флаггер = свой гейт `gates.voice` БЕЗ фолда (версия логируется с прогоном) · оси A–C полноценно, ось D (регистр пары) отдельным счётчиком-индикатором · reveal-скоуп СОКРАЩЁН до флаггера спойлер-утечки без `reveal_ch` (амендмент ФОРМЫ D21 п.3, интент сохранён `spoilerBlocked`) · D21 п.1г амендирован: слот инъекции НЕ выбран, оба провода, решит полигон-эксперимент · D21 п.8 помечен model-bound (действующий порядок: драфт ~$0.04, редактор ~$0.07–0.10) (30.07, оркестратор №9). ✅ Приёмка: воркфлоу 7 верификаторов, 82 клейма = 71 CONFIRMED / 9 PARTIAL / 2 REFUTED (оба — полнота отчёта; поправки фактов — в ревью-шапке отчёта). Диспозиции пингов: (1) сокращение reveal ратифицировано; (2) `glossary.speech` → deprecated (журнал-строки вместо статической матрицы), `first_person` → переиспользуется полем профиля; (3) цифра D21 п.8 контрактно привязана к модели; (4) «драфт-слот дефолт» снят; (5) структурная неполнота оси D зафиксирована ДО фазы 2. **Новые «РОВНО ТАК» фазы 2 (из проб верификации):** (а) бюджет инъекции входит в денежный потолок/reserve — дозакрыть в отчёте стройки; (б) критик полноты в отчёте фазы 2 обязателен; (в) `Materialize` получает фильтр типа записи — строки голоса/обращений НЕ попадают в AC-матчер/глоссарную инъекцию как обычные строки (`memory.go:264-270` матчит всё с непустым Dst); (г) условный voice-тег копирует порядок «исключение скоупом ДО подсчёта условия» (`memory.go:352-357`); (д) `voiceSnap`/отдельный ключ снапшота — ЗАПРЕЩЁН (ломает `repin.go:72`); (е) `address_pair` хранится по стабильному ключу (src+sense), не по autoincrement `glossary.id`; (ж) sticky-carry, отклонённый окном, добавляется в `Selection.Rejected` (`memory.go:495-497` — сейчас молчаливый дроп); (з) ВКЛЮЧЕНИЕ инъекции = полный `--resnapshot` книги через consent-гейт, bank-only-класс на самом флипе не ожидается (omitempty защищает только добавление поля, не флип 0→300); (и) v1 переиспользует `GlossaryTokenBudget` в сигнатуре отбора — отдельный voice-бюджет потребовал бы зеркала в `repin.go:186`. Опционально фазе 2: линт «gap в окнах одного src» (сид грузится молча с дырой). Отчёт: `archive/reports/PACK19_DESIGN_2026-07-30.md` (ревью-шапка). + +## D39.56 — СТРОЙКА ПАКА-19 ПРИНЯТА И ЗАЛЕНДЕНА ($0, +672/−71, 16+7 файлов; ПАК-19 ЗАКРЫТ ЦЕЛИКОМ): голос/обращения = таблицы v13/v14 с УСЛОВНЫМ фолдом в `memory_version` (при выключенной инъекции не пишется НИ БАЙТА — golden байт-идентичен, ни одна книга не двигается) · ты/вы-флаггер осей A–C + D-индикатор в `gates.voice` (версия не фолдится, едет с прогоном) · флаггер спойлер-утечки поверх `Selection.Rejected` (вне гейта — стоячий механизм C1) · девять «ровно так» D39.55 исполнены, запреты (д)/(и) держатся (`repin.go`/`snapshot.go` не тронуты ни строкой) · ратифицированное отклонение: атрибуция ЦЕЛЕВАЯ, не исходная (выравнивания src↔tgt в движке нет; целевой хвост атрибуции матчится банком через `speech_verb`+`decl`) · `speech-cue.txt` zh НЕ шипнут (байты двигают `LangpackVersion` = перекупка обеих волн всех zh-книг — решение владельца, бэклог 13б) (31.07, оркестратор №9). ✅ + +Приёмка короткая (норма 30.07): сьют `-race` перегнан оркестратором — зелёный; vet/gofmt чисто; диффы repin/snapshot/testdata пусты; записка D21.10 сверена (переписана в «BUILT», стабильный ключ и заморозка в ней). Мутации сессии 23/23 красные (две выжившие первого круга = дыры тестов, закрыты с премис-проверками). Критик полноты честен: оси до холодного прогона дают 0 срабатываний («не измерено», не «чисто») · ось A может ловить цитату · ось B precision-over-recall · качество осей = функция полноты `decl` сида · SourceReplies без выравнивания не сопоставим с Replies. Хвосты: бэклог 13б (`speech-cue.txt` — владельцу) · 49а (ALTER-миграции v8–v14 не идемпотентны вопреки шапке `migrate.go:9-11`) · golden-счётчики голоса — кандидат следующего ратифицированного пере-капчера. Первое измерение = `gates.voice` ON на холодном мини-прогоне (бэклог 24). Отчёт: `archive/reports/PACK19_BUILD_2026-07-30.md` (ревью-шапка). diff --git a/docs/archive/reports/PACK19_BUILD_2026-07-30.md b/docs/archive/reports/PACK19_BUILD_2026-07-30.md new file mode 100644 index 00000000..f34543f4 --- /dev/null +++ b/docs/archive/reports/PACK19_BUILD_2026-07-30.md @@ -0,0 +1,172 @@ +# ПАК-19 «голос и состояние» — ФАЗА 2 (стройка). Отчёт бэкенд-сессии, 2026-07-30 + +**Аддендумы получены релеем:** директива фазы 2 (оркестратор №9, 30.07) + D39.55 + ревью-шапка `PACK19_DESIGN_2026-07-30.md`. Сессия НЕ коммитит; `git mv` не было. + +> **Ревью-шапка (оркестратор №9, 31.07): ПРИНЯТО И ЗАЛЕНДЕНО, D39.56.** Короткая приёмка по норме 30.07: +> полный сьют `-race -count=1` перегнан оркестратором — зелёный, vet чист; `repin.go`/`snapshot.go`/`testdata` +> не тронуты ни строкой (диффы пусты — запреты (д)/(и) и «golden байт-идентичен» подтверждены исполнением); +> записка D21.10 в `migrate.go` сверена — фиксирует стабильный ключ `(src, sense)` и неснятую заморозку. +> Диспозиции пингов §7: (1) ЦЕЛЕВАЯ атрибуция принята — выравнивания src↔tgt в движке нет, скоуп не менялся; +> (2) `speech-cue.txt` zh НЕ шипить до решения владельца — бэклог 13б; (3) golden не расширять — верно по +> директиве, счётчики голоса = кандидат следующего ратифицированного пере-капчера; (4) флаггер утечки вне +> гейта — принят с записью (стоячий механизм C1, `retrieval_state` вне хешей); (5) записка сверена. Находка +> критика полноты §6.9 (ALTER-миграции не идемпотентны вопреки шапке) — бэклог 49а. Первое измерение осей — +> включение `gates.voice` на холодном мини-прогоне (бэклог 24). + +**Гейты:** `go build` ✅ · `go vet ./...` ✅ · `gofmt -l` чист по всем моим файлам (флагнут только `internal/llm/llm.go` — предсуществующий нит бэклога-53, мной не тронут) · **полный сьют `-race -count=1` зелёный** · **golden: capture байт-идентичен** (`git diff --stat testdata/` пуст) · **мутации 23/23 красные** · диффстат **+672/−71, 16 файлов + 7 новых**. + +--- + +## 1. Эхо-блок + +1. Шесть пунктов директивы построены; инъекции нет — кода за §3.V3 не написано. +2. Три сущности = отдельные таблицы, фолд — внутрь `memory_version` (запрет (д) соблюдён). +3. Фолд условный и сегодня НИКОГДА не пишется ⇒ хеш каждой книги байт-идентичен, golden пуст. +4. (в) закрыт компиляцией, а не фильтром: voice/address — отдельные Go-типы. +5. Утечка считается только по несработавшему-по-окну при СРАБОТАВШЕМ ключе; sticky-реджекты (ж) в счёт не входят. +6. Регистр — абстрактный (`informal`/`formal`), поверхности — целевые данные. +7. `gates.voice` не фолдится; `VoiceCheckVersion` едет с прогоном и в отчёт. +8. `speech-cue.txt` для zh НЕ шипнут — его байты двигают `LangpackVersion`. Решение владельцу (§7). +9. Одно отклонение от буквы дизайна (механизм атрибуции — целевой, не исходный) — не молчу, §7 пинг 1. +10. Не делаю: инъекцию · Annotator · слой-2 · ru-target долг · платные прогоны · коммиты. + +--- + +## 2. Что построено (по пунктам директивы) + +| # | Пункт | Как сделано | Улика | +|---|---|---|---| +| 1 | Схема трёх типов + сид + одна транзакция + условный фолд | миграция v13 (`voice_profiles`, `address_pairs`), v14 (4 колонки `retrieval_state`); `ReplaceBank` заменяет ВСЕ три типа в одной tx; `BankInput`/`MaterializeBank`/`ComputeVersionScopedIn` | `store/migrate.go` v13/v14 · `store/glossary.go:ReplaceBank` · `membank/memory.go:ComputeVersionScopedIn` | +| 1а | `speech` → deprecated, `first_person` → поле профиля | колонка помечена в схеме; `self_ref` живёт в `voice_profiles` | `migrate.go:184` · `store/voice.go` | +| 2 | Флаггер спойлер-утечки | `Bank.SpoilerLeaks(rejected, output)` поверх `Selection.Rejected` + `dstFormPresent`; НОЛЬ схемных изменений | `membank/mempostcheck.go` | +| 3 | Ты/вы-флаггер | `internal/checks/voice.go`, оси A–C в `Total()`, ось D — отдельное поле; свой гейт `gates.voice`, версия не фолдится | `checks/voice.go` · `config/pipeline.go:VoiceGate` | +| 4 | Данные V5 | 5 новых целевых категорий в `target-ru.txt`; опциональный ИСХОДНЫЙ `speech-cue.txt` (формат+загрузчик+тест) | `lang/data/target-ru.txt` · `lang/langpack.go:parseSpeechCue` | +| 5 | Инъекция не проводится | `BankInput.InjectVoice` не имеет конфиг-ключа и всегда `false` | `pipeline/seeding.go` (комментарий у `bankIn`) | +| 6 | Фикс-строка D39.54 | комментарий поля `SHA256` → «canonical (comment-stripped)» | `pipeline/render.go` | + +**Что НЕ построено сознательно:** колонка `reveal_ch` и фаза на алиасах (D39.55 сократил скоуп; записка D21.10 в `migrate.go` переписана и объясняет, почему две строки с непересекающимися окнами уже выражают pre/post). + +--- + +## 3. Девять «РОВНО ТАК» — исполнение + +| | Требование | Исполнено | Улика / мутация | +|---|---|---|---| +| (а) | бюджет инъекции в потолке/reserve | §5 отдельным абзацем | — | +| (б) | критик полноты | §6 отдельной секцией | — | +| (в) | `Materialize` не пускает голос в AC/инъекцию | **сильнее фильтра — компиляцией**: `store.VoiceProfile`/`AddressPair` не присваиваемы в `[]store.GlossaryEntry`, матчер видит только `BankInput.Rows` | M23 (искусственно вливаю профили в rows) → красный `TestVoiceProfileNeverReachesTheMatcherOrTheInjection` | +| (г) | порядок «исключение скоупом ДО подсчёта условия» | фолд собирает `inScope` в строчном цикле, затем voice-цикл фильтрует ПЕРЕД `hasVoice = true` | M3 (переставить) → красный `TestVoiceFoldRespectsTheMinedScopeBeforeCountingIt` | +| (д) | свой ключ снапшота запрещён | ни одного нового ключа payload; `repin.go` не тронут ни строкой | `git diff` по `repin.go` пуст; golden payload не двигается | +| (е) | `address_pair` по стабильному ключу | колонки `speaker_src/speaker_sense/addressee_src/addressee_sense`; `glossary.id` нигде не участвует | `store/voice.go`, `migrate.go` v13 | +| (ж) | sticky-carry, отклонённый окном → `Rejected` | одна строка в `Select` | M4 → красный `TestStickyCarryBlockedByWindowIsRecordedAsRejected` | +| (з) | включение инъекции = полный `--resnapshot` | `InjectVoice` входит в `memory_version` ⇒ флип двигает ОБА wave-снапшота; bank-only на флипе не обещаю | M1/M2 → красные `TestVoiceRowsDoNotMoveTheBankVersionWhileUninjected` / `TestInjectedVoiceRowsMoveTheBankVersion` | +| (и) | v1 переиспользует `GlossaryTokenBudget` | отдельного voice-бюджета не заведено; `repin.go:186` не тронут | — | +| опц. | линт «gap в окнах одного src» | `VoiceWindowGaps`, WARN при сиде, никогда не фатал | `TestUnknownVoiceCharactersAndWindowGaps` | + +**Уточнение по (ж), которое важнее самой правки.** Ветка недостижима через прод-драйвер: `precomputeSticky` сбрасывает окно на каждой границе главы (`wave.go:82-85`), поэтому carry всегда приходит в ТУ ЖЕ главу, где сработал, и `spoilerBlocked(e, chapter)` даёт тот же ответ. То есть это починка на границе ЭКСПОРТИРОВАННОГО API, а не смена поведения — и именно поэтому golden остался байт-идентичным. Тест бьёт по `Select` напрямую. + +--- + +## 4. Найдено и починено ИСПОЛНЕНИЕМ (то, чего дизайн не видел) + +1. **Отсчёт исходных реплик двоил.** `“来吧。”方源说道。他走了。“不。”` давал 2 атрибутированные реплики на одном кие: окно «до открывающей кавычки» дотягивалось до кия ПРЕДЫДУЩЕЙ реплики. Исправлено потреблением: кий атрибутирует не более одной реплики (`countSourceReplies`, `consumed`). Мутация M13 возвращает дефект → красный. +2. **Две мои же мутации выжили в первом круге** — обе оказались реальными дырами тестов, не формальностью: + - **M12** (снять предусловие «на сцене ровно двое»): мой «третий персонаж» стоял В АТРИБУЦИИ, поэтому кейс на самом деле проверял неоднозначную атрибуцию, а не эвристику адресата. Перенёс третье имя в НАРРАТИВ + добавил утверждение `Attributed == 1`, чтобы кейс не мог снова уехать не на то правило. + - **M14** (убрать инертный путь без T/V-данных): мой «инертный» кейс не имел и маркеров реплик, так что T/V-гард был замаскирован гардом сегментации. Добавлен кейс «цель РАЗМЕЧАЕТ диалог, но T/V-различения не имеет» (en-подобная цель) — он изолирует именно T/V-гард. + Обе дыры закрыты, второй круг: **23/23 красные**. +3. **Тест апгрейда схемы был бы вакуумным** без премис-проверки: добавил два `SELECT`, которые ОБЯЗАНЫ упасть на старой БД до `Open` — иначе тест «апгрейда» ничего не доказывает. + +--- + +## 5. Бюджет инъекции и денежный потолок — закрытие REFUTED-1 (ровно так (а)) + +**Сегодня вопрос закрыт ПО ПОСТРОЕНИЮ, и это надо сказать прямо, а не выдать за проектную заслугу:** инъекции нет, `InjectVoice` не имеет конфиг-ключа, ни один байт голоса не попадает в `msgs`, поэтому промпт-оценка не меняется, `EstimateTokens` (`render.go:349`) считает те же строки, `ledger.EstimateUSD` получает тот же `promptEst`, и `Store.Reserve` (`stagerun.go:450-452`) резервирует ту же сумму. Потолок не ослеплён, потому что резервировать нечего. + +**Что механизм обязан будет сделать, когда инъекция поедет** (записываю, чтобы полигон и будущая сессия не искали это заново): + +- Инъекция — ОТДЕЛЬНОЕ system-сообщение (`render.go:245`), и `promptEst` считается по СОБРАННЫМ `msgs`, а не по шаблону. Значит voice-блок попадёт в оценку автоматически, БЕЗ правки резервации — это свойство того, что оценка живёт после сборки сообщений, а не до неё. Проверяемое утверждение, не обещание: если добавить блок и не тронуть `stagerun.go`, резерв вырастет. +- Ослепить потолок может ровно один сценарий: блок, добавленный в `msgs` ПОСЛЕ вычисления `promptEst`. В текущем коде такого пути нет (сборка → хеш → оценка → резерв, один порядок в `stagerun.go`), и сохранить этот порядок — обязательное «ровно так» пака инъекции. +- Порядок величины (пере-мер §V6 дизайна по действующему прайсу): драфт-слот ≈ **$0.042/книга-500** при 300 ток./чанк и $0.14/M; редакторский ≈ **$0.065** при $0.435/M. Резерв на вызов растёт на ≤$0.0002 — на три порядка ниже флора `min_max_tokens`, то есть в потолок не упирается. +- `gates.terminology.budget_usd`-подобный отдельный потолок голосу НЕ нужен: инъекция не является отдельным КЛАССОМ ВЫЗОВОВ (у неё нет своих чекпойнтов), она удорожает существующие вызовы, а их уже накрывает книжный потолок. + +--- + +## 6. Критик полноты — закрытие REFUTED-2 (ровно так (б)) + +Спрашиваю себя: что в этом паке НЕ покрыто, НЕ измерено или измерено слабее, чем звучит. + +1. **Оси B и D держатся на данных, которых на стенде нет.** Профилей и реестра ни у одной стендовой книги нет, `gates.voice` выключен, `speech-cue.txt` не шипнут. Значит на реальном тексте флаггер сегодня даёт ровно ноль срабатываний — и это НЕ «чисто», а «не измерено». Первое настоящее измерение — холодный мини-прогон (бэклог-24 уже держит строку «срабатывания голос-флаггера»); до него точность осей не известна. +2. **Ось A может ловить цитату.** Реплика, внутри которой персонаж ЦИТИРУЕТ чужие слова с другим регистром, засчитается противоречием. Вето по множественности этого не снимает. Класс не закрыт и не обойдён — назван. +3. **Ось B узка по построению.** Она срабатывает только когда в реплике есть ОБЩЕЕ местоимение и нет заявленного самоназвания. Реплика, где служанка не называет себя вовсе, — молчание, а не сигнал. Это осознанный precision-over-recall, но «доля сохранённых маркеров» из research/15 этим НЕ меряется. +4. **Атрибуция ловит только ту форму имени, что есть в `decl`.** Мой же тест это и показал: без «Фан Юаню» неоднозначная атрибуция читается как однозначная. Значит качество осей B/C/D — функция ПОЛНОТЫ склонений в сиде, а не только кода. Для полигона это прямое следствие: неполный `decl` тихо снижает `Attributed`, а не флаги. +5. **Сегментация реплик — построчная.** Реплика, разорванная переносом строки внутри одного хода, разъедется на две; реплика без маркера (косвенная речь) не видна вовсе. +6. **`SourceReplies` меряет исходник, но сопоставить с целевым `Replies` нельзя** — выравнивания нет. Это два независимых счётчика; читать разницу как «потеряли реплики» без ручной сверки нельзя. +7. **Схема есть, потребителя-инъекции нет.** `register`, `lexicon_markers`, `exemplars`, `brightness`, `address_default`, `form`, `closeness` хранятся, валидируются и (при включённой инъекции) хешируются — но сегодня их не читает НИКТО. Это ратифицированное состояние (D21 п.2), но честное имя ему — «зарезервировано», а не «работает». +8. **Не измерено: холодный старт голоса.** Как поведёт себя флаггер на книге, где банк пуст, а профили есть, — не проверено ни фикстурой, ни прогоном. +9. **Пре-существующее, не моё, но рядом:** ALTER-шаги миграций (v8–v14) не идемпотентны, вопреки шапке `migrate.go:9-11`, которая обещает сходимость полу-применённой БД. Счётчик версий это скрывает. Мой апгрейд-тест показывает, что путь v12→v14 работает; про полу-применённый шаг не утверждаю ничего. + +--- + +## 7. Пинги оркестратору + +1. **Отклонение от буквы дизайна, названное вслух: атрибуция ЦЕЛЕВАЯ, а не исходная.** Фаза 1 набросала исходный пре-гейт «имя + 说/道 у кавычки». При стройке выяснилось то, что на бумаге не видно: кий называет говорящего в ИСХОДНИКЕ, а проверять надо реплику в ЦЕЛИ, и выравнивания src↔tgt в движке нет — перенести спикера через границу нечем. Целевой стороне выравнивание не нужно: хвост атрибуции русской тире-строки уже называет спикера словами, которые банк умеет матчить (`speech_verb` + `decl`-формы). Поэтому оси A–D работают на целевой стороне, а исходный `speech-cue.txt` построен и используется ТОЛЬКО как знаменатель `SourceReplies`. Скоуп не расширен и не сужен; изменился механизм внутри пункта. +2. **Решение владельца: шипить ли `configs/langpacks/zh/speech-cue.txt`.** Файл опциональный, формат и загрузчик готовы, но его байты входят в `Version()` пакета (`langpack.go:188-191`) ⇒ `LangpackVersion` (`snapshot.go:376`) ⇒ **пере-покупка ОБЕИХ волн всех zh-книг**. Цена ≠ ноль, польза = один знаменатель. Поэтому не шипнул: тот же аргумент, которым `terminology.txt` не шипят при совпадении с дефолтом (`langpack.go:245-247`). Пока файла нет, `SourceReplies` = 0 = «не измерено». +3. **golden сознательно НЕ расширен** новыми счётчиками. Директива требует пустой маскированный дифф; добавление полей в capture — это ратифицированный пере-капчер, которого в директиве нет. Новые счётчики закрыты сквозными тестами через настоящий драйвер. Если оркестратор хочет их в golden — это отдельная строка «сдвинутой оси». +4. **Флаггер утечки НЕ гейтится `gates.voice`** (сознательно): спойлер-окно — стоячий механизм безопасности C1, а не опциональный замер. Следствие: у книги с окнами счётчик `n_spoiler_leaks` начнёт заполняться без всякого включения. На провод и на снапшот это не влияет (`retrieval_state` пересчитывается каждый прогон), но это ИЗМЕНЕНИЕ наблюдаемого поведения без флага — называю строкой, а не прячу. +5. **Записка D21.10 в `migrate.go` переписана** (была «reserved, not code»): теперь она фиксирует, что два типа построены, третий не нужен, и почему. Это зона бэкенда, но записка была ссылкой D-лога — сверьте формулировку. + +--- + +## 8. Сдвинутые оси (п.6 «ровно так») + +Одна, и она нулевая по байтам: `memory_version` получил УСЛОВНОЕ слагаемое, которое при `InjectVoice=false` не пишется. Проверено исполнением с двух сторон — байт-идентичность (`TestVoiceRowsDoNotMoveTheBankVersionWhileUninjected`, плюс сверка с `ComputeVersion` старой формы) и golden байт-в-байт. Ни одна книга не двигается этим паком. + +Наблюдаемое поведение, изменившееся без флага: `n_spoiler_blocked` теперь считает и sticky-реджекты (ветка недостижима в проде, §3), `n_spoiler_leaks` заполняется всегда (§7 пинг 4). Оба — `retrieval_state`, самозалечивающиеся, вне хешей. + +--- + +## 9. Мутации (23/23 красные) + +| | Мутация | Убита тестом | +|---|---|---| +| M1 | фолд игнорирует условие инъекции (фолдит всегда) | `TestVoiceRowsDoNotMoveTheBankVersionWhileUninjected` | +| M2 | voice-строки не фолдятся даже при инъекции | `TestInjectedVoiceRowsMoveTheBankVersion` | +| M3 | тег ставится ДО скоуп-фильтра (нарушение (г)) | `TestVoiceFoldRespectsTheMinedScopeBeforeCountingIt` | +| M4 | sticky-реджект снова тихо дропается | `TestStickyCarryBlockedByWindowIsRecordedAsRejected` | +| M5 | утечка считает и sticky-реджекты (скоуп расширен) | `TestSpoilerLeakFlagger` | +| M6 | утечка не проверяет присутствие рендеринга | `TestSpoilerLeakFlagger` | +| M7 | ось A без вето по множественности | `TestVoiceAxisATVContradictionInOneReply` | +| M8 | ось B срабатывает при ПРИСУТСТВУЮЩЕМ самоназвании | `TestVoiceAxisBSelfReferenceFlattened` | +| M9 | ось D втянута в `Total()` | `TestVoiceAxisDIsAnIndicatorOutsideTheCount` | +| M10 | атрибуция принимает неоднозначного спикера | `TestVoiceAttributionRequiresOneSpeakerAndASpeechVerb` | +| M11 | атрибуция без глагола речи | то же | +| M12 | ось D без предусловия «на сцене ровно двое» | `TestVoiceAxisDIsAnIndicatorOutsideTheCount` | +| M13 | кий атрибутирует несколько реплик | `TestVoiceSourceReplyDenominator` | +| M14 | флаггер работает без целевых T/V-данных | `TestVoiceFlaggerIsInertWithoutTargetData` | +| M15 | цитата принимается за реплику (нет глагола/вето) | `TestVoiceQuotedRepliesNeedASpeechVerbAndRespectTheInnerVeto` | +| M16 | драйвер игнорирует гейт | `TestVoiceFlaggerIsSilentWhenTheGateIsOff` | +| M17 | гард «персонажа нет в банке» обезврежен | `TestSeedRefusesAVoiceProfileForAnUnknownCharacter` | +| M18 | пересекающиеся окна профиля принимаются | `TestBankSeedRefusesMalformedVoiceSections` | +| M19 | `ReplaceBank` не пишет пары | `TestVoiceContentRoundTripsThroughTheBankReplace` | +| M20 | проекция игнорирует окно профиля | `TestVoiceProjectionRespectsWindows` | +| M21 | `setVoiceState` не пишет в строку | `TestVoiceFlaggerReachesTheStoreAndTheReport` | +| M22 | противоречие регистров в сиде принимается | `TestBankSeedRefusesMalformedVoiceSections` | +| M23 | содержимое профиля вливается в глоссарные строки | `TestVoiceProfileNeverReachesTheMatcherOrTheInjection` | + +Харнесс: `scratchpad/pack19/mutate.py` (правка → именованный тест → восстановление файла; `.mutbak` не остались, `git status` чист). + +--- + +## 10. Тесты (26 новых `^func Test`, исполнено) + +`checks/voice_test.go` 9 · `membank/memvoice_test.go` 10 · `pipeline/voicerun_test.go` 5 · `store/voice_test.go` 2. Сквозные — через настоящий драйвер (`TranslateBook` с мок-провайдером): флаггер доезжает до `retrieval_state` и `QualityReport` и НЕ флагает юнит; гейт выключен ⇒ ноль байт; утечка всплывает без гейта; сид с профилем неизвестного персонажа останавливает прогон ДО вызовов; голос/пары round-trip через `ReplaceBank`. Схема: апгрейд БД v12→v14 с премис-проверкой + полная замена трёх типов. + +--- + +## 11. Что осталось владельцу/оркестратору + +- решение по `speech-cue.txt` для zh (§7 пинг 2); +- включение `gates.voice` на холодном мини-прогоне — первое настоящее измерение точности осей (§6 п.1); +- инъекция — отдельное решение после полигон-эксперимента D21 п.2; при включении см. §5. + +**СТОП. Не коммичу.**