Land pack nineteen build as D39.56: voice and address schema with conditional fold, tu-vous flagger axes, spoiler leak flagger, snapshots unmoved, golden byte-identical
This commit is contained in:
Claude (backend session)
parent
282b97db2d
commit
8b1d8d14b7
27 changed files with 2614 additions and 75 deletions
|
|
@ -45,6 +45,15 @@ type Checkers struct {
|
|||
innerMarker []string
|
||||
yoHomograph map[string]bool // target-general: ё↔е homograph whitelist (cheapgates yofikator)
|
||||
magnitudeStem map[string]int // target-general: ru magnitude word stem → base-10 exponent (cheapgates)
|
||||
// The pack-19 voice flagger's TARGET data (voice.go): the T/V surfaces and the plural veto over
|
||||
// them, the generic self-reference pronouns, and the reply markers this target sets speech with.
|
||||
// All NORMALIZED at compile time so they compare against normalized output directly. A target that
|
||||
// ships none of them leaves the flagger inert.
|
||||
tvInformal, tvFormal, tvPluralVeto, selfRefGeneric []string
|
||||
replyDash, replyOpen, replyClose string
|
||||
// speechCue is the SOURCE-side quoted-turn alphabet (pair langpack). nil → the source-reply
|
||||
// denominator is not measured.
|
||||
speechCue *lang.SpeechCue
|
||||
|
||||
shichenRE, ruHoursRE, chengRE, decimalFractionRE *regexp.Regexp
|
||||
qianwanOKRE, shushiwanOKRE, shushiwanFireRE *regexp.Regexp
|
||||
|
|
@ -82,6 +91,16 @@ func CompileCheckers(dc *lang.DCCheckerData, tc lang.TargetChecks) *Checkers {
|
|||
innerMarker: tc.List("inner_marker"),
|
||||
yoHomograph: listToSet(tc.List("yo_homograph")),
|
||||
magnitudeStem: listToStemExp(tc.List("magnitude_stem")),
|
||||
// pack-19 voice data. Normalized here so every comparison in voice.go is against the same form
|
||||
// the output is normalized into; the reply markers stay RAW because they are matched on the
|
||||
// un-normalized line (normalization folds the dash variants and eats the line breaks).
|
||||
tvInformal: normalizeList(tc.List("tv_informal")),
|
||||
tvFormal: normalizeList(tc.List("tv_formal")),
|
||||
tvPluralVeto: normalizeList(tc.List("tv_plural_veto")),
|
||||
selfRefGeneric: normalizeList(tc.List("self_ref_generic")),
|
||||
replyDash: first(tc.List("reply_dash")),
|
||||
replyOpen: first(tc.List("reply_open")),
|
||||
replyClose: first(tc.List("reply_close")),
|
||||
}
|
||||
if dc != nil {
|
||||
c.numeral, c.ruHours, c.registerNeg = dc.Numeral, dc.RuHours, dc.RegisterNeg
|
||||
|
|
@ -184,6 +203,42 @@ func DCCheckerData(p *lang.Pack) *lang.DCCheckerData {
|
|||
return nil
|
||||
}
|
||||
|
||||
// CompileCheckersFor is CompileCheckers over a whole pack — the form the runner uses, because the
|
||||
// pack-19 voice flagger needs the SOURCE-side speech-cue table too and a nil pack must still yield the
|
||||
// target-general checkers. Kept as a second entry point rather than a third parameter so no existing
|
||||
// caller (every checker test) has to learn about a table it does not use.
|
||||
func CompileCheckersFor(p *lang.Pack, tc lang.TargetChecks) *Checkers {
|
||||
c := CompileCheckers(DCCheckerData(p), tc)
|
||||
if p != nil {
|
||||
c.speechCue = p.SpeechCue
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// normalizeList folds a target wordlist into the matching form (NFC, lower, ё→е), dropping entries that
|
||||
// normalize to nothing. Deduped, authored order preserved.
|
||||
func normalizeList(xs []string) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, x := range xs {
|
||||
n := text.NormalizeTargetForm(x)
|
||||
if n == "" || seen[n] {
|
||||
continue
|
||||
}
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// first returns the first entry of a single-valued data category, or "" when the target states none.
|
||||
func first(xs []string) string {
|
||||
if len(xs) == 0 {
|
||||
return ""
|
||||
}
|
||||
return xs[0]
|
||||
}
|
||||
|
||||
// mustPairRE compiles a pair detection pattern by key; a missing key → nil (inert sub-checker), a malformed
|
||||
// regex → panic (a corrupt pack, not a silent no-op — the same fail-loud discipline the loader keeps).
|
||||
func mustPairRE(p map[string]string, key string) *regexp.Regexp {
|
||||
|
|
|
|||
419
backend/internal/checks/voice.go
Normal file
419
backend/internal/checks/voice.go
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
package checks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"textmachine/backend/internal/text"
|
||||
)
|
||||
|
||||
// voice.go: the deterministic $0 VOICE flagger (pack-19, ratified D39.55) — the layer-1 half of D21's
|
||||
// voice detector. Like every checker here it is PURE observability, never a disposition, and tuned
|
||||
// PRECISION over recall: a miss is silence on a real defect, a false positive is noise on good prose.
|
||||
//
|
||||
// WHAT IT CAN SEE, AND WHY THAT IS THE SHAPE IT IS. The check D21 describes — "a ты/вы flip in an
|
||||
// attributed reply against the registry of PAIRS" — needs BOTH ends of a pair, and only one end is
|
||||
// deterministically reachable without the Annotator this pack does not build (a speaker cue names the
|
||||
// speaker; the addressee is named by almost nothing). So the flagger stands on three axes that need no
|
||||
// addressee at all, and reports the fourth separately:
|
||||
//
|
||||
// A. tv_contradiction — one reply carries both a T and a V surface. Needs neither end.
|
||||
// B. self_ref_flat — an attributed speaker with a declared self-designation refers to herself with
|
||||
// a plain pronoun instead (the measured flattening class, research/15 §Пробы).
|
||||
// C. ng_lexicon — an attributed speaker says a word her profile forbids. Needs the speaker only.
|
||||
// D. pair_register — the registry check. Reported as an INDICATOR, excluded from Total(), because
|
||||
// its addressee comes from a heuristic (exactly two characters on stage), not
|
||||
// from attribution.
|
||||
//
|
||||
// ATTRIBUTION IS TARGET-SIDE, and that is a refinement of the phase-1 design (which sketched a
|
||||
// source-side 说/道 pre-gate): a source cue names a speaker in the SOURCE, while the reply to check is in
|
||||
// the TARGET, and this engine has no source↔target alignment to carry one to the other. The target side
|
||||
// needs none — the attribution tail of a Russian dash line already names the speaker in words the bank
|
||||
// can match. The source cue data is still read, for the honest denominator only (SourceReplies).
|
||||
//
|
||||
// EVERYTHING language-specific is DATA: T/V surfaces, the plural veto, the generic self-reference
|
||||
// pronouns and the reply markers are TARGET data (lang.TargetChecks); the speech cues and quote marks
|
||||
// are SOURCE data (lang.SpeechCue). A target that ships no T/V rows runs the whole flagger inert.
|
||||
|
||||
// VoiceCheckVersion versions the rules in this file. It is deliberately NOT snapshot-folded (see
|
||||
// config.VoiceGate) — it is logged with the run instead, exactly as terminologyVersion is.
|
||||
const VoiceCheckVersion = "voice-v1-tv+selfref+ng+pair-indicator"
|
||||
|
||||
// VoiceCharacter is one character as the flagger sees it: an opaque identity, the target surfaces that
|
||||
// name it in the output, and the two profile fields the checks read. Built by the driver from the bank
|
||||
// (surfaces) and the voice profiles (self-ref / ng); this package never touches the store.
|
||||
type VoiceCharacter struct {
|
||||
Key string // stable identity (src\x1fsense) — compared, never rendered into a message
|
||||
Name string // human-readable label for the detail line
|
||||
Forms []string // NORMALIZED target surfaces (base dst + decl forms)
|
||||
SelfRef string // NORMALIZED declared self-designation; "" = the profile states none
|
||||
NG []string // NORMALIZED forbidden lexemes
|
||||
}
|
||||
|
||||
// VoiceAddressPair is one ordered register record projected onto the current chapter.
|
||||
type VoiceAddressPair struct {
|
||||
Speaker, Addressee string // VoiceCharacter.Key
|
||||
Register string // "informal" | "formal" — abstract; the surfaces are target data
|
||||
}
|
||||
|
||||
// VoiceRegistry is the per-unit projection the flagger runs against.
|
||||
type VoiceRegistry struct {
|
||||
Chars []VoiceCharacter
|
||||
Pairs []VoiceAddressPair
|
||||
}
|
||||
|
||||
// HasData reports whether there is anything to check against.
|
||||
func (r VoiceRegistry) HasData() bool { return len(r.Chars) > 0 }
|
||||
|
||||
// VoiceResult is one unit's flagger outcome. Every field is omitempty, so a clean unit (and any unit of
|
||||
// a book that does not enable the gate) serialises to nothing.
|
||||
type VoiceResult struct {
|
||||
TVContradiction int `json:"tv_contradiction,omitempty"`
|
||||
SelfRefFlat int `json:"self_ref_flat,omitempty"`
|
||||
NGLexicon int `json:"ng_lexicon,omitempty"`
|
||||
// PairRegister is the axis-D INDICATOR: excluded from Total() on purpose. Its addressee is inferred
|
||||
// from "exactly two characters are on stage", which is a heuristic — mixing it into the headline
|
||||
// count would let a structurally weaker signal inflate a number the operator reads as defects.
|
||||
PairRegister int `json:"pair_register,omitempty"`
|
||||
// Replies / Attributed are the denominators. Without them a count is unreadable: 3 flags out of 4
|
||||
// attributed replies is a broken voice, 3 out of 300 is noise.
|
||||
Replies int `json:"replies,omitempty"`
|
||||
Attributed int `json:"attributed,omitempty"`
|
||||
// SourceReplies is the source-side quoted-turn count (pair data). 0 when the source ships no
|
||||
// speech-cue file — which is "not measured", not "none found".
|
||||
SourceReplies int `json:"source_replies,omitempty"`
|
||||
Detail []string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Total is the flagged count that reaches n_voice_flags: axes A–C only (see PairRegister).
|
||||
func (v VoiceResult) Total() int { return v.TVContradiction + v.SelfRefFlat + v.NGLexicon }
|
||||
|
||||
// voiceReply is one segmented turn: what was said, and the attribution around it.
|
||||
type voiceReply struct {
|
||||
speech string
|
||||
attr string
|
||||
}
|
||||
|
||||
// RunVoiceChecks runs the flagger over one unit's SOURCE and FINAL text against the projected registry.
|
||||
// Pure and deterministic (sorted detail, no map iteration in output). A nil spec, an empty registry or a
|
||||
// target with no T/V data yields a zero result — the inert path a pair without the distinction takes.
|
||||
func RunVoiceChecks(source, final string, reg VoiceRegistry, c *Checkers) VoiceResult {
|
||||
var res VoiceResult
|
||||
if c == nil {
|
||||
return res
|
||||
}
|
||||
res.SourceReplies = c.countSourceReplies(source)
|
||||
if !reg.HasData() || (len(c.tvInformal) == 0 && len(c.tvFormal) == 0) {
|
||||
return res
|
||||
}
|
||||
replies := c.splitReplies(final)
|
||||
res.Replies = len(replies)
|
||||
if len(replies) == 0 {
|
||||
return res
|
||||
}
|
||||
// Who is on stage at all — the basis of the axis-D addressee heuristic. Computed once over the whole
|
||||
// unit, because a speaker named in one reply's attribution is still present for the next one.
|
||||
onStage := reg.present(final)
|
||||
|
||||
detail := map[string]bool{}
|
||||
for _, rp := range replies {
|
||||
speech := []rune(text.NormalizeTargetForm(rp.speech))
|
||||
if len(speech) == 0 {
|
||||
continue
|
||||
}
|
||||
hasT := anyWholeWord(speech, c.tvInformal)
|
||||
hasV := anyWholeWord(speech, c.tvFormal)
|
||||
// Axis A. «Ты иди, а вы оба ждите» is correct Russian addressing two parties, so a plural marker
|
||||
// vetoes the contradiction — the same shape inner_marker has over speech_verb.
|
||||
if hasT && hasV && !anyWholeWord(speech, c.tvPluralVeto) {
|
||||
res.TVContradiction++
|
||||
detail[fmt.Sprintf("voice: one reply mixes an informal and a formal address: %s", previewRunes(speech))] = true
|
||||
}
|
||||
sp, ok := reg.speakerOf(rp.attr, c)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
res.Attributed++
|
||||
// Axis B. The evidence is that she referred to herself AT ALL (a generic pronoun) while her
|
||||
// declared designation is absent — not that she used a pronoun, which everyone does.
|
||||
if sp.SelfRef != "" && anyWholeWord(speech, c.selfRefGeneric) && !wholeWord(speech, sp.SelfRef) {
|
||||
res.SelfRefFlat++
|
||||
detail[fmt.Sprintf("voice: %s refers to herself plainly; the profile declares %q", sp.Name, sp.SelfRef)] = true
|
||||
}
|
||||
// Axis C.
|
||||
for _, ng := range sp.NG {
|
||||
if wholeWord(speech, ng) {
|
||||
res.NGLexicon++
|
||||
detail[fmt.Sprintf("voice: %s says %q, which the profile forbids", sp.Name, ng)] = true
|
||||
}
|
||||
}
|
||||
// Axis D (indicator). Only when the addressee is unambiguous by the two-on-stage heuristic.
|
||||
if !hasT && !hasV {
|
||||
continue
|
||||
}
|
||||
addressee, ok := onStage.other(sp.Key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
want, ok := reg.registerFor(sp.Key, addressee)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
got := registerFormal
|
||||
if hasT {
|
||||
got = registerInformal
|
||||
}
|
||||
if got != want {
|
||||
res.PairRegister++
|
||||
detail[fmt.Sprintf("voice(indicator): %s addresses the other character as %q, the registry says %q", sp.Name, got, want)] = true
|
||||
}
|
||||
}
|
||||
res.Detail = sortedKeys(detail)
|
||||
return res
|
||||
}
|
||||
|
||||
// registerInformal / registerFormal mirror the abstract vocabulary membank validates the seed against.
|
||||
// They are category names, not target words.
|
||||
const (
|
||||
registerInformal = "informal"
|
||||
registerFormal = "formal"
|
||||
)
|
||||
|
||||
// stageSet is the set of characters whose surfaces appear in a unit, in registry order.
|
||||
type stageSet []string
|
||||
|
||||
// other returns the single OTHER character on stage, and whether the "exactly two, one of them the
|
||||
// speaker" precondition of the axis-D heuristic holds. Any other cast size gives no addressee.
|
||||
func (s stageSet) other(speaker string) (string, bool) {
|
||||
if len(s) != 2 {
|
||||
return "", false
|
||||
}
|
||||
switch {
|
||||
case s[0] == speaker:
|
||||
return s[1], true
|
||||
case s[1] == speaker:
|
||||
return s[0], true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// present returns the characters whose surfaces occur in the text, in registry order (deterministic).
|
||||
func (r VoiceRegistry) present(final string) stageSet {
|
||||
norm := []rune(text.NormalizeTargetForm(final))
|
||||
var out stageSet
|
||||
for _, ch := range r.Chars {
|
||||
if anyWholeWord(norm, ch.Forms) {
|
||||
out = append(out, ch.Key)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// speakerOf resolves a reply's attribution to a character. It requires BOTH a target speech verb (so a
|
||||
// narrative sentence that merely names somebody is not an attribution) and EXACTLY ONE character
|
||||
// surface: «— …, — сказал Фан Юань Бай Нинбин» names two, and guessing which one speaks is precisely the
|
||||
// coreference this pack does not have. Ambiguity yields no speaker, which is the silent, precise answer.
|
||||
func (r VoiceRegistry) speakerOf(attr string, c *Checkers) (VoiceCharacter, bool) {
|
||||
if strings.TrimSpace(attr) == "" {
|
||||
return VoiceCharacter{}, false
|
||||
}
|
||||
low := []rune(text.NormalizeTargetForm(attr))
|
||||
if !anySubstring(low, c.speechVerb) || anySubstring(low, c.innerMarker) {
|
||||
return VoiceCharacter{}, false
|
||||
}
|
||||
var found VoiceCharacter
|
||||
n := 0
|
||||
for _, ch := range r.Chars {
|
||||
if anyWholeWord(low, ch.Forms) {
|
||||
found, n = ch, n+1
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
return VoiceCharacter{}, false
|
||||
}
|
||||
return found, true
|
||||
}
|
||||
|
||||
// registerFor returns the projected register for an ordered pair.
|
||||
func (r VoiceRegistry) registerFor(speaker, addressee string) (string, bool) {
|
||||
for _, p := range r.Pairs {
|
||||
if p.Speaker == speaker && p.Addressee == addressee {
|
||||
return p.Register, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// splitReplies segments the target text into turns using TARGET data alone.
|
||||
//
|
||||
// A dash line is one speaker's whole turn: splitting «— A, — сказал X. — B» on the marker alternates
|
||||
// speech (odd segments) and attribution (even ones), which is the Russian convention as a rule rather
|
||||
// than as a Go literal. A quoted span is a turn only when its tail carries a speech verb — a bare
|
||||
// «…» is as likely a citation or a thought, and the inner-speech veto removes the rest.
|
||||
func (c *Checkers) splitReplies(final string) []voiceReply {
|
||||
if c == nil || (c.replyDash == "" && (c.replyOpen == "" || c.replyClose == "")) {
|
||||
return nil
|
||||
}
|
||||
var out []voiceReply
|
||||
for _, raw := range strings.Split(final, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if c.replyDash != "" && strings.HasPrefix(line, c.replyDash) {
|
||||
segs := strings.Split(line, c.replyDash)
|
||||
var speech, attr []string
|
||||
for i, s := range segs {
|
||||
switch {
|
||||
case i == 0: // text before the opening marker (empty by the HasPrefix above)
|
||||
case i%2 == 1:
|
||||
speech = append(speech, s)
|
||||
default:
|
||||
attr = append(attr, s)
|
||||
}
|
||||
}
|
||||
out = append(out, voiceReply{speech: strings.Join(speech, " "), attr: strings.Join(attr, " ")})
|
||||
continue
|
||||
}
|
||||
if c.replyOpen == "" || c.replyClose == "" {
|
||||
continue
|
||||
}
|
||||
rest := line
|
||||
for {
|
||||
i := strings.Index(rest, c.replyOpen)
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
after := rest[i+len(c.replyOpen):]
|
||||
j := strings.Index(after, c.replyClose)
|
||||
if j < 0 {
|
||||
break
|
||||
}
|
||||
speech := after[:j]
|
||||
tail := after[j+len(c.replyClose):]
|
||||
// The attribution of a quoted turn is its tail up to the next quoted turn on the line.
|
||||
attr := tail
|
||||
if k := strings.Index(tail, c.replyOpen); k >= 0 {
|
||||
attr = tail[:k]
|
||||
}
|
||||
low := []rune(text.NormalizeTargetForm(attr))
|
||||
if anySubstring(low, c.speechVerb) && !anySubstring(low, c.innerMarker) {
|
||||
out = append(out, voiceReply{speech: speech, attr: attr})
|
||||
}
|
||||
rest = tail
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// countSourceReplies counts quoted turns in the SOURCE that a cue attributes — the denominator that says
|
||||
// how much direct speech the chapter had, independently of what the translation produced. 0 when the
|
||||
// source ships no speech-cue data, which the caller reads as "not measured".
|
||||
//
|
||||
// A cue attributes AT MOST ONE turn. The scan takes the cue-AFTER position first (the zh «…」X说道» shape)
|
||||
// and marks the text it consumed, so the next turn's cue-BEFORE scan cannot claim the same words again —
|
||||
// without that, `“来吧。”方源说道。他走了。“不。”` counted two attributed turns on one cue (measured while
|
||||
// building this). Both positions are accepted so a cue-before language needs no Go branch.
|
||||
func (c *Checkers) countSourceReplies(source string) int {
|
||||
if c == nil || c.speechCue == nil {
|
||||
return 0
|
||||
}
|
||||
sc := c.speechCue
|
||||
isQuote := func(r rune) bool { return sc.QuoteOpen[r] || sc.QuoteClose[r] }
|
||||
hasCue := func(s string) bool {
|
||||
for _, cue := range sc.Cues {
|
||||
if strings.Contains(s, cue) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
rs := []rune(source)
|
||||
n, consumed := 0, 0
|
||||
for i := 0; i < len(rs); i++ {
|
||||
if !sc.QuoteOpen[rs[i]] {
|
||||
continue
|
||||
}
|
||||
j := i + 1
|
||||
for ; j < len(rs) && !sc.QuoteClose[rs[j]]; j++ {
|
||||
}
|
||||
if j >= len(rs) {
|
||||
break // an unclosed quote: no turn to attribute
|
||||
}
|
||||
// Cue AFTER the closing mark, up to the next quote mark or the window edge.
|
||||
hi := j + 1
|
||||
for hi < len(rs) && hi-(j+1) < sc.Window && !isQuote(rs[hi]) {
|
||||
hi++
|
||||
}
|
||||
// Cue BEFORE the opening mark, back to the previous quote mark, the window edge, or whatever an
|
||||
// earlier turn already claimed — whichever comes first.
|
||||
lo := i
|
||||
for lo > consumed && i-lo < sc.Window && !isQuote(rs[lo-1]) {
|
||||
lo--
|
||||
}
|
||||
switch {
|
||||
case hasCue(string(rs[j+1 : hi])):
|
||||
n++
|
||||
consumed = hi
|
||||
case hasCue(string(rs[lo:i])):
|
||||
n++
|
||||
consumed = i
|
||||
}
|
||||
i = j
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// wholeWord reports whether form occurs in the normalized haystack bounded by non-word runes.
|
||||
func wholeWord(hay []rune, form string) bool {
|
||||
f := []rune(form)
|
||||
if len(f) == 0 || len(f) > len(hay) {
|
||||
return false
|
||||
}
|
||||
for i := 0; i+len(f) <= len(hay); i++ {
|
||||
if !text.RunesEqual(hay[i:i+len(f)], f) {
|
||||
continue
|
||||
}
|
||||
if (i == 0 || !isWordRune(hay[i-1])) && (i+len(f) == len(hay) || !isWordRune(hay[i+len(f)])) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func anyWholeWord(hay []rune, forms []string) bool {
|
||||
for _, f := range forms {
|
||||
if wholeWord(hay, f) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// anySubstring is the loose test used for ATTRIBUTION vocabulary (speech verbs, inner markers), matching
|
||||
// how the dialogue-dash rule already reads those same lists: a verb may carry a prefix or a suffix the
|
||||
// list does not enumerate.
|
||||
func anySubstring(hay []rune, needles []string) bool {
|
||||
s := string(hay)
|
||||
for _, n := range needles {
|
||||
if n != "" && strings.Contains(s, n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func previewRunes(rs []rune) string { return preview(string(rs)) }
|
||||
220
backend/internal/checks/voice_test.go
Normal file
220
backend/internal/checks/voice_test.go
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
package checks
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/lang"
|
||||
)
|
||||
|
||||
// voice_test.go pins the four axes AND the inert paths — the second half matters as much as the first,
|
||||
// because "a pair without T/V activates nothing without a Go edit" is the §0 promise this flagger makes.
|
||||
|
||||
func ruVoiceCheckers() *Checkers { return CompileCheckers(nil, lang.TargetChecksFor("ru")) }
|
||||
|
||||
// служанка is the worked example of research/15: a declared self-designation, a forbidden lexeme, and a
|
||||
// formal register toward her master.
|
||||
func maid() VoiceCharacter {
|
||||
return VoiceCharacter{Key: "maid", Name: "Бай Нинбин", Forms: []string{"бай нинбин", "бай нинбин"},
|
||||
SelfRef: "ваша служанка", NG: []string{"ладно"}}
|
||||
}
|
||||
|
||||
// The declined forms are what the bank actually supplies (decl), and axis-D/attribution precision
|
||||
// depends on them: without «фан юаню» the ambiguous-attribution case below would read as unambiguous.
|
||||
func master() VoiceCharacter {
|
||||
return VoiceCharacter{Key: "master", Name: "Фан Юань", Forms: []string{"фан юань", "фан юаню", "фан юаня"}}
|
||||
}
|
||||
|
||||
func twoCharRegistry() VoiceRegistry {
|
||||
return VoiceRegistry{
|
||||
Chars: []VoiceCharacter{maid(), master()},
|
||||
Pairs: []VoiceAddressPair{{Speaker: "maid", Addressee: "master", Register: "formal"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceAxisATVContradictionInOneReply(t *testing.T) {
|
||||
c := ruVoiceCheckers()
|
||||
got := RunVoiceChecks("", "— Ты придёшь, но вас я не звала, — сказала Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.TVContradiction != 1 {
|
||||
t.Fatalf("one reply carrying both registers must flag, got %+v", got)
|
||||
}
|
||||
if got.Total() == 0 {
|
||||
t.Fatal("axis A must reach the headline count")
|
||||
}
|
||||
// The plural veto: addressing one person and a group in one turn is correct Russian, not a flip.
|
||||
got = RunVoiceChecks("", "— Ты иди, а вы оба ждите, — сказала Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.TVContradiction != 0 {
|
||||
t.Fatalf("a plural-addressing marker must veto the contradiction, got %+v", got)
|
||||
}
|
||||
// One register alone is not a contradiction.
|
||||
got = RunVoiceChecks("", "— Вы придёте? — спросила Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.TVContradiction != 0 {
|
||||
t.Fatalf("a single register is no contradiction, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceAxisBSelfReferenceFlattened(t *testing.T) {
|
||||
c := ruVoiceCheckers()
|
||||
got := RunVoiceChecks("", "— Я всё сделаю, — сказала Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.SelfRefFlat != 1 {
|
||||
t.Fatalf("a plain pronoun where the profile declares a self-designation must flag, got %+v", got)
|
||||
}
|
||||
// The declared designation present → no flag, even beside a pronoun.
|
||||
got = RunVoiceChecks("", "— Ваша служанка всё сделает, я не подведу, — сказала Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.SelfRefFlat != 0 {
|
||||
t.Fatalf("the declared designation is present — nothing was flattened, got %+v", got)
|
||||
}
|
||||
// No self-reference at all → no evidence either way.
|
||||
got = RunVoiceChecks("", "— Хорошо, — сказала Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.SelfRefFlat != 0 {
|
||||
t.Fatalf("a reply with no self-reference is not evidence of flattening, got %+v", got)
|
||||
}
|
||||
// A character with no declared self_ref is never checked on this axis.
|
||||
reg := twoCharRegistry()
|
||||
reg.Chars[0].SelfRef = ""
|
||||
if got := RunVoiceChecks("", "— Я всё сделаю, — сказала Бай Нинбин.", reg, c); got.SelfRefFlat != 0 {
|
||||
t.Fatalf("no declared self_ref means no axis-B check, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceAxisCForbiddenLexeme(t *testing.T) {
|
||||
c := ruVoiceCheckers()
|
||||
got := RunVoiceChecks("", "— Ладно, ваша служанка сделает, — сказала Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.NGLexicon != 1 {
|
||||
t.Fatalf("a forbidden lexeme in an attributed reply must flag, got %+v", got)
|
||||
}
|
||||
if len(got.Detail) == 0 || !strings.Contains(strings.Join(got.Detail, "|"), "Бай Нинбин") {
|
||||
t.Fatalf("the detail must name the character, got %v", got.Detail)
|
||||
}
|
||||
// The SAME word in somebody else's reply is not this character's defect.
|
||||
got = RunVoiceChecks("", "— Ладно, — сказал Фан Юань.", twoCharRegistry(), c)
|
||||
if got.NGLexicon != 0 {
|
||||
t.Fatalf("another speaker's word must not be charged to this profile, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceAxisDIsAnIndicatorOutsideTheCount(t *testing.T) {
|
||||
c := ruVoiceCheckers()
|
||||
// Both characters are on stage, the maid speaks, the registry says she owes him the formal register,
|
||||
// and she uses the informal one.
|
||||
got := RunVoiceChecks("", "— Ты придёшь, Фан Юань? — спросила Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.PairRegister != 1 {
|
||||
t.Fatalf("a register contradicting the registry must be indicated, got %+v", got)
|
||||
}
|
||||
if got.Total() != 0 {
|
||||
t.Fatalf("axis D must stay OUT of the headline count, got total=%d (%+v)", got.Total(), got)
|
||||
}
|
||||
// Matching the registry → nothing.
|
||||
got = RunVoiceChecks("", "— Вы придёте, Фан Юань? — спросила Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.PairRegister != 0 {
|
||||
t.Fatalf("the declared register must not be indicated, got %+v", got)
|
||||
}
|
||||
// A third character on stage breaks the two-on-stage heuristic → no addressee, no indicator.
|
||||
reg := twoCharRegistry()
|
||||
reg.Chars = append(reg.Chars, VoiceCharacter{Key: "third", Name: "Третий", Forms: []string{"третий"}})
|
||||
// The third name sits in the NARRATIVE, not in the attribution, so the speaker stays unambiguous and
|
||||
// what is actually under test is the two-on-stage precondition rather than attribution ambiguity.
|
||||
got = RunVoiceChecks("", "— Ты придёшь, Фан Юань? — спросила Бай Нинбин.\nТретий молчал.", reg, c)
|
||||
if got.Attributed != 1 {
|
||||
t.Fatalf("the speaker must still be attributed — otherwise this case tests the wrong rule: %+v", got)
|
||||
}
|
||||
if got.PairRegister != 0 {
|
||||
t.Fatalf("with three characters on stage the addressee is unknown — no indicator, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceAttributionRequiresOneSpeakerAndASpeechVerb(t *testing.T) {
|
||||
c := ruVoiceCheckers()
|
||||
// No attribution at all: the reply still counts, but nothing speaker-scoped is checked.
|
||||
got := RunVoiceChecks("", "— Я всё сделаю.", twoCharRegistry(), c)
|
||||
if got.Replies != 1 || got.Attributed != 0 || got.SelfRefFlat != 0 {
|
||||
t.Fatalf("an unattributed reply is counted but not charged to anybody, got %+v", got)
|
||||
}
|
||||
// TWO characters in the attribution: guessing which one speaks is the coreference this pack lacks.
|
||||
got = RunVoiceChecks("", "— Я всё сделаю, — сказала Бай Нинбин Фан Юаню.", twoCharRegistry(), c)
|
||||
if got.Attributed != 0 {
|
||||
t.Fatalf("an ambiguous attribution must yield no speaker, got %+v", got)
|
||||
}
|
||||
// A narrative sentence that merely names somebody is not an attribution.
|
||||
got = RunVoiceChecks("", "— Я всё сделаю, — Бай Нинбин смотрела в окно.", twoCharRegistry(), c)
|
||||
if got.Attributed != 0 {
|
||||
t.Fatalf("no speech verb means no attribution, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceQuotedRepliesNeedASpeechVerbAndRespectTheInnerVeto(t *testing.T) {
|
||||
c := ruVoiceCheckers()
|
||||
got := RunVoiceChecks("", "«Я всё сделаю», — сказала Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.Replies != 1 || got.SelfRefFlat != 1 {
|
||||
t.Fatalf("a quoted turn with a speech verb is a reply, got %+v", got)
|
||||
}
|
||||
// Inner speech is a thought, not an addressed turn: no register, no self-designation duty.
|
||||
got = RunVoiceChecks("", "«Я всё сделаю», — пробормотала про себя Бай Нинбин.", twoCharRegistry(), c)
|
||||
if got.Replies != 0 {
|
||||
t.Fatalf("inner speech is not an addressed reply, got %+v", got)
|
||||
}
|
||||
// A bare citation is not a reply either.
|
||||
got = RunVoiceChecks("", "На двери висела табличка «Я всё сделаю».", twoCharRegistry(), c)
|
||||
if got.Replies != 0 {
|
||||
t.Fatalf("a citation is not a reply, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The §0 promise, executed: with no target T/V data the whole flagger is inert — no Go edit needed for a
|
||||
// pair whose target has no such distinction.
|
||||
func TestVoiceFlaggerIsInertWithoutTargetData(t *testing.T) {
|
||||
bare := CompileCheckers(nil, lang.TargetChecks{})
|
||||
got := RunVoiceChecks("", "— Ты придёшь, но вас я не звала, — сказала Бай Нинбин.", twoCharRegistry(), bare)
|
||||
if got.Total() != 0 || got.Replies != 0 || got.PairRegister != 0 {
|
||||
t.Fatalf("a target with no voice data must flag nothing, got %+v", got)
|
||||
}
|
||||
// And with data but no registry (a book that authored no profiles) — equally inert.
|
||||
if got := RunVoiceChecks("", "— Ты и вы, — сказал он.", VoiceRegistry{}, ruVoiceCheckers()); got.Total() != 0 {
|
||||
t.Fatalf("an empty registry must flag nothing, got %+v", got)
|
||||
}
|
||||
if RunVoiceChecks("", "текст", twoCharRegistry(), nil).Total() != 0 {
|
||||
t.Fatal("a nil checker spec must be inert, not a panic")
|
||||
}
|
||||
// The case that isolates the T/V guard from the segmentation guard: a target that DOES mark dialogue
|
||||
// (so replies are found and attributed) but has NO T/V distinction — an en-like target. Nothing on the
|
||||
// register axes may fire, with no Go edit anywhere.
|
||||
noTV := &Checkers{
|
||||
replyDash: "—",
|
||||
speechVerb: []string{"сказала"},
|
||||
selfRefGeneric: []string{"я"},
|
||||
}
|
||||
got = RunVoiceChecks("", "— Ты придёшь, но вас я не звала, — сказала Бай Нинбин.", twoCharRegistry(), noTV)
|
||||
if got.Total() != 0 || got.Replies != 0 || got.PairRegister != 0 {
|
||||
t.Fatalf("a target with dialogue markers but no T/V distinction must flag nothing, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The source-side denominator: read from PAIR data, and 0 (not "none") when the source ships no file.
|
||||
func TestVoiceSourceReplyDenominator(t *testing.T) {
|
||||
c := ruVoiceCheckers()
|
||||
if got := RunVoiceChecks("“来吧。”方源说道。", "", twoCharRegistry(), c); got.SourceReplies != 0 {
|
||||
t.Fatalf("without a speech-cue table the count is NOT MEASURED, got %d", got.SourceReplies)
|
||||
}
|
||||
c.speechCue = &lang.SpeechCue{
|
||||
Cues: []string{"说", "道"},
|
||||
QuoteOpen: map[rune]bool{'“': true},
|
||||
QuoteClose: map[rune]bool{'”': true},
|
||||
Window: 12,
|
||||
}
|
||||
got := RunVoiceChecks("“来吧。”方源说道。他走了。“不。”", "", twoCharRegistry(), c)
|
||||
if got.SourceReplies != 1 {
|
||||
t.Fatalf("only the cue-attributed quoted turn counts, got %d", got.SourceReplies)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceResultIsDeterministic(t *testing.T) {
|
||||
c := ruVoiceCheckers()
|
||||
const final = "— Ладно, я всё сделаю, но вас я не звала, — сказала Бай Нинбин."
|
||||
first := RunVoiceChecks("", final, twoCharRegistry(), c)
|
||||
for i := 0; i < 20; i++ {
|
||||
if got := RunVoiceChecks("", final, twoCharRegistry(), c); strings.Join(got.Detail, "|") != strings.Join(first.Detail, "|") ||
|
||||
got.Total() != first.Total() {
|
||||
t.Fatalf("run %d diverged: %+v vs %+v", i, got, first)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -251,6 +251,26 @@ type Gates struct {
|
|||
Banknote BanknoteGate `yaml:"banknote"`
|
||||
Repair RepairGate `yaml:"repair"`
|
||||
Terminology TerminologyGate `yaml:"terminology"`
|
||||
Voice VoiceGate `yaml:"voice"`
|
||||
}
|
||||
|
||||
// VoiceGate controls the deterministic $0 voice flagger (pack-19, D39.55): T/V contradictions,
|
||||
// flattened self-designations and forbidden lexemes in attributed replies, measured against the book's
|
||||
// voice profiles and address-register journal.
|
||||
//
|
||||
// Opt-in and OFF by default like every other gate here, and — like gates.terminology, unlike coverage /
|
||||
// sanitizer / banknote — deliberately NOT snapshot-folded even when on. The reason is the same one: it
|
||||
// touches neither the wire nor a checkpoint's resolved verdict. It makes no call, changes no message and
|
||||
// never becomes a disposition; its counters live in retrieval_state, which every run recomputes from the
|
||||
// stored text and which self-heals on resume. Folding it would re-bill a whole wave for switching on a
|
||||
// measurement — and folding its VERSION would re-bill every book of every pair for a rule edit, because
|
||||
// StyleCheckVersion (the sibling it might otherwise have joined) is folded unconditionally.
|
||||
//
|
||||
// The accepted cost of not folding: editing a rule shifts recorded counts between runs under one
|
||||
// snapshot. checks.VoiceCheckVersion is logged with the run and carried in the report so the numbers
|
||||
// stay attributable to the rules that produced them — the mitigation the terminologist already uses.
|
||||
type VoiceGate struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// TerminologyGate controls the TERMINOLOGIST role (pack-20, D39.42 п.1): between the draft wave and the
|
||||
|
|
|
|||
|
|
@ -107,3 +107,62 @@ sanitizer_preamble (?is)^\s*ниже\s+(?:привед[её]н|представ
|
|||
sanitizer_trailing_note (?i)^[*_>\s-]{0,4}(?:(?:примечани[ея]|заметк[аи]|комментари[йяю]|пояснени[ея]|сноск[аи])\s*:|(?:примечани[ея]|заметк[аи]|комментари[йяю])\s+(?:переводчик|редактор)[а-яё]*|прим\.\s*(?:перев|ред)\.?)
|
||||
sanitizer_edit_meta (?im)^[*_>\s-]{0,4}(?:(?:внесённ|внесен)[а-яё]+\s+правк|основн[а-яё]+\s+правк[а-яё]*(?:\s+для\s+справк[а-яё]+|\s*:)|что\s+(?:было\s+)?(?:изменен|исправлен)|список\s+(?:правок|изменений)|список\s+внесённых)
|
||||
sanitizer_invalid_sign (?i)(?:^|\P{L})[ъь][а-яё]|[ъь][ъь]
|
||||
# --- pack-19 voice flagger (T/V + attributed-reply checks) -----------------------------------------
|
||||
# The T/V distinction is a fact about the TARGET language, so it lives here: a target that ships no
|
||||
# tv_informal/tv_formal rows runs the whole voice flagger inert, with no Go edit. Surfaces are matched
|
||||
# as WHOLE WORDS on the normalized output, lower-cased; only PRONOUNS and possessives are listed —
|
||||
# verb agreement is morphology and belongs to the ru-target layer, not to a $0 flagger.
|
||||
tv_informal ты
|
||||
tv_informal тебя
|
||||
tv_informal тебе
|
||||
tv_informal тобой
|
||||
tv_informal тобою
|
||||
tv_informal твой
|
||||
tv_informal твоя
|
||||
tv_informal твоё
|
||||
tv_informal твое
|
||||
tv_informal твои
|
||||
tv_informal твоего
|
||||
tv_informal твоей
|
||||
tv_informal твоих
|
||||
tv_informal твоим
|
||||
tv_informal твою
|
||||
tv_formal вы
|
||||
tv_formal вас
|
||||
tv_formal вам
|
||||
tv_formal вами
|
||||
tv_formal ваш
|
||||
tv_formal ваша
|
||||
tv_formal ваше
|
||||
tv_formal ваши
|
||||
tv_formal вашего
|
||||
tv_formal вашей
|
||||
tv_formal ваших
|
||||
tv_formal вашим
|
||||
tv_formal вашу
|
||||
# tv_plural_veto: Russian «вы» is ALSO plural-you, so a reply addressing several people legitimately
|
||||
# mixes «ты» and «вы» («Ты иди, а вы оба ждите»). A reply carrying one of these markers is not counted
|
||||
# as a T/V contradiction — the same veto shape inner_marker has over speech_verb, and for the same
|
||||
# precision-over-recall reason: a miss here is silence on a real clash, not noise on good prose.
|
||||
tv_plural_veto оба
|
||||
tv_plural_veto обе
|
||||
tv_plural_veto вдвоём
|
||||
tv_plural_veto втроём
|
||||
tv_plural_veto все вы
|
||||
tv_plural_veto вы все
|
||||
tv_plural_veto господа
|
||||
tv_plural_veto каждый из вас
|
||||
# self_ref_generic: the PLAIN first-person surfaces. They are the evidence that a character referred to
|
||||
# herself at all — the check is «she said "я" where her profile says "ваша служанка"», i.e. the measured
|
||||
# flattening class (research/15 §Пробы, marker 奴婢), never «she said "я"» on its own.
|
||||
self_ref_generic я
|
||||
self_ref_generic меня
|
||||
self_ref_generic мне
|
||||
self_ref_generic мной
|
||||
self_ref_generic мною
|
||||
# reply segmentation: how this target sets direct speech. reply_dash opens a dash-marked turn at line
|
||||
# start; reply_open/reply_close bound a quoted turn. Data, not a Go literal, so a target with other
|
||||
# conventions (ja 「」) states its own without touching the segmentation algorithm.
|
||||
reply_dash —
|
||||
reply_open «
|
||||
reply_close »
|
||||
|
|
|
|||
|
|
@ -82,6 +82,12 @@ type Pack struct {
|
|||
// data/algorithm boundary the miner tables keep.
|
||||
Heading *HeadingRule
|
||||
|
||||
// SpeechCue is the OPTIONAL SOURCE-side direct-speech alphabet (configs/langpacks/<src>/speech-cue.txt);
|
||||
// nil when the source ships no file, and then the source-side reply count is simply 0. It is source
|
||||
// data because "a quoted turn is 「…」 with 说/道 beside it" is a fact about the source language, not
|
||||
// about the pair or the algorithm.
|
||||
SpeechCue *SpeechCue
|
||||
|
||||
// Terminology is the OPTIONAL pair sizing of the terminologist's source contexts
|
||||
// (configs/langpacks/<pair>/terminology.txt); nil when the pair ships no file. It exists because the
|
||||
// KWIC width is measured in RUNES, and N runes buy a different amount of context in every script, so
|
||||
|
|
@ -91,6 +97,16 @@ type Pack struct {
|
|||
version string
|
||||
}
|
||||
|
||||
// SpeechCue is the source language's direct-speech alphabet: the attribution cues and the quote marks
|
||||
// that bound a quoted turn, plus how far from a quote mark a cue still counts as attributing it. Data
|
||||
// only — the counting ALGORITHM lives in the pipeline, like every other table here.
|
||||
type SpeechCue struct {
|
||||
Cues []string // attribution cues in authored order (说/道/问 …)
|
||||
QuoteOpen map[rune]bool // opening quote marks
|
||||
QuoteClose map[rune]bool // closing quote marks
|
||||
Window int // runes on either side of a quote mark within which a cue attributes it
|
||||
}
|
||||
|
||||
// TerminologySizing is the pair's own "how much source context is one context". A zero field means
|
||||
// unstated and falls through to the engine default; the order lives in pipeline.terminologyOpts.
|
||||
type TerminologySizing struct {
|
||||
|
|
@ -241,6 +257,20 @@ func Load(root, sourceLang, targetLang string) (*Pack, error) {
|
|||
p.DCCheckers = dc
|
||||
}
|
||||
|
||||
// Optional per-SOURCE speech-cue alphabet (pack-19). Same optional contract as heading.txt, but read
|
||||
// from the SOURCE directory: 说/道 and 「」 are facts about the source language, not about the pair.
|
||||
if sb, ok, serr := readOptional(root, sourceLang, "speech-cue.txt"); serr != nil {
|
||||
return nil, fmt.Errorf("langpack %q speech-cue.txt: %w", pair, serr)
|
||||
} else if ok {
|
||||
h.Write([]byte("\x00" + sourceLang + "/speech-cue.txt\x00"))
|
||||
h.Write(sb)
|
||||
sc, perr := parseSpeechCue(sb)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("langpack %q speech-cue.txt: %w", pair, perr)
|
||||
}
|
||||
p.SpeechCue = sc
|
||||
}
|
||||
|
||||
// Optional per-pair TERMINOLOGIST sizing. Same contract as heading.txt: ABSENT → nil, the engine
|
||||
// defaults apply and Version() is byte-stable, so shipping the mechanism re-bills nobody. A pair whose
|
||||
// measured value equals the default deliberately ships no file — the bytes would move the pack version
|
||||
|
|
@ -588,6 +618,54 @@ func parseHeading(b []byte) (*HeadingRule, error) {
|
|||
return hr, nil
|
||||
}
|
||||
|
||||
// parseSpeechCue reads speech-cue.txt into a SpeechCue. Format: `key<TAB>value` per non-comment line,
|
||||
// keys cue | quote_open | quote_close | window. A file that ships cues but no quote marks (or the
|
||||
// reverse) can never attribute anything, so it is refused rather than loaded inert — the same
|
||||
// "never silently empty" contract the required tables keep.
|
||||
func parseSpeechCue(b []byte) (*SpeechCue, error) {
|
||||
sc := &SpeechCue{QuoteOpen: map[rune]bool{}, QuoteClose: map[rune]bool{}}
|
||||
for i, raw := range strings.Split(string(b), "\n") {
|
||||
t := strings.TrimSpace(strings.TrimRight(raw, "\r"))
|
||||
if t == "" || strings.HasPrefix(t, "#") {
|
||||
continue
|
||||
}
|
||||
f := strings.SplitN(t, "\t", 2)
|
||||
if len(f) != 2 || strings.TrimSpace(f[1]) == "" {
|
||||
return nil, fmt.Errorf("line %d: want `key<TAB>value` with a non-empty value (%q)", i+1, t)
|
||||
}
|
||||
key, val := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
|
||||
switch key {
|
||||
case "cue":
|
||||
sc.Cues = append(sc.Cues, val)
|
||||
case "quote_open", "quote_close":
|
||||
r := []rune(val)
|
||||
if len(r) != 1 {
|
||||
return nil, fmt.Errorf("line %d: %s %q must be a single rune", i+1, key, val)
|
||||
}
|
||||
if key == "quote_open" {
|
||||
sc.QuoteOpen[r[0]] = true
|
||||
} else {
|
||||
sc.QuoteClose[r[0]] = true
|
||||
}
|
||||
case "window":
|
||||
n, err := strconv.Atoi(val)
|
||||
if err != nil || n <= 0 {
|
||||
return nil, fmt.Errorf("line %d: window must be a positive integer (%q)", i+1, val)
|
||||
}
|
||||
sc.Window = n
|
||||
default:
|
||||
return nil, fmt.Errorf("line %d: unknown key %q (want cue|quote_open|quote_close|window)", i+1, key)
|
||||
}
|
||||
}
|
||||
if len(sc.Cues) == 0 || len(sc.QuoteOpen) == 0 || len(sc.QuoteClose) == 0 {
|
||||
return nil, fmt.Errorf("speech-cue needs at least one cue and both quote_open and quote_close (a half table can never attribute anything)")
|
||||
}
|
||||
if sc.Window == 0 {
|
||||
sc.Window = 12 // engine default: a name+cue sits within a dozen runes of the quote in practice
|
||||
}
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
// parseTerminology reads terminology.txt into a TerminologySizing. Format: `key<TAB>value`, keys
|
||||
// kwic_per_term | kwic_width, each optional but positive when present. Fail-loud like parseHeading; an
|
||||
// empty file is refused too, because its bytes move Version() while changing nothing.
|
||||
|
|
|
|||
|
|
@ -170,6 +170,10 @@ type Bank struct {
|
|||
// equality but STABILITY: base stays fixed as mined rows are added; only enriched moves.
|
||||
enrichedVersion string
|
||||
baseVersion string
|
||||
// voices/pairs are the pack-19 record types. They are bank CONTENT but never matchable SURFACES, so
|
||||
// they sit beside the automaton rather than inside it: nothing here can put them in keyOwners.
|
||||
voices []store.VoiceProfile
|
||||
pairs []store.AddressPair
|
||||
}
|
||||
|
||||
// PickedEntry is one selected record for a chunk with its firing key and disposition.
|
||||
|
|
@ -233,13 +237,35 @@ type Selection struct {
|
|||
func (b *Bank) Version() string { return b.enrichedVersion }
|
||||
func (b *Bank) BaseVersion() string { return b.baseVersion }
|
||||
|
||||
// Materialize builds a Bank from the book's stored glossary rows (ORDER
|
||||
// BY-stable — GlossaryForBook). Pure and deterministic. It computes memoryVersion as a
|
||||
// BankInput is everything a book's bank is materialized and hashed from: the glossary rows plus the two
|
||||
// pack-19 record types. It exists so the fold has ONE argument that can grow, and so the separation that
|
||||
// matters is carried by the type system: only Rows ever reaches the matcher, so a voice profile cannot
|
||||
// become a matchable term by anyone forgetting a filter.
|
||||
type BankInput struct {
|
||||
Rows []store.GlossaryEntry
|
||||
Voices []store.VoiceProfile
|
||||
Pairs []store.AddressPair
|
||||
// InjectVoice reports whether the voice/address rows reach the WIRE on this run. It is the CONDITION
|
||||
// of their fold: while they only feed the $0 flagger (which recomputes every run and self-heals),
|
||||
// hashing them would re-bill a book for authoring a profile that changed no byte the model sees.
|
||||
// The moment they are injected, they must fold — the D39.42 п.3 class, in the other direction.
|
||||
InjectVoice bool
|
||||
}
|
||||
|
||||
// Materialize builds a Bank from the book's stored glossary rows alone — the pre-pack-19 form, kept for
|
||||
// every caller that has no voice/address content.
|
||||
func Materialize(rows []store.GlossaryEntry, gateOn bool) *Bank {
|
||||
return MaterializeBank(BankInput{Rows: rows}, gateOn)
|
||||
}
|
||||
|
||||
// MaterializeBank builds a Bank from the book's stored bank content (ORDER BY-stable — GlossaryForBook /
|
||||
// VoiceProfilesForBook / AddressPairsForBook). Pure and deterministic. It computes memoryVersion as a
|
||||
// content hash of the frozen APPROVED rows (D8: content-hash, not a version-counter —
|
||||
// drift-proof) plus the normalization + matcher algorithm versions, so ANY change to
|
||||
// the approved glossary OR to the deterministic machinery is a loud --resnapshot (F1).
|
||||
func Materialize(rows []store.GlossaryEntry, gateOn bool) *Bank {
|
||||
b := &Bank{keyOwners: map[string][]int{}}
|
||||
func MaterializeBank(in BankInput, gateOn bool) *Bank {
|
||||
rows := in.Rows
|
||||
b := &Bank{keyOwners: map[string][]int{}, voices: in.Voices, pairs: in.Pairs}
|
||||
var allKeys []string
|
||||
seenKey := map[string]bool{}
|
||||
|
||||
|
|
@ -302,11 +328,18 @@ func Materialize(rows []store.GlossaryEntry, gateOn bool) *Bank {
|
|||
}
|
||||
sort.Strings(allKeys) // deterministic automaton construction
|
||||
b.ac = buildAC(allKeys)
|
||||
b.enrichedVersion = ComputeVersion(rows, gateOn) // all approved incl mined (the edit wave)
|
||||
b.baseVersion = ComputeVersionScoped(rows, gateOn, true) // excl Source:mined (the draft wave)
|
||||
b.enrichedVersion = ComputeVersionScopedIn(in, gateOn, false) // all approved incl mined (the edit wave)
|
||||
b.baseVersion = ComputeVersionScopedIn(in, gateOn, true) // excl Source:mined (the draft wave)
|
||||
return b
|
||||
}
|
||||
|
||||
// Voices returns the book's voice profiles (frozen for the job). Read-only: callers project them onto a
|
||||
// chapter, never mutate them.
|
||||
func (b *Bank) Voices() []store.VoiceProfile { return b.voices }
|
||||
|
||||
// Pairs returns the book's address-register journal entries (frozen for the job).
|
||||
func (b *Bank) Pairs() []store.AddressPair { return b.pairs }
|
||||
|
||||
// ComputeVersion is the F1 content-hash: the frozen rows (ORDER BY-stable) plus the
|
||||
// normalization and matcher algorithm versions. Approved-only per D8/§8 when the post-check
|
||||
// hard gate is OFF (auto/draft injected-content changes are caught at the per-chunk
|
||||
|
|
@ -330,11 +363,29 @@ func ComputeVersion(rows []store.GlossaryEntry, gateOn bool) string {
|
|||
// pre-split fold (no extra field), so the enriched hash / existing snapshots do not move; the
|
||||
// excludeMined=true variant adds a domain separator so base and enriched never collide.
|
||||
func ComputeVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool) string {
|
||||
return ComputeVersionScopedIn(BankInput{Rows: rows}, gateOn, excludeMined)
|
||||
}
|
||||
|
||||
// ComputeVersionScopedIn is ComputeVersionScoped over the whole bank input — the form that also folds the
|
||||
// pack-19 voice/address rows, CONDITIONALLY (BankInput.InjectVoice).
|
||||
//
|
||||
// The condition is the money contract of pack-19. While those rows only feed the $0 flagger they change
|
||||
// no byte the model sees, and a book must not re-pay a wave for authoring a profile; the flagger's
|
||||
// counters live in retrieval_state, which is recomputed every run and self-heals. The moment they are
|
||||
// injected they DO change the wire, and then not folding them would reopen exactly the class D39.42 п.3
|
||||
// closed — a wire change the snapshot cannot see, so projectRebill projects $0 for a real re-payment.
|
||||
// A book with no voice rows, or a run with the injection off, hashes BYTE-IDENTICALLY to before this
|
||||
// existed: nothing is written at all.
|
||||
func ComputeVersionScopedIn(in BankInput, gateOn, excludeMined bool) string {
|
||||
rows := in.Rows
|
||||
h := sha256.New()
|
||||
h.Write([]byte("tm-memory-v2\x00"))
|
||||
h.Write([]byte(text.NormVersion() + "\x00" + matchVersion + "\x00"))
|
||||
h.Write([]byte("gate:" + strconv.FormatBool(gateOn) + "\x00"))
|
||||
hasUnverified := false
|
||||
// inScope collects the characters this fold's ROW scope actually contains, so the voice/address fold
|
||||
// below can apply the SAME scope before deciding anything (see the loop at the end).
|
||||
inScope := map[[2]string]bool{}
|
||||
if excludeMined {
|
||||
h.Write([]byte("base-excl-mined\x00")) // domain separator: base ≠ enriched even over identical rows
|
||||
}
|
||||
|
|
@ -355,6 +406,7 @@ func ComputeVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool)
|
|||
if r.Status != "approved" {
|
||||
hasUnverified = true
|
||||
}
|
||||
inScope[[2]string{r.Src, r.Sense}] = true
|
||||
// A fixed, length-prefixed field layout so no content can forge a boundary.
|
||||
writeField(h, r.Status)
|
||||
writeField(h, r.Src)
|
||||
|
|
@ -385,6 +437,40 @@ func ComputeVersionScoped(rows []store.GlossaryEntry, gateOn, excludeMined bool)
|
|||
if hasUnverified {
|
||||
h.Write([]byte("editor-unverified-section-v1\x00"))
|
||||
}
|
||||
// The pack-19 record types, appended AFTER everything above so a book without them is byte-identical
|
||||
// to the pre-pack-19 hash by construction, not by argument.
|
||||
//
|
||||
// The two loops copy the order the row loop above uses and for the same reason: the SCOPE exclusion is
|
||||
// applied FIRST, and only a row that survived it may set the tag condition. A profile whose character
|
||||
// is not in this scope (its term is Source:mined and this is the base/draft fold) describes somebody
|
||||
// the wave never sees — folding it would move the draft version when a mined-only character's profile
|
||||
// is edited, which is precisely the base/enriched separation the mined filter exists to keep.
|
||||
hasVoice := false
|
||||
for _, v := range in.Voices {
|
||||
if !in.InjectVoice || !inScope[[2]string{v.Src, v.Sense}] {
|
||||
continue
|
||||
}
|
||||
hasVoice = true
|
||||
for _, f := range []string{v.Src, v.Sense, v.Register, v.SelfRef, v.AddressDefault,
|
||||
v.LexiconMarkers, v.NGLexicon, v.Exemplars, v.Brightness,
|
||||
strconv.Itoa(v.SinceCh), strconv.Itoa(v.UntilCh)} {
|
||||
writeField(h, f)
|
||||
}
|
||||
}
|
||||
for _, p := range in.Pairs {
|
||||
if !in.InjectVoice ||
|
||||
!inScope[[2]string{p.SpeakerSrc, p.SpeakerSense}] || !inScope[[2]string{p.AddresseeSrc, p.AddresseeSense}] {
|
||||
continue
|
||||
}
|
||||
hasVoice = true
|
||||
for _, f := range []string{p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense,
|
||||
p.Register, p.Form, p.Closeness, strconv.Itoa(p.SinceCh), strconv.Itoa(p.UntilCh)} {
|
||||
writeField(h, f)
|
||||
}
|
||||
}
|
||||
if hasVoice {
|
||||
h.Write([]byte("voice-address-v1\x00"))
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
|
|
@ -493,6 +579,12 @@ func (b *Bank) Select(chunk string, chapter int, stickyPrev map[string]Injection
|
|||
continue
|
||||
}
|
||||
if spoilerBlocked(e, chapter) {
|
||||
// RECORDED, not dropped (pack-19): a window-blocked carry is a spoiler reject like any other,
|
||||
// and a silent drop makes n_spoiler_blocked under-count. Unreachable through the production
|
||||
// driver — precomputeSticky resets the window at every chapter boundary, so a carry always
|
||||
// arrives at the SAME chapter it fired in and cannot have become blocked — but Select is
|
||||
// exported and the invariant that protects it lives in another package.
|
||||
sel.Rejected = append(sel.Rejected, PickedEntry{entry: e, Sticky: true, Disp: Reject})
|
||||
continue
|
||||
}
|
||||
hits[e.id] = PickedEntry{entry: e, Sticky: true, Disp: carryDisp}
|
||||
|
|
|
|||
|
|
@ -172,3 +172,53 @@ func containsWholeWord(hay, form []rune) bool {
|
|||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SpoilerLeak is one rendering the spoiler window REJECTED for this chapter that appeared in the output
|
||||
// anyway — the self-inflicted spoiler the windows exist to prevent (D21 п.3 / research/15 §2.2).
|
||||
type SpoilerLeak struct {
|
||||
Src string `json:"src"`
|
||||
Dst string `json:"dst"`
|
||||
// Since/Until are the window that excluded the row, so the operator sees at a glance whether the
|
||||
// leak is a premature reveal (before since_ch) or a stale one (after until_ch).
|
||||
SinceCh int `json:"since_ch"`
|
||||
UntilCh int `json:"until_ch"`
|
||||
}
|
||||
|
||||
// SpoilerLeaks checks the chunk's REJECTED records against the model output: the row's key fired here,
|
||||
// the window said the chapter must not know this rendering yet, and the rendering is in the output all
|
||||
// the same. It is the reveal half of D21 п.3, built on the machinery that already exists — the selection
|
||||
// records its rejects, and dstFormPresent already answers "is this rendering here" decl-aware — so it
|
||||
// needs no schema and no second matcher.
|
||||
//
|
||||
// SCOPE, ratified (D39.55): "a leak on a FIRED key". Two classes are therefore OUT, and out on purpose
|
||||
// rather than by oversight:
|
||||
//
|
||||
// - a STICKY carry rejected by the window (Sticky==true). Its key did NOT fire in this chunk, so the
|
||||
// model was not looking at the entity here; counting it would silently widen the ratified scope on
|
||||
// the back of the pack-19 fix that put those carries into Rejected at all;
|
||||
// - a SOURCE-ANCHORED reveal — an identity twist whose source surface never occurs (the text says
|
||||
// "the stranger", not the name). Nothing fires, so nothing is rejected, so there is nothing to
|
||||
// check. Closing that class needs a target-side index of post-reveal renderings, which is a
|
||||
// different mechanism and not this pack's.
|
||||
//
|
||||
// Pure and deterministic (rejected order, which Select fixes). Observability only — never a disposition.
|
||||
func (b *Bank) SpoilerLeaks(rejected []PickedEntry, output string) []SpoilerLeak {
|
||||
nout := []rune(text.NormalizeTargetForm(output))
|
||||
var out []SpoilerLeak
|
||||
for _, p := range rejected {
|
||||
if !p.valid() || p.Sticky {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(p.entry.dst) == "" {
|
||||
continue // nothing to leak
|
||||
}
|
||||
if !dstFormPresent(p.entry, nout) {
|
||||
continue
|
||||
}
|
||||
out = append(out, SpoilerLeak{
|
||||
Src: p.entry.src, Dst: p.entry.dst,
|
||||
SinceCh: p.entry.sinceCh, UntilCh: p.entry.untilCh,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,19 +25,21 @@ import (
|
|||
|
||||
// --- manual seed file (YAML) ----------------------------------------------------
|
||||
|
||||
// LoadGlossarySeed parses a glossary seed YAML into store entries. A manual seed is
|
||||
// LoadBankSeed parses a seed YAML into the bank's three record types. A manual seed is
|
||||
// curated, so an empty status defaults to "approved" (the author downgrades explicitly
|
||||
// with status: draft|auto). Validation is fail-loud: a term without src, or with an
|
||||
// unknown status, is a config error (a silently-dropped term is exactly the A-class
|
||||
// hole this bank exists to close).
|
||||
func LoadGlossarySeed(path string) ([]store.GlossaryEntry, error) {
|
||||
// hole this bank exists to close). The voices:/addresses: sections follow the same
|
||||
// contract — see loadVoiceSections.
|
||||
func LoadBankSeed(path string) (BankSeed, error) {
|
||||
var bs BankSeed
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("membank: read glossary seed %s: %w", path, err)
|
||||
return bs, fmt.Errorf("membank: read glossary seed %s: %w", path, err)
|
||||
}
|
||||
var sf seed.File
|
||||
if err := yaml.Unmarshal(raw, &sf); err != nil {
|
||||
return nil, fmt.Errorf("membank: parse glossary seed %s: %w", path, err)
|
||||
return bs, fmt.Errorf("membank: parse glossary seed %s: %w", path, err)
|
||||
}
|
||||
var out []store.GlossaryEntry
|
||||
var problems []string
|
||||
|
|
@ -87,7 +89,7 @@ func LoadGlossarySeed(path string) ([]store.GlossaryEntry, error) {
|
|||
if t.Decl != nil {
|
||||
b, mErr := json.Marshal(declInfo{Invariant: t.Decl.Invariant, Forms: t.Decl.Forms})
|
||||
if mErr != nil {
|
||||
return nil, mErr
|
||||
return bs, mErr
|
||||
}
|
||||
decl = string(b)
|
||||
}
|
||||
|
|
@ -149,9 +151,14 @@ func LoadGlossarySeed(path string) ([]store.GlossaryEntry, error) {
|
|||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||||
return bs, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||||
}
|
||||
return out, nil
|
||||
voices, pairs, verr := loadVoiceSections(path, &sf)
|
||||
if verr != nil {
|
||||
return BankSeed{}, verr
|
||||
}
|
||||
bs.Terms, bs.Voices, bs.Pairs = out, voices, pairs
|
||||
return bs, nil
|
||||
}
|
||||
|
||||
// ApprovedSharedKeyCollisions detects the alias-generalization of the D16.1 polysemy livelock
|
||||
|
|
|
|||
379
backend/internal/membank/memvoice.go
Normal file
379
backend/internal/membank/memvoice.go
Normal file
|
|
@ -0,0 +1,379 @@
|
|||
package membank
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"textmachine/backend/internal/seed"
|
||||
"textmachine/backend/internal/store"
|
||||
"textmachine/backend/internal/text"
|
||||
)
|
||||
|
||||
// memvoice.go: the two D21 bank record types pack-19 added — the per-character VOICE profile and the
|
||||
// ordered ADDRESS pair — from seed file to store rows, plus the projection the flagger consumes.
|
||||
//
|
||||
// They are bank content and follow the bank's rules: curated only (no miner emits them), validated
|
||||
// fail-loud at load (a silently inert profile is the class this bank exists to close), windowed on the
|
||||
// SAME chapter axis as the spoiler window, and folded into the same memory_version.
|
||||
//
|
||||
// What they are NOT: matchable terms. A profile is keyed BY a character, it is not a surface to find in
|
||||
// the text — so they never enter the Aho-Corasick automaton or the glossary injection. That separation
|
||||
// is carried by the Go type system (BankInput.Rows is the only field Materialize matches over), not by a
|
||||
// runtime filter that a later edit could forget.
|
||||
|
||||
// registerInformal / registerFormal are the ABSTRACT T/V vocabulary. They are typological category
|
||||
// names, not target words: which surfaces realise them is target data, so a target with no T/V
|
||||
// distinction ships no surfaces and the whole mechanism stays inert.
|
||||
const (
|
||||
registerInformal = "informal"
|
||||
registerFormal = "formal"
|
||||
)
|
||||
|
||||
func validRegister(s string) bool { return s == registerInformal || s == registerFormal }
|
||||
|
||||
// BankSeed is one parsed seed file: the terms plus the two pack-19 record types.
|
||||
type BankSeed struct {
|
||||
Terms []store.GlossaryEntry
|
||||
Voices []store.VoiceProfile
|
||||
Pairs []store.AddressPair
|
||||
}
|
||||
|
||||
// LoadGlossarySeed parses the TERMS of a seed file — the pre-pack-19 entry point, kept for every caller
|
||||
// that has no use for the other two record types.
|
||||
func LoadGlossarySeed(path string) ([]store.GlossaryEntry, error) {
|
||||
s, err := LoadBankSeed(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Terms, nil
|
||||
}
|
||||
|
||||
// loadVoiceSections validates and materializes the voices:/addresses: sections of a parsed seed file.
|
||||
// Fail-loud on anything that would be silently inert or would crash the UNIQUE constraint mid-run, which
|
||||
// is the same contract the terms half keeps.
|
||||
func loadVoiceSections(path string, sf *seed.File) ([]store.VoiceProfile, []store.AddressPair, error) {
|
||||
var problems []string
|
||||
var voices []store.VoiceProfile
|
||||
voiceKeys := map[[4]string]bool{}
|
||||
for i, v := range sf.Voices {
|
||||
src, sense := strings.TrimSpace(v.Src), strings.TrimSpace(v.Sense)
|
||||
if src == "" {
|
||||
problems = append(problems, fmt.Sprintf("voice %d: src is required (it names the glossary term this profile belongs to)", i))
|
||||
continue
|
||||
}
|
||||
if d := strings.TrimSpace(v.AddressDefault); d != "" && !validRegister(d) {
|
||||
problems = append(problems, fmt.Sprintf("voice %q: address_default must be %s|%s, got %q", src, registerInformal, registerFormal, d))
|
||||
continue
|
||||
}
|
||||
k := [4]string{src, sense, fmt.Sprint(v.SinceCh), fmt.Sprint(v.UntilCh)}
|
||||
if voiceKeys[k] {
|
||||
problems = append(problems, fmt.Sprintf("voice %q: duplicate (src, sense=%q, since_ch=%d, until_ch=%d) — a character has one profile per window", src, sense, v.SinceCh, v.UntilCh))
|
||||
continue
|
||||
}
|
||||
voiceKeys[k] = true
|
||||
markers, err := cleanList(v.LexiconMarkers, "voice "+src+" lexicon_markers")
|
||||
if err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
continue
|
||||
}
|
||||
ng, err := cleanList(v.NGLexicon, "voice "+src+" ng_lexicon")
|
||||
if err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
continue
|
||||
}
|
||||
ex, err := cleanList(v.Exemplars, "voice "+src+" exemplars")
|
||||
if err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
continue
|
||||
}
|
||||
// A profile that states NOTHING is an authoring slip: it costs a bank row and a hash, and directs
|
||||
// nothing. Refused for the same reason an empty langpack table is (never silently empty).
|
||||
if strings.TrimSpace(v.Register) == "" && strings.TrimSpace(v.SelfRef) == "" &&
|
||||
strings.TrimSpace(v.AddressDefault) == "" && len(markers) == 0 && len(ng) == 0 && len(ex) == 0 {
|
||||
problems = append(problems, fmt.Sprintf("voice %q: the profile states nothing (needs at least one of register|self_ref|address_default|lexicon_markers|ng_lexicon|exemplars)", src))
|
||||
continue
|
||||
}
|
||||
voices = append(voices, store.VoiceProfile{
|
||||
Src: src, Sense: sense,
|
||||
Register: strings.TrimSpace(v.Register), SelfRef: strings.TrimSpace(v.SelfRef),
|
||||
AddressDefault: strings.TrimSpace(v.AddressDefault),
|
||||
LexiconMarkers: encodeList(markers), NGLexicon: encodeList(ng), Exemplars: encodeList(ex),
|
||||
Brightness: strings.TrimSpace(v.Brightness), SinceCh: v.SinceCh, UntilCh: v.UntilCh,
|
||||
})
|
||||
}
|
||||
// Two profiles of one character valid at the same chapter: the projection would have to pick, and it
|
||||
// has no basis to. Same rule, same reason, as two renderings of one term in one chapter.
|
||||
for i := range voices {
|
||||
for j := 0; j < i; j++ {
|
||||
a, b := voices[i], voices[j]
|
||||
if a.Src == b.Src && a.Sense == b.Sense && windowsOverlap(a.SinceCh, a.UntilCh, b.SinceCh, b.UntilCh) {
|
||||
problems = append(problems, fmt.Sprintf("voice %q: windows [%d,%d] and [%d,%d] overlap — a character has ONE profile per chapter",
|
||||
a.Src, b.SinceCh, b.UntilCh, a.SinceCh, a.UntilCh))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var pairs []store.AddressPair
|
||||
pairKeys := map[[6]string]bool{}
|
||||
for i, p := range sf.Addresses {
|
||||
sp, spSense := strings.TrimSpace(p.Speaker), strings.TrimSpace(p.SpeakerSense)
|
||||
ad, adSense := strings.TrimSpace(p.Addressee), strings.TrimSpace(p.AddresseeSense)
|
||||
if sp == "" || ad == "" {
|
||||
problems = append(problems, fmt.Sprintf("address %d: speaker and addressee are both required", i))
|
||||
continue
|
||||
}
|
||||
if sp == ad && spSense == adSense {
|
||||
problems = append(problems, fmt.Sprintf("address %q: speaker and addressee are the same character — a register is a fact about a PAIR", sp))
|
||||
continue
|
||||
}
|
||||
if !validRegister(strings.TrimSpace(p.Register)) {
|
||||
problems = append(problems, fmt.Sprintf("address %q→%q: register must be %s|%s, got %q (the value is a typological category, never a target word)",
|
||||
sp, ad, registerInformal, registerFormal, p.Register))
|
||||
continue
|
||||
}
|
||||
k := [6]string{sp, spSense, ad, adSense, fmt.Sprint(p.SinceCh), fmt.Sprint(p.UntilCh)}
|
||||
if pairKeys[k] {
|
||||
problems = append(problems, fmt.Sprintf("address %q→%q: duplicate (speaker, addressee, since_ch=%d, until_ch=%d)", sp, ad, p.SinceCh, p.UntilCh))
|
||||
continue
|
||||
}
|
||||
pairKeys[k] = true
|
||||
pairs = append(pairs, store.AddressPair{
|
||||
SpeakerSrc: sp, SpeakerSense: spSense, AddresseeSrc: ad, AddresseeSense: adSense,
|
||||
Register: strings.TrimSpace(p.Register), Form: strings.TrimSpace(p.Form),
|
||||
Closeness: strings.TrimSpace(p.Closeness), SinceCh: p.SinceCh, UntilCh: p.UntilCh,
|
||||
})
|
||||
}
|
||||
// Contradictory registers for one ordered pair at one chapter. A ты↔вы switch is legitimate — as a
|
||||
// SECOND row with a non-overlapping window, which is what makes the table a journal.
|
||||
for i := range pairs {
|
||||
for j := 0; j < i; j++ {
|
||||
a, b := pairs[i], pairs[j]
|
||||
if a.SpeakerSrc != b.SpeakerSrc || a.SpeakerSense != b.SpeakerSense ||
|
||||
a.AddresseeSrc != b.AddresseeSrc || a.AddresseeSense != b.AddresseeSense {
|
||||
continue
|
||||
}
|
||||
if a.Register != b.Register && windowsOverlap(a.SinceCh, a.UntilCh, b.SinceCh, b.UntilCh) {
|
||||
problems = append(problems, fmt.Sprintf("address %q→%q: registers %q [%d,%d] and %q [%d,%d] overlap — one pair has ONE register per chapter (a switch is a new window, not an overlap)",
|
||||
a.SpeakerSrc, a.AddresseeSrc, b.Register, b.SinceCh, b.UntilCh, a.Register, a.SinceCh, a.UntilCh))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, nil, fmt.Errorf("glossary seed %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||||
}
|
||||
return voices, pairs, nil
|
||||
}
|
||||
|
||||
// cleanList trims a seed string list and refuses an empty member: a blank exemplar or marker is an
|
||||
// authoring slip that would inject an empty line and count toward a budget.
|
||||
func cleanList(in []string, what string) ([]string, error) {
|
||||
var out []string
|
||||
for _, s := range in {
|
||||
t := strings.TrimSpace(s)
|
||||
if t == "" {
|
||||
return nil, fmt.Errorf("%s: an entry is empty", what)
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// encodeList renders a string list as the JSON stored in a column. Empty → "" (not "null"/"[]"), so an
|
||||
// unstated field costs no bytes in the row and none in the hash.
|
||||
func encodeList(in []string) string {
|
||||
if len(in) == 0 {
|
||||
return ""
|
||||
}
|
||||
b, err := json.Marshal(in)
|
||||
if err != nil { // a []string cannot fail to marshal
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// decodeList reads a stored JSON list back. A malformed blob yields nil rather than an error: the column
|
||||
// is written by encodeList alone, so a bad value is impossible without direct DB surgery, and the
|
||||
// flagger's honest answer to "no data" is to check nothing.
|
||||
func decodeList(s string) []string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
if json.Unmarshal([]byte(s), &out) != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// UnknownVoiceCharacters reports voice/address rows whose (src, sense) names no term in the bank. Such a
|
||||
// row is silently inert — the character has no target surfaces, so nothing can ever attribute a reply to
|
||||
// it — which is exactly the A-class hole the bank exists to close, so the caller fails the run loud.
|
||||
// Checked over the FULL entry set (seed + ruby + mined + auto), because a character may legitimately be
|
||||
// signed in a delta rather than the base seed. Deterministic (input order).
|
||||
func UnknownVoiceCharacters(entries []store.GlossaryEntry, voices []store.VoiceProfile, pairs []store.AddressPair) []string {
|
||||
known := map[[2]string]bool{}
|
||||
for _, e := range entries {
|
||||
known[[2]string{e.Src, e.Sense}] = true
|
||||
}
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
add := func(msg string) {
|
||||
if !seen[msg] {
|
||||
seen[msg] = true
|
||||
out = append(out, msg)
|
||||
}
|
||||
}
|
||||
for _, v := range voices {
|
||||
if !known[[2]string{v.Src, v.Sense}] {
|
||||
add(fmt.Sprintf("voice profile %q (sense %q) names no term in the bank", v.Src, v.Sense))
|
||||
}
|
||||
}
|
||||
for _, p := range pairs {
|
||||
if !known[[2]string{p.SpeakerSrc, p.SpeakerSense}] {
|
||||
add(fmt.Sprintf("address speaker %q (sense %q) names no term in the bank", p.SpeakerSrc, p.SpeakerSense))
|
||||
}
|
||||
if !known[[2]string{p.AddresseeSrc, p.AddresseeSense}] {
|
||||
add(fmt.Sprintf("address addressee %q (sense %q) names no term in the bank", p.AddresseeSrc, p.AddresseeSense))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// VoiceWindowGaps reports a character whose profiles leave a chapter UNCOVERED between two windows
|
||||
// (…until_ch=10 then since_ch=20: chapters 11–19 have no voice at all). Not an error — a deliberate gap
|
||||
// is legitimate — but it is invisible in a seed file and is far more often a typo, so the caller logs it.
|
||||
// The optional lint D39.55 asked for, applied where the data actually is.
|
||||
func VoiceWindowGaps(voices []store.VoiceProfile) []string {
|
||||
byChar := map[[2]string][]store.VoiceProfile{}
|
||||
var order [][2]string
|
||||
for _, v := range voices {
|
||||
k := [2]string{v.Src, v.Sense}
|
||||
if _, ok := byChar[k]; !ok {
|
||||
order = append(order, k)
|
||||
}
|
||||
byChar[k] = append(byChar[k], v)
|
||||
}
|
||||
var out []string
|
||||
for _, k := range order {
|
||||
win := byChar[k]
|
||||
if len(win) < 2 {
|
||||
continue
|
||||
}
|
||||
sort.Slice(win, func(i, j int) bool { return win[i].SinceCh < win[j].SinceCh })
|
||||
for i := 1; i < len(win); i++ {
|
||||
prevUntil := win[i-1].UntilCh
|
||||
if prevUntil == 0 { // open-ended: nothing after it can be a gap
|
||||
continue
|
||||
}
|
||||
if win[i].SinceCh > prevUntil+1 {
|
||||
out = append(out, fmt.Sprintf("voice %q: chapters %d–%d have no profile (window ends at %d, next starts at %d)",
|
||||
k[0], prevUntil+1, win[i].SinceCh-1, prevUntil, win[i].SinceCh))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CharacterView is one character as an OUTPUT-side checker needs to see it: an opaque identity, the
|
||||
// normalized target surfaces that name it, and the two profile fields the deterministic checks read. It
|
||||
// is deliberately free of store and checker types — membank owns the projection, the driver owns the
|
||||
// adaptation — so neither package has to know the other's vocabulary.
|
||||
type CharacterView struct {
|
||||
Key string // src\x1fsense — compared, never rendered into a message
|
||||
Name string // the approved rendering, for a human-readable detail line
|
||||
Forms []string
|
||||
SelfRef string
|
||||
NG []string
|
||||
}
|
||||
|
||||
// AddressView is one ordered register record projected onto a chapter.
|
||||
type AddressView struct {
|
||||
Speaker, Addressee, Register string // Speaker/Addressee are CharacterView.Key
|
||||
}
|
||||
|
||||
// VoiceProjection resolves the bank's voice/address content ONTO one chapter: the profiles and register
|
||||
// records whose window contains it, joined to the terms that give each character its target surfaces.
|
||||
//
|
||||
// It is a PROJECTION, never stored state — which is what keeps address_pairs a journal with one source
|
||||
// of truth (D21 п.2) instead of a second store that could disagree with it. A character whose term is
|
||||
// spoiler-blocked at this chapter is dropped entirely: it cannot be matched in the output, so a profile
|
||||
// for it could only produce noise. Pure and deterministic (bank order).
|
||||
func (b *Bank) VoiceProjection(chapter int) ([]CharacterView, []AddressView) {
|
||||
if b == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// The character's surfaces come from the term row valid at this chapter (the same window rule the
|
||||
// injection uses), so a pre/post-reveal handoff projects the rendering the chapter is allowed to see.
|
||||
forms := func(src, sense string) (view CharacterView, ok bool) {
|
||||
for i := range b.entries {
|
||||
e := &b.entries[i]
|
||||
if e.src != src || e.sense != sense || spoilerBlocked(e, chapter) {
|
||||
continue
|
||||
}
|
||||
base := text.NormalizeTargetForm(e.dst)
|
||||
if base == "" {
|
||||
continue
|
||||
}
|
||||
view.Key = src + "\x1f" + sense
|
||||
view.Name = e.dst
|
||||
view.Forms = append([]string{base}, e.declForms...)
|
||||
return view, true
|
||||
}
|
||||
return view, false
|
||||
}
|
||||
byKey := map[string]*CharacterView{}
|
||||
var chars []CharacterView
|
||||
order := []string{}
|
||||
need := func(src, sense string) *CharacterView {
|
||||
key := src + "\x1f" + sense
|
||||
if v, ok := byKey[key]; ok {
|
||||
return v
|
||||
}
|
||||
v, ok := forms(src, sense)
|
||||
if !ok {
|
||||
byKey[key] = nil
|
||||
return nil
|
||||
}
|
||||
byKey[key] = &v
|
||||
order = append(order, key)
|
||||
return &v
|
||||
}
|
||||
for _, vp := range b.voices {
|
||||
if vp.SinceCh > 0 && chapter < vp.SinceCh {
|
||||
continue
|
||||
}
|
||||
if vp.UntilCh > 0 && chapter > vp.UntilCh {
|
||||
continue
|
||||
}
|
||||
v := need(vp.Src, vp.Sense)
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
v.SelfRef = text.NormalizeTargetForm(vp.SelfRef)
|
||||
for _, w := range decodeList(vp.NGLexicon) {
|
||||
if n := text.NormalizeTargetForm(w); n != "" {
|
||||
v.NG = append(v.NG, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
var addrs []AddressView
|
||||
for _, p := range b.pairs {
|
||||
if p.SinceCh > 0 && chapter < p.SinceCh {
|
||||
continue
|
||||
}
|
||||
if p.UntilCh > 0 && chapter > p.UntilCh {
|
||||
continue
|
||||
}
|
||||
sp, ad := need(p.SpeakerSrc, p.SpeakerSense), need(p.AddresseeSrc, p.AddresseeSense)
|
||||
if sp == nil || ad == nil {
|
||||
continue
|
||||
}
|
||||
addrs = append(addrs, AddressView{Speaker: sp.Key, Addressee: ad.Key, Register: p.Register})
|
||||
}
|
||||
for _, k := range order {
|
||||
chars = append(chars, *byKey[k])
|
||||
}
|
||||
return chars, addrs
|
||||
}
|
||||
269
backend/internal/membank/memvoice_test.go
Normal file
269
backend/internal/membank/memvoice_test.go
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
package membank
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/lang"
|
||||
"textmachine/backend/internal/store"
|
||||
)
|
||||
|
||||
// memvoice_test.go pins the pack-19 record types on the axes that cost money if they are wrong: the
|
||||
// CONDITIONAL fold (a book must not re-pay for authoring a profile, and must re-pay the moment one
|
||||
// reaches the wire), the scope order the fold shares with the mined filter, and the structural promise
|
||||
// that a voice profile is never a matchable term.
|
||||
|
||||
func term(src, dst string) store.GlossaryEntry {
|
||||
return store.GlossaryEntry{Src: src, Dst: dst, Status: "approved", Source: "seed"}
|
||||
}
|
||||
|
||||
func profile(src string) store.VoiceProfile {
|
||||
return store.VoiceProfile{Src: src, Register: "ледяная краткость", SelfRef: "ваша служанка"}
|
||||
}
|
||||
|
||||
// The load-bearing money property: while the injection is off, voice/address content is invisible to the
|
||||
// hash, so a book that gains a whole cast of profiles keeps its snapshot and re-pays nothing.
|
||||
func TestVoiceRowsDoNotMoveTheBankVersionWhileUninjected(t *testing.T) {
|
||||
rows := []store.GlossaryEntry{term("方源", "Фан Юань")}
|
||||
bare := ComputeVersionScopedIn(BankInput{Rows: rows}, false, false)
|
||||
withVoice := ComputeVersionScopedIn(BankInput{
|
||||
Rows: rows,
|
||||
Voices: []store.VoiceProfile{profile("方源")},
|
||||
Pairs: []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "方源", Register: "formal"}},
|
||||
}, false, false)
|
||||
if bare != withVoice {
|
||||
t.Fatalf("an UNINJECTED profile moved the bank version — every book with one would re-pay a wave\n bare %s\n with %s", bare, withVoice)
|
||||
}
|
||||
// And the pre-pack-19 entry point must reproduce the same bytes, so no existing book moves at all.
|
||||
if legacy := ComputeVersion(rows, false); legacy != bare {
|
||||
t.Fatalf("the glossary-only fold changed shape: %s vs %s", legacy, bare)
|
||||
}
|
||||
}
|
||||
|
||||
// The other half of the same contract: once injected, the rows are wire bytes, so any edit MUST be a
|
||||
// loud --resnapshot. Not folding them then would reopen the class D39.42 п.3 closed.
|
||||
func TestInjectedVoiceRowsMoveTheBankVersion(t *testing.T) {
|
||||
rows := []store.GlossaryEntry{term("方源", "Фан Юань")}
|
||||
off := ComputeVersionScopedIn(BankInput{Rows: rows, Voices: []store.VoiceProfile{profile("方源")}}, false, false)
|
||||
on := ComputeVersionScopedIn(BankInput{Rows: rows, Voices: []store.VoiceProfile{profile("方源")}, InjectVoice: true}, false, false)
|
||||
if off == on {
|
||||
t.Fatal("turning the injection on left the bank version unchanged — the flip would be a silent re-payment")
|
||||
}
|
||||
edited := profile("方源")
|
||||
edited.SelfRef = "я"
|
||||
after := ComputeVersionScopedIn(BankInput{Rows: rows, Voices: []store.VoiceProfile{edited}, InjectVoice: true}, false, false)
|
||||
if after == on {
|
||||
t.Fatal("editing an INJECTED profile left the bank version unchanged — a wire change the snapshot cannot see")
|
||||
}
|
||||
// A pair edit is the same class.
|
||||
p1 := BankInput{Rows: rows, Pairs: []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "formal"}}, InjectVoice: true}
|
||||
p2 := p1
|
||||
p2.Pairs = []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "informal"}}
|
||||
rows2 := []store.GlossaryEntry{term("方源", "Фан Юань"), term("白凝冰", "Бай Нинбин")}
|
||||
p1.Rows, p2.Rows = rows2, rows2
|
||||
if ComputeVersionScopedIn(p1, false, false) == ComputeVersionScopedIn(p2, false, false) {
|
||||
t.Fatal("flipping a pair's register left the bank version unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
// The scope order (D39.55 «ровно так» (г)): the exclusion runs BEFORE the tag condition, so a profile
|
||||
// whose character is mined-only folds into the ENRICHED version and NOT into the base — otherwise
|
||||
// editing a mined-only character's profile would move the DRAFT wave, which never sees that character.
|
||||
func TestVoiceFoldRespectsTheMinedScopeBeforeCountingIt(t *testing.T) {
|
||||
mined := term("蛊", "Гу")
|
||||
mined.Source = "mined"
|
||||
rows := []store.GlossaryEntry{term("方源", "Фан Юань"), mined}
|
||||
withMinedProfile := BankInput{Rows: rows, Voices: []store.VoiceProfile{profile("蛊")}, InjectVoice: true}
|
||||
noProfile := BankInput{Rows: rows, InjectVoice: true}
|
||||
|
||||
if got, want := ComputeVersionScopedIn(withMinedProfile, false, true), ComputeVersionScopedIn(noProfile, false, true); got != want {
|
||||
t.Fatalf("a mined-only character's profile folded into the BASE version — the draft wave would re-pay for a character it never sees\n got %s\n want %s", got, want)
|
||||
}
|
||||
if ComputeVersionScopedIn(withMinedProfile, false, false) == ComputeVersionScopedIn(noProfile, false, false) {
|
||||
t.Fatal("a mined character's profile did NOT fold into the enriched version — the edit wave would not re-pay for a wire change")
|
||||
}
|
||||
}
|
||||
|
||||
// (в) A voice profile is bank content, never a matchable surface: nothing it carries may reach the
|
||||
// automaton or either injection block. The type system makes it structural; this pins the observable.
|
||||
func TestVoiceProfileNeverReachesTheMatcherOrTheInjection(t *testing.T) {
|
||||
const canary = "СЕКРЕТПРОФИЛЬ"
|
||||
vp := store.VoiceProfile{Src: "方源", Register: canary, SelfRef: canary, Exemplars: `["` + canary + `"]`}
|
||||
b := MaterializeBank(BankInput{
|
||||
Rows: []store.GlossaryEntry{term("方源", "Фан Юань")},
|
||||
Voices: []store.VoiceProfile{vp},
|
||||
Pairs: []store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "formal", Form: canary}},
|
||||
}, false)
|
||||
sel := b.Select("方源 стоял тут.", 1, nil, 0)
|
||||
if len(sel.Injected) != 1 {
|
||||
t.Fatalf("the TERM must still fire normally, got %d injected", len(sel.Injected))
|
||||
}
|
||||
tx := lang.InjectionTextsFor("ru")
|
||||
for name, block := range map[string]string{
|
||||
"draft": RenderGlossaryBlock(sel.Injected, tx),
|
||||
"editor": RenderEditorConstraintBlock(sel.Injected, tx),
|
||||
} {
|
||||
if strings.Contains(block, canary) {
|
||||
t.Fatalf("%s injection leaked voice-profile content: %q", name, block)
|
||||
}
|
||||
}
|
||||
// And the profile's own src must not have become a second, separate match.
|
||||
if n := len(b.Select(canary, 1, nil, 0).Injected); n != 0 {
|
||||
t.Fatalf("profile content became matchable: %d hits on the canary", n)
|
||||
}
|
||||
}
|
||||
|
||||
// (ж) A sticky carry the window rejects is RECORDED, not dropped: n_spoiler_blocked must not under-count.
|
||||
func TestStickyCarryBlockedByWindowIsRecordedAsRejected(t *testing.T) {
|
||||
late := term("真相", "правда")
|
||||
late.SinceCh = 5
|
||||
b := Materialize([]store.GlossaryEntry{late}, false)
|
||||
// The carry arrives at chapter 1, where the window blocks it. Reachable only through the exported
|
||||
// API — precomputeSticky resets per chapter — which is exactly why the guard belongs here.
|
||||
sel := b.Select("ничего не совпадает", 1, map[string]InjectionDisposition{
|
||||
b.entries[0].id: Confirmed,
|
||||
}, 0)
|
||||
if len(sel.Injected) != 0 {
|
||||
t.Fatalf("a blocked carry must not inject, got %d", len(sel.Injected))
|
||||
}
|
||||
if len(sel.Rejected) != 1 || !sel.Rejected[0].Sticky {
|
||||
t.Fatalf("a window-blocked sticky carry must be recorded as a sticky reject, got %+v", sel.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpoilerLeakFlagger(t *testing.T) {
|
||||
late := store.GlossaryEntry{Src: "真相", Dst: "её брат", Status: "approved", Source: "seed", SinceCh: 5,
|
||||
Decl: `{"forms":["её брата"]}`}
|
||||
b := Materialize([]store.GlossaryEntry{late}, false)
|
||||
sel := b.Select("真相 出现了。", 1, nil, 0) // the key FIRES, the window rejects it
|
||||
if len(sel.Rejected) != 1 {
|
||||
t.Fatalf("setup: want one rejected row, got %d", len(sel.Rejected))
|
||||
}
|
||||
if n := len(b.SpoilerLeaks(sel.Rejected, "Он молчал о прошлом.")); n != 0 {
|
||||
t.Fatalf("no leak expected, got %d", n)
|
||||
}
|
||||
leaks := b.SpoilerLeaks(sel.Rejected, "Это был её брат.")
|
||||
if len(leaks) != 1 || leaks[0].Src != "真相" || leaks[0].SinceCh != 5 {
|
||||
t.Fatalf("the rejected rendering reached the output and must be flagged, got %+v", leaks)
|
||||
}
|
||||
if n := len(b.SpoilerLeaks(sel.Rejected, "Он ждал её брата.")); n != 1 {
|
||||
t.Fatalf("a DECLINED form of the rejected rendering is the same leak, got %d", n)
|
||||
}
|
||||
// The ratified scope is "a leak on a FIRED key": a sticky reject did not fire here, so it is out —
|
||||
// otherwise the (ж) fix above would silently widen what this flagger claims to measure.
|
||||
sticky := []PickedEntry{{entry: sel.Rejected[0].entry, Sticky: true, Disp: Reject}}
|
||||
if n := len(b.SpoilerLeaks(sticky, "Это был её брат.")); n != 0 {
|
||||
t.Fatalf("a STICKY reject is outside the ratified scope, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceProjectionRespectsWindows(t *testing.T) {
|
||||
rows := []store.GlossaryEntry{term("方源", "Фан Юань"), term("白凝冰", "Бай Нинбин")}
|
||||
early := profile("方源")
|
||||
early.UntilCh = 10
|
||||
late := profile("方源")
|
||||
late.SinceCh = 11
|
||||
late.SelfRef = "я"
|
||||
b := MaterializeBank(BankInput{
|
||||
Rows: rows,
|
||||
Voices: []store.VoiceProfile{early, late},
|
||||
Pairs: []store.AddressPair{
|
||||
{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "formal", UntilCh: 10},
|
||||
{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "informal", SinceCh: 11},
|
||||
},
|
||||
}, false)
|
||||
chars, addrs := b.VoiceProjection(3)
|
||||
if len(chars) == 0 || chars[0].SelfRef != "ваша служанка" {
|
||||
t.Fatalf("chapter 3 must project the EARLY profile, got %+v", chars)
|
||||
}
|
||||
if len(addrs) != 1 || addrs[0].Register != "formal" {
|
||||
t.Fatalf("chapter 3 must project the formal register, got %+v", addrs)
|
||||
}
|
||||
_, addrs = b.VoiceProjection(20)
|
||||
if len(addrs) != 1 || addrs[0].Register != "informal" {
|
||||
t.Fatalf("chapter 20 must project the switched register, got %+v", addrs)
|
||||
}
|
||||
// A character whose term is spoiler-blocked cannot be matched in the output, so it is not projected.
|
||||
hidden := term("影", "Тень")
|
||||
hidden.SinceCh = 30
|
||||
b2 := MaterializeBank(BankInput{Rows: []store.GlossaryEntry{hidden}, Voices: []store.VoiceProfile{profile("影")}}, false)
|
||||
if chars, _ := b2.VoiceProjection(1); len(chars) != 0 {
|
||||
t.Fatalf("a spoiler-blocked character must not project, got %+v", chars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBankSeedLoadsVoiceSections(t *testing.T) {
|
||||
bs, err := LoadBankSeed(writeSeed(t, `
|
||||
terms:
|
||||
- src: 方源
|
||||
dst: Фан Юань
|
||||
- src: 白凝冰
|
||||
dst: Бай Нинбин
|
||||
voices:
|
||||
- src: 白凝冰
|
||||
register: ледяная краткость
|
||||
self_ref: ваша служанка
|
||||
ng_lexicon: [ладно, окей]
|
||||
exemplars: ["Слушаюсь."]
|
||||
addresses:
|
||||
- speaker: 白凝冰
|
||||
addressee: 方源
|
||||
register: formal
|
||||
since_ch: 1
|
||||
until_ch: 40
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(bs.Voices) != 1 || bs.Voices[0].SelfRef != "ваша служанка" || bs.Voices[0].NGLexicon != `["ладно","окей"]` {
|
||||
t.Fatalf("voice section not materialized: %+v", bs.Voices)
|
||||
}
|
||||
if len(bs.Pairs) != 1 || bs.Pairs[0].Register != "formal" || bs.Pairs[0].UntilCh != 40 {
|
||||
t.Fatalf("address section not materialized: %+v", bs.Pairs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBankSeedRefusesMalformedVoiceSections(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"empty profile": "voices:\n - src: 方源\n",
|
||||
"bad register": "addresses:\n - speaker: A\n addressee: B\n register: ты\n",
|
||||
"self-addressed pair": "addresses:\n - speaker: A\n addressee: A\n register: formal\n",
|
||||
"overlapping windows": "voices:\n - src: A\n self_ref: x\n until_ch: 10\n - src: A\n self_ref: y\n since_ch: 5\n",
|
||||
"contradictory pair": "addresses:\n - speaker: A\n addressee: B\n register: formal\n until_ch: 10\n - speaker: A\n addressee: B\n register: informal\n since_ch: 5\n",
|
||||
"empty exemplar": "voices:\n - src: A\n exemplars: [\"\"]\n",
|
||||
"missing addressee": "addresses:\n - speaker: A\n register: formal\n",
|
||||
}
|
||||
for name, body := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := LoadBankSeed(writeSeed(t, body)); err == nil {
|
||||
t.Fatal("a malformed voice/address section must fail loud at load, before any billing")
|
||||
}
|
||||
})
|
||||
}
|
||||
// A ты↔вы switch — the same pair, non-overlapping windows — is the legitimate journal shape.
|
||||
if _, err := LoadBankSeed(writeSeed(t, "addresses:\n - speaker: A\n addressee: B\n register: formal\n until_ch: 10\n - speaker: A\n addressee: B\n register: informal\n since_ch: 11\n")); err != nil {
|
||||
t.Fatalf("a register switch across non-overlapping windows is legitimate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownVoiceCharactersAndWindowGaps(t *testing.T) {
|
||||
rows := []store.GlossaryEntry{term("方源", "Фан Юань")}
|
||||
bad := UnknownVoiceCharacters(rows,
|
||||
[]store.VoiceProfile{{Src: "нет-такого"}},
|
||||
[]store.AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "тоже-нет"}})
|
||||
if len(bad) != 2 {
|
||||
t.Fatalf("both the unknown profile and the unknown addressee must be named, got %v", bad)
|
||||
}
|
||||
if n := len(UnknownVoiceCharacters(rows, []store.VoiceProfile{{Src: "方源"}}, nil)); n != 0 {
|
||||
t.Fatalf("a known character must not be reported, got %d", n)
|
||||
}
|
||||
gaps := VoiceWindowGaps([]store.VoiceProfile{
|
||||
{Src: "A", UntilCh: 10}, {Src: "A", SinceCh: 20},
|
||||
})
|
||||
if len(gaps) != 1 || !strings.Contains(gaps[0], "11–19") {
|
||||
t.Fatalf("the uncovered range must be named, got %v", gaps)
|
||||
}
|
||||
if n := len(VoiceWindowGaps([]store.VoiceProfile{{Src: "A", UntilCh: 10}, {Src: "A", SinceCh: 11}})); n != 0 {
|
||||
t.Fatalf("adjacent windows are not a gap, got %d", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -103,7 +103,7 @@ var roleInjectionRenderers = map[string]injectionRenderer{
|
|||
// selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED
|
||||
// records (what the model saw); spoiler/eviction are the dropped-and-logged totals;
|
||||
// post-check misses are recorded only when the translator actually produced text.
|
||||
func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel membank.Selection, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult, bank bankFlags, bankProps string) error {
|
||||
func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel membank.Selection, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult, bank bankFlags, bankProps string, voice checks.VoiceResult, leaks []membank.SpoilerLeak) error {
|
||||
rs := store.RetrievalState{
|
||||
BookID: r.Book.BookID, Chapter: ch.Chapter, ChunkIdx: ch.ChunkIdx, SnapshotID: snapID,
|
||||
NStyleFlags: cheap.Total(),
|
||||
|
|
@ -117,6 +117,7 @@ func (r *Runner) persistRetrievalState(snapID string, ch chunk.Chunk, sel memban
|
|||
rs.StyleDetail = string(b)
|
||||
}
|
||||
}
|
||||
setVoiceState(&rs, voice, leaks)
|
||||
for _, p := range sel.Injected {
|
||||
if p.Sticky {
|
||||
rs.NSticky++
|
||||
|
|
|
|||
|
|
@ -113,6 +113,24 @@ type QualityReport struct {
|
|||
UnverifiedShown int `json:"unverified_shown,omitempty"`
|
||||
UnverifiedFollowed int `json:"unverified_followed,omitempty"`
|
||||
|
||||
// The pack-19 flaggers (D39.55). VoiceFlags is axes A-C — the T/V contradictions, flattened
|
||||
// self-designations and forbidden lexemes the run found in ATTRIBUTED replies; VoiceReplies /
|
||||
// VoiceAttributed are its denominators, without which the count cannot be read. VoicePairRegister is
|
||||
// axis D, the registry check, reported apart from the count because its addressee comes from a
|
||||
// heuristic rather than from attribution. SpoilerLeaks is the reveal half of D21 п.3: renderings the
|
||||
// spoiler window rejected for their chapter that reached the shipped text anyway — the only one of
|
||||
// the four that is a safety signal rather than a style measurement. None gates anything; all
|
||||
// omitempty, so a book without voice content reports byte-identically to before.
|
||||
VoiceFlags int `json:"voice_flags,omitempty"`
|
||||
VoiceReplies int `json:"voice_replies,omitempty"`
|
||||
VoiceAttributed int `json:"voice_attributed,omitempty"`
|
||||
VoicePairRegister int `json:"voice_pair_register,omitempty"`
|
||||
SpoilerLeaks int `json:"spoiler_leaks,omitempty"`
|
||||
// VoiceCheckVersion names the rules that produced those counts. The voice gate is deliberately not
|
||||
// snapshot-folded (config.VoiceGate), so the version travels with the numbers instead — the same
|
||||
// mitigation the terminologist uses for the same trade-off.
|
||||
VoiceCheckVersion string `json:"voice_check_version,omitempty"`
|
||||
|
||||
// EscalationHops / SpendByModel are the money-side content-label provenance (B6): how many fallback
|
||||
// CALLS the book actually paid for (the per-unit `escalated` boolean cannot count them) and how the
|
||||
// spend splits across model slugs — which is what answers "what did the label-routed endpoint cost"
|
||||
|
|
@ -248,6 +266,16 @@ func (r *Runner) QualityReport() (*QualityReport, error) {
|
|||
rep.TrustGated += rs.NTrustGatedSuppress
|
||||
rep.UnverifiedShown += rs.NUnverifiedShown
|
||||
rep.UnverifiedFollowed += rs.NUnverifiedFollowed
|
||||
rep.VoiceFlags += rs.NVoiceFlags
|
||||
rep.SpoilerLeaks += rs.NSpoilerLeaks
|
||||
if rs.VoiceDetail != "" {
|
||||
var v checks.VoiceResult
|
||||
if json.Unmarshal([]byte(rs.VoiceDetail), &v) == nil {
|
||||
rep.VoiceReplies += v.Replies
|
||||
rep.VoiceAttributed += v.Attributed
|
||||
rep.VoicePairRegister += v.PairRegister
|
||||
}
|
||||
}
|
||||
if gateOn && rs.NPostcheckMiss > 0 {
|
||||
withheld[chunkKey{rs.Chapter, rs.ChunkIdx}] = true
|
||||
}
|
||||
|
|
@ -405,6 +433,9 @@ func (r *Runner) QualityReport() (*QualityReport, error) {
|
|||
return nil, merr
|
||||
}
|
||||
rep.SpendByModel = byModel
|
||||
if r.Pipeline.Gates.Voice.Enabled {
|
||||
rep.VoiceCheckVersion = checks.VoiceCheckVersion
|
||||
}
|
||||
if len(r.Book.ContentLabels) > 0 {
|
||||
rep.ContentLabels = r.Book.ContentLabels
|
||||
rep.Routing = r.contentRoutingRows()
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ type PromptTemplate struct {
|
|||
System string
|
||||
FewShot string
|
||||
User string
|
||||
SHA256 string // hash of the raw file — part of the snapshot
|
||||
SHA256 string // hash of the CANONICAL (comment-stripped) file — part of the snapshot
|
||||
}
|
||||
|
||||
// commentOpen/commentClose delimit an editorial comment in a prompt file — a note to whoever
|
||||
|
|
|
|||
|
|
@ -268,7 +268,7 @@ func (r *Runner) loadLangPack() error {
|
|||
// embedded target data (target-general lists, keyed by target lang). Built here so it exists even for a
|
||||
// no-langpack book (a nil pack → inert pair checkers; the target lists still load). r.pack is set below.
|
||||
defer func() {
|
||||
r.checkers = checks.CompileCheckers(checks.DCCheckerData(r.pack), lang.TargetChecksFor(r.Book.TargetLang))
|
||||
r.checkers = checks.CompileCheckersFor(r.pack, lang.TargetChecksFor(r.Book.TargetLang))
|
||||
}()
|
||||
if r.Book.LangpackRoot == "" {
|
||||
return nil // no langpack declared → nil pack, miner inert
|
||||
|
|
|
|||
|
|
@ -22,12 +22,15 @@ import (
|
|||
// approved dst-collisions are logged (not fatal — some collisions are legitimate).
|
||||
func (r *Runner) seedGlossary(ctx context.Context) error {
|
||||
var entries []store.GlossaryEntry
|
||||
var voices []store.VoiceProfile
|
||||
var pairs []store.AddressPair
|
||||
if r.Book.GlossarySeed != "" {
|
||||
seed, err := membank.LoadGlossarySeed(r.Book.GlossarySeed)
|
||||
bank, err := membank.LoadBankSeed(r.Book.GlossarySeed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries = append(entries, seed...)
|
||||
entries = append(entries, bank.Terms...)
|
||||
voices, pairs = bank.Voices, bank.Pairs
|
||||
}
|
||||
manualSrcs := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
|
|
@ -96,14 +99,40 @@ func (r *Runner) seedGlossary(ctx context.Context) error {
|
|||
if cols := membank.ApprovedSharedKeyCollisions(entries); len(cols) > 0 {
|
||||
return fmt.Errorf("pipeline: glossary shared-key collisions (A2 / D16.1 livelock class):\n - %s", strings.Join(cols, "\n - "))
|
||||
}
|
||||
if err := r.Store.ReplaceGlossary(r.Book.BookID, entries); err != nil {
|
||||
return fmt.Errorf("pipeline: replace glossary for %s (%d entries): %w", r.Book.BookID, len(entries), err)
|
||||
// A voice/address row naming a character the bank does not have is SILENTLY inert — nothing can ever
|
||||
// attribute a reply to it — which is the A-class hole this bank exists to close, so it stops the run.
|
||||
// Checked over the FULL entry set (seed + ruby + mined + auto), because a character may legitimately be
|
||||
// signed in a delta rather than in the base seed.
|
||||
if unknown := membank.UnknownVoiceCharacters(entries, voices, pairs); len(unknown) > 0 {
|
||||
return fmt.Errorf("pipeline: voice/address rows name characters absent from the bank (a profile for a term that does not exist can never fire):\n - %s", strings.Join(unknown, "\n - "))
|
||||
}
|
||||
if err := r.Store.ReplaceBank(r.Book.BookID, entries, voices, pairs); err != nil {
|
||||
return fmt.Errorf("pipeline: replace bank for %s (%d terms, %d voices, %d pairs): %w", r.Book.BookID, len(entries), len(voices), len(pairs), err)
|
||||
}
|
||||
rows, err := r.Store.GlossaryForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline: read glossary for %s: %w", r.Book.BookID, err)
|
||||
}
|
||||
r.memory = membank.Materialize(rows, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||||
storedVoices, err := r.Store.VoiceProfilesForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline: read voice profiles for %s: %w", r.Book.BookID, err)
|
||||
}
|
||||
storedPairs, err := r.Store.AddressPairsForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline: read address pairs for %s: %w", r.Book.BookID, err)
|
||||
}
|
||||
// A chapter range no profile covers is legitimate but is far more often a typo, and it is invisible in
|
||||
// a seed file — so it is logged, never fatal (the optional lint of D39.55).
|
||||
if gaps := membank.VoiceWindowGaps(storedVoices); len(gaps) > 0 {
|
||||
r.Log.WarnContext(ctx, "voice profile windows leave chapters uncovered (deliberate is fine; a typo is not)",
|
||||
"book", r.Book.BookID, "gaps", strings.Join(gaps, "; "))
|
||||
}
|
||||
// InjectVoice is FALSE and has no config knob: pack-19 builds the schema and the flagger, and D21 п.2
|
||||
// holds the injection conditional until the polygon experiment. It is the fold's condition, so while
|
||||
// it is false a book with voice rows hashes exactly as it did without them and nobody re-pays for
|
||||
// authoring a profile. Wiring the injection means setting it and accepting a full --resnapshot.
|
||||
bankIn := membank.BankInput{Rows: rows, Voices: storedVoices, Pairs: storedPairs}
|
||||
r.memory = membank.MaterializeBank(bankIn, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||||
// The DRAFT wave selects over a BASE-scoped bank (Source:mined excluded) so its injection is
|
||||
// byte-identical across a bank-mining enrichment — matching the draft-wave snapshot (baseMemoryVersion),
|
||||
// which keeps «one re-payment» honest at the WIRE level, not only the version-hash level. Only the
|
||||
|
|
@ -119,7 +148,9 @@ func (r *Runner) seedGlossary(ctx context.Context) error {
|
|||
baseRows = append(baseRows, row)
|
||||
}
|
||||
if hasMined {
|
||||
r.baseMemory = membank.Materialize(baseRows, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||||
baseIn := bankIn
|
||||
baseIn.Rows = baseRows
|
||||
r.baseMemory = membank.MaterializeBank(baseIn, r.Pipeline.Gates.Glossary.PostcheckGate)
|
||||
} else {
|
||||
r.baseMemory = r.memory
|
||||
}
|
||||
|
|
@ -137,7 +168,8 @@ func (r *Runner) seedGlossary(ctx context.Context) error {
|
|||
"book", r.Book.BookID, "conflicts", strings.Join(cols, "; "))
|
||||
}
|
||||
r.Log.InfoContext(ctx, "glossary materialized", "book", r.Book.BookID,
|
||||
"entries", len(rows), "memory_version", r.memory.Version()[:12])
|
||||
"entries", len(rows), "voices", len(storedVoices), "address_pairs", len(storedPairs),
|
||||
"memory_version", r.memory.Version()[:12])
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
82
backend/internal/pipeline/voicerun.go
Normal file
82
backend/internal/pipeline/voicerun.go
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"textmachine/backend/internal/checks"
|
||||
"textmachine/backend/internal/membank"
|
||||
"textmachine/backend/internal/store"
|
||||
)
|
||||
|
||||
// voicerun.go: the driver half of the pack-19 flaggers — the adapter that turns the bank's chapter
|
||||
// projection into the checker's plain-value registry, and the two $0 calls the final wave makes.
|
||||
//
|
||||
// Both are OBSERVABILITY: neither can flag a chunk, neither touches the wire, and both are recomputed
|
||||
// from stored text on every run, so a resume re-derives them without re-billing anything.
|
||||
|
||||
// voiceRegistry adapts membank's chapter projection to the checker vocabulary. Two packages, two value
|
||||
// types, one trivial mapping — which is what keeps checks free of store types and membank free of
|
||||
// checker types. Empty when the book has no voice/address content (the flagger is then inert).
|
||||
func voiceRegistry(mem *membank.Bank, chapter int) checks.VoiceRegistry {
|
||||
if mem == nil {
|
||||
return checks.VoiceRegistry{}
|
||||
}
|
||||
chars, addrs := mem.VoiceProjection(chapter)
|
||||
if len(chars) == 0 {
|
||||
return checks.VoiceRegistry{}
|
||||
}
|
||||
reg := checks.VoiceRegistry{Chars: make([]checks.VoiceCharacter, 0, len(chars))}
|
||||
for _, c := range chars {
|
||||
reg.Chars = append(reg.Chars, checks.VoiceCharacter{
|
||||
Key: c.Key, Name: c.Name, Forms: c.Forms, SelfRef: c.SelfRef, NG: c.NG,
|
||||
})
|
||||
}
|
||||
for _, a := range addrs {
|
||||
reg.Pairs = append(reg.Pairs, checks.VoiceAddressPair{
|
||||
Speaker: a.Speaker, Addressee: a.Addressee, Register: a.Register,
|
||||
})
|
||||
}
|
||||
return reg
|
||||
}
|
||||
|
||||
// runVoiceChecks runs the voice flagger over one unit's shipped text, when the gate is on. A book with
|
||||
// the gate off, or with no voice content, gets a zero result and writes zeros — byte-identical to a run
|
||||
// before this existed.
|
||||
func (r *Runner) runVoiceChecks(mem *membank.Bank, chapter int, source, final string) checks.VoiceResult {
|
||||
if !r.Pipeline.Gates.Voice.Enabled || final == "" {
|
||||
return checks.VoiceResult{}
|
||||
}
|
||||
return checks.RunVoiceChecks(source, final, voiceRegistry(mem, chapter), r.checkers)
|
||||
}
|
||||
|
||||
// spoilerLeaks checks the chapter's REJECTED bank rows against the shipped text: a rendering the spoiler
|
||||
// window excluded that reached the reader anyway. Unlike the voice flagger it is NOT gated — the spoiler
|
||||
// window is a standing safety mechanism (C1), not an opt-in measurement, and a leak is the one thing the
|
||||
// window exists to prevent. $0 and observability-only, like every sibling on the retrieval-state row.
|
||||
func (r *Runner) spoilerLeaks(mem *membank.Bank, sel membank.Selection, final string) []membank.SpoilerLeak {
|
||||
if mem == nil || final == "" || len(sel.Rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
return mem.SpoilerLeaks(sel.Rejected, final)
|
||||
}
|
||||
|
||||
// setVoiceState writes the two pack-19 flagger results onto a retrieval-state row. Zero results write
|
||||
// zeros and an empty detail, so a gate-off book's row is byte-identical to what it was before these
|
||||
// columns existed. Shared by the draft-chunk and edit-unit paths so the two cannot drift.
|
||||
func setVoiceState(rs *store.RetrievalState, voice checks.VoiceResult, leaks []membank.SpoilerLeak) {
|
||||
rs.NVoiceFlags, rs.VoiceDetail = voice.Total(), ""
|
||||
// The detail carries the WHOLE result, not only the flagged axes: the denominators (replies,
|
||||
// attributed, source replies) are what make the count readable, and the axis-D indicator is reported
|
||||
// beside them precisely because it is excluded from the count.
|
||||
if voice.Total() > 0 || voice.PairRegister > 0 || voice.Replies > 0 {
|
||||
if b, err := json.Marshal(voice); err == nil {
|
||||
rs.VoiceDetail = string(b)
|
||||
}
|
||||
}
|
||||
rs.NSpoilerLeaks, rs.SpoilerLeakDetail = len(leaks), ""
|
||||
if len(leaks) > 0 {
|
||||
if b, err := json.Marshal(leaks); err == nil {
|
||||
rs.SpoilerLeakDetail = string(b)
|
||||
}
|
||||
}
|
||||
}
|
||||
200
backend/internal/pipeline/voicerun_test.go
Normal file
200
backend/internal/pipeline/voicerun_test.go
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// voicerun_test.go: the pack-19 flaggers THROUGH THE REAL DRIVER — a seed with voice/address sections, a
|
||||
// mock editor whose output carries the defect, and the assertion that the count reaches retrieval_state
|
||||
// and the quality report WITHOUT flagging the unit.
|
||||
|
||||
const voiceGateBlock = "\ngates:\n voice:\n enabled: true\n"
|
||||
|
||||
// voiceSeed carries both new sections plus the terms they name. 白凝冰 owes 方源 the formal register and
|
||||
// calls herself «ваша служанка» — the worked example of research/15.
|
||||
const voiceSeed = `
|
||||
terms:
|
||||
- src: 方源
|
||||
dst: Фан Юань
|
||||
decl: { forms: [Фан Юаня, Фан Юаню] }
|
||||
- src: 白凝冰
|
||||
dst: Бай Нинбин
|
||||
voices:
|
||||
- src: 白凝冰
|
||||
register: ледяная краткость
|
||||
self_ref: ваша служанка
|
||||
ng_lexicon: [ладно]
|
||||
addresses:
|
||||
- speaker: 白凝冰
|
||||
addressee: 方源
|
||||
register: formal
|
||||
`
|
||||
|
||||
const voiceSource = "白凝冰对方源说话。"
|
||||
|
||||
func TestVoiceFlaggerReachesTheStoreAndTheReport(t *testing.T) {
|
||||
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
// One reply, attributed, that flattens the self-designation AND says a forbidden word.
|
||||
return "— Ладно, я всё сделаю, — сказала Бай Нинбин.", "stop"
|
||||
}
|
||||
return "ЧЕРНОВИК", "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{
|
||||
source: voiceSource, glossarySeed: voiceSeed, gatesYAML: voiceGateBlock,
|
||||
})
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
res, err := r.TranslateBook(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Flagged != 0 {
|
||||
t.Fatalf("the voice flagger is observability — it must never flag a unit, got flagged=%d", res.Flagged)
|
||||
}
|
||||
rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
|
||||
if err != nil || rs == nil {
|
||||
t.Fatalf("no retrieval state persisted: %v", err)
|
||||
}
|
||||
if rs.NVoiceFlags == 0 {
|
||||
t.Fatalf("the voice flags did not reach retrieval_state: %+v", rs)
|
||||
}
|
||||
if !strings.Contains(rs.VoiceDetail, "Бай Нинбин") {
|
||||
t.Fatalf("the detail must name the character: %q", rs.VoiceDetail)
|
||||
}
|
||||
rep, err := r.QualityReport()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rep.VoiceFlags != rs.NVoiceFlags || rep.VoiceReplies == 0 || rep.VoiceAttributed == 0 {
|
||||
t.Fatalf("the report must aggregate the flags AND their denominators: %+v", rep)
|
||||
}
|
||||
if rep.VoiceCheckVersion == "" {
|
||||
t.Fatal("the gate is not snapshot-folded, so the report MUST carry the rule version the counts came from")
|
||||
}
|
||||
}
|
||||
|
||||
// The gate is off by default, and off means zero — not "computed and hidden".
|
||||
func TestVoiceFlaggerIsSilentWhenTheGateIsOff(t *testing.T) {
|
||||
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
return "— Ладно, я всё сделаю, — сказала Бай Нинбин.", "stop"
|
||||
}
|
||||
return "ЧЕРНОВИК", "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: voiceSource, glossarySeed: voiceSeed})
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
|
||||
if err != nil || rs == nil {
|
||||
t.Fatalf("no retrieval state persisted: %v", err)
|
||||
}
|
||||
if rs.NVoiceFlags != 0 || rs.VoiceDetail != "" {
|
||||
t.Fatalf("gate off must write nothing at all: %+v", rs)
|
||||
}
|
||||
rep, err := r.QualityReport()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rep.VoiceFlags != 0 || rep.VoiceCheckVersion != "" {
|
||||
t.Fatalf("gate off must report nothing: %+v", rep)
|
||||
}
|
||||
}
|
||||
|
||||
// The reveal half of D21 п.3: a rendering the spoiler window rejected for this chapter that reached the
|
||||
// shipped text. NOT gated by gates.voice — the window is a standing safety mechanism.
|
||||
func TestSpoilerLeakSurfacesUngated(t *testing.T) {
|
||||
const seed = `
|
||||
terms:
|
||||
- src: 真相
|
||||
dst: её брат
|
||||
since_ch: 5
|
||||
`
|
||||
srv := newJSONProvider(&reqRec{}, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
return "Это был её брат.", "stop" // the post-reveal rendering, in chapter 1
|
||||
}
|
||||
return "ЧЕРНОВИК", "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: "真相 出现了。", glossarySeed: seed})
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
res, err := r.TranslateBook(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Flagged != 0 {
|
||||
t.Fatalf("a leak is observability, not a disposition, got flagged=%d", res.Flagged)
|
||||
}
|
||||
rs, err := r.Store.GetRetrievalState("test-book", 1, 0)
|
||||
if err != nil || rs == nil {
|
||||
t.Fatalf("no retrieval state persisted: %v", err)
|
||||
}
|
||||
if rs.NSpoilerLeaks != 1 || !strings.Contains(rs.SpoilerLeakDetail, "真相") {
|
||||
t.Fatalf("the spoiler leak did not reach retrieval_state: %+v", rs)
|
||||
}
|
||||
rep, err := r.QualityReport()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rep.SpoilerLeaks != 1 {
|
||||
t.Fatalf("the report must aggregate the leak: %+v", rep)
|
||||
}
|
||||
}
|
||||
|
||||
// A voice/address row naming a character the bank does not hold is silently inert — so the run must stop
|
||||
// at seeding, before any provider call, rather than translate a book whose profiles can never fire.
|
||||
func TestSeedRefusesAVoiceProfileForAnUnknownCharacter(t *testing.T) {
|
||||
const seed = `
|
||||
terms:
|
||||
- src: 方源
|
||||
dst: Фан Юань
|
||||
voices:
|
||||
- src: 白凝冰
|
||||
self_ref: ваша служанка
|
||||
`
|
||||
srv := newJSONProvider(&reqRec{}, func(string) (string, string) { return "ЧЕРНОВИК", "stop" })
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: voiceSource, glossarySeed: seed})
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
_, err := r.TranslateBook(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("a profile for a term that does not exist must stop the run")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "白凝冰") {
|
||||
t.Fatalf("the error must name the character: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The voice content is bank content: it is replaced in the SAME transaction as the glossary and read
|
||||
// back for the projection, so a re-seed converges every column exactly as the terms do.
|
||||
func TestVoiceContentRoundTripsThroughTheBankReplace(t *testing.T) {
|
||||
srv := newJSONProvider(&reqRec{}, func(string) (string, string) { return "ЧЕРНОВИК", "stop" })
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{source: voiceSource, glossarySeed: voiceSeed})
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
if _, err := r.TranslateBook(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
voices, err := r.Store.VoiceProfilesForBook("test-book")
|
||||
if err != nil || len(voices) != 1 {
|
||||
t.Fatalf("voice profiles not persisted: %v %+v", err, voices)
|
||||
}
|
||||
if voices[0].SelfRef != "ваша служанка" || voices[0].NGLexicon != `["ладно"]` {
|
||||
t.Fatalf("voice profile columns did not round-trip: %+v", voices[0])
|
||||
}
|
||||
pairs, err := r.Store.AddressPairsForBook("test-book")
|
||||
if err != nil || len(pairs) != 1 || pairs[0].Register != "formal" {
|
||||
t.Fatalf("address pairs not persisted: %v %+v", err, pairs)
|
||||
}
|
||||
}
|
||||
|
|
@ -386,8 +386,14 @@ func (r *Runner) runDraftChunk(ctx context.Context, draftSnapshot string, ch chu
|
|||
if isFinalWave && !seq.flagged && seq.finalText != "" && isRuTarget(r.Book.TargetLang) {
|
||||
cheap = checks.RunCheapGates(ch.Text, seq.finalText, seq.finalText, r.cheapGateConfig())
|
||||
}
|
||||
var voice checks.VoiceResult
|
||||
var leaks []membank.SpoilerLeak
|
||||
if isFinalWave && !seq.flagged && seq.finalText != "" {
|
||||
voice = r.runVoiceChecks(r.baseMemory, ch.Chapter, ch.Text, seq.finalText)
|
||||
leaks = r.spoilerLeaks(r.baseMemory, memSel, seq.finalText)
|
||||
}
|
||||
if r.baseMemory != nil {
|
||||
if err := r.persistRetrievalState(draftSnapshot, ch, memSel, postMisses, outputChecked, cheap, seq.bankTelem, seq.bankProps); err != nil {
|
||||
if err := r.persistRetrievalState(draftSnapshot, ch, memSel, postMisses, outputChecked, cheap, seq.bankTelem, seq.bankProps, voice, leaks); err != nil {
|
||||
return seq, err
|
||||
}
|
||||
}
|
||||
|
|
@ -498,8 +504,24 @@ func (r *Runner) runEditUnit(ctx context.Context, editSnapshot string, unit edit
|
|||
"chapter", unit.Chapter, "leader_chunk", unit.FirstChunkIdx, "style_flags", cheap.Total())
|
||||
}
|
||||
}
|
||||
// The pack-19 flaggers, on the same shipped text and under the same "clean unit" condition as the
|
||||
// cheap gates: a flagged unit ships nothing, so measuring its prose would report defects nobody reads.
|
||||
var voice checks.VoiceResult
|
||||
var leaks []membank.SpoilerLeak
|
||||
if !flagged && finalText != "" {
|
||||
voice = r.runVoiceChecks(r.memory, unit.Chapter, leader.Text, finalText)
|
||||
if voice.Total() > 0 {
|
||||
r.Log.InfoContext(ctx, "voice flagger fired on the edit unit (observability, not a gate)",
|
||||
"chapter", unit.Chapter, "leader_chunk", unit.FirstChunkIdx,
|
||||
"voice_flags", voice.Total(), "attributed_replies", voice.Attributed, "version", checks.VoiceCheckVersion)
|
||||
}
|
||||
if leaks = r.spoilerLeaks(r.memory, editSel, finalText); len(leaks) > 0 {
|
||||
r.Log.WarnContext(ctx, "spoiler leak: a rendering this chapter must not know yet reached the output",
|
||||
"chapter", unit.Chapter, "leader_chunk", unit.FirstChunkIdx, "leaks", len(leaks), "first", leaks[0].Src)
|
||||
}
|
||||
}
|
||||
if r.memory != nil {
|
||||
if err := r.mergeUnitRetrievalState(unit.Chapter, unit.FirstChunkIdx, postMisses, outputChecked, cheap); err != nil {
|
||||
if err := r.mergeUnitRetrievalState(unit.Chapter, unit.FirstChunkIdx, postMisses, outputChecked, cheap, voice, leaks); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
|
@ -553,7 +575,7 @@ func (r *Runner) recordSkippedStages(ctx context.Context, staged []wavedStage, s
|
|||
// telemetry. A read-modify-write (single-writer safe; each unit owns a distinct leader chunk_idx) preserves
|
||||
// the draft fields and adds the unit-level post-check/style fields, so the read-models aggregate injection
|
||||
// per draft chunk and post-check/style per unit. A missing row (r.memory nil path) is a no-op.
|
||||
func (r *Runner) mergeUnitRetrievalState(chapter, firstChunkIdx int, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult) error {
|
||||
func (r *Runner) mergeUnitRetrievalState(chapter, firstChunkIdx int, misses membank.PostcheckResult, outputChecked bool, cheap checks.CheapGateResult, voice checks.VoiceResult, leaks []membank.SpoilerLeak) error {
|
||||
rs, err := r.Store.GetRetrievalState(r.Book.BookID, chapter, firstChunkIdx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline: read leader retrieval_state ch%d/chunk%d: %w", chapter, firstChunkIdx, err)
|
||||
|
|
@ -568,6 +590,7 @@ func (r *Runner) mergeUnitRetrievalState(chapter, firstChunkIdx int, misses memb
|
|||
rs.StyleDetail = string(b)
|
||||
}
|
||||
}
|
||||
setVoiceState(rs, voice, leaks)
|
||||
rs.NPostcheckMiss = 0
|
||||
rs.PostcheckDetail = ""
|
||||
if outputChecked {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,45 @@ package seed
|
|||
// File is a whole seed YAML document.
|
||||
type File struct {
|
||||
Terms []Term `yaml:"terms"`
|
||||
// Voices and Addresses are the two D21 record types added by pack-19. They are OWNER-CURATED only —
|
||||
// no miner emits them — so they have no counterpart on the mined-delta side of this schema.
|
||||
Voices []Voice `yaml:"voices"`
|
||||
Addresses []Address `yaml:"addresses"`
|
||||
}
|
||||
|
||||
// Voice is one character's voice profile for a chapter window (research/15 §1.5). Src/Sense name the
|
||||
// glossary term this profile belongs to — the same stable key the bank uses everywhere else.
|
||||
type Voice struct {
|
||||
Src string `yaml:"src"`
|
||||
Sense string `yaml:"sense"`
|
||||
// Register is one line of register+tone; SelfRef is the character's own self-designation (the field
|
||||
// D7 reserved as first_person); AddressDefault is the T/V default outside the pair registry.
|
||||
Register string `yaml:"register"`
|
||||
SelfRef string `yaml:"self_ref"`
|
||||
AddressDefault string `yaml:"address_default"` // informal|formal
|
||||
LexiconMarkers []string `yaml:"lexicon_markers"`
|
||||
NGLexicon []string `yaml:"ng_lexicon"`
|
||||
Exemplars []string `yaml:"exemplars"`
|
||||
Brightness string `yaml:"brightness"`
|
||||
SinceCh int `yaml:"since_ch"`
|
||||
UntilCh int `yaml:"until_ch"`
|
||||
}
|
||||
|
||||
// Address is one entry of the ты/вы transition journal: the register the SPEAKER uses toward the
|
||||
// ADDRESSEE over a chapter window. A register change is a new row with a non-overlapping window, never
|
||||
// an edit of this one — the switch is a story event.
|
||||
type Address struct {
|
||||
Speaker string `yaml:"speaker"`
|
||||
SpeakerSense string `yaml:"speaker_sense"`
|
||||
Addressee string `yaml:"addressee"`
|
||||
AddresseeSense string `yaml:"addressee_sense"`
|
||||
// Register is ABSTRACT (informal|formal): which target surfaces realise it is target data, so a
|
||||
// target language without a T/V distinction activates nothing.
|
||||
Register string `yaml:"register"`
|
||||
Form string `yaml:"form"`
|
||||
Closeness string `yaml:"closeness"`
|
||||
SinceCh int `yaml:"since_ch"`
|
||||
UntilCh int `yaml:"until_ch"`
|
||||
}
|
||||
|
||||
// Term is one seeded term: the source surface, its rendering and the metadata the bank
|
||||
|
|
|
|||
|
|
@ -98,15 +98,34 @@ type RetrievalState struct {
|
|||
// row is a candidate the model is entitled to reject.
|
||||
NUnverifiedShown int
|
||||
NUnverifiedFollowed int
|
||||
// NVoiceFlags / VoiceDetail are the deterministic voice flagger (pack-19, gates.voice): T/V
|
||||
// contradictions, flattened self-designations and forbidden lexemes in attributed replies. Zero for
|
||||
// every gate-off unit. NSpoilerLeaks / SpoilerLeakDetail are the reveal half of D21 п.3: a rendering
|
||||
// the spoiler window REJECTED for this chapter that reached the output anyway. Both are observability
|
||||
// (never a disposition) and re-derived each run, like every sibling on this row.
|
||||
NVoiceFlags int
|
||||
VoiceDetail string // JSON
|
||||
NSpoilerLeaks int
|
||||
SpoilerLeakDetail string // JSON
|
||||
}
|
||||
|
||||
// ReplaceGlossary replaces a book's ENTIRE glossary (rows + aliases) with the given
|
||||
// entries in ONE transaction, and appends a B1 revision record for every term whose
|
||||
// approved rendering (dst) changed since the previous set. Scoped to book_id, so other
|
||||
// books are untouched. Each entry's BookID must equal bookID. term_id linkage is
|
||||
// established within the tx: a glossary row is inserted, its rowid taken, then its
|
||||
// aliases inserted against that id (ids are fresh each replace — never hashed).
|
||||
// ReplaceGlossary replaces a book's glossary with no voice/address rows — the pre-pack-19 form, kept
|
||||
// because most callers (and every glossary-only test) have nothing else to write.
|
||||
func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error {
|
||||
return s.ReplaceBank(bookID, entries, nil, nil)
|
||||
}
|
||||
|
||||
// ReplaceBank replaces a book's ENTIRE bank — glossary rows + aliases + voice profiles + address pairs —
|
||||
// with the given content in ONE transaction, and appends a B1 revision record for every term whose
|
||||
// approved rendering (dst) changed since the previous set. Scoped to book_id, so other books are
|
||||
// untouched. Each entry's BookID must equal bookID. term_id linkage is established within the tx: a
|
||||
// glossary row is inserted, its rowid taken, then its aliases inserted against that id (ids are fresh
|
||||
// each replace — never hashed).
|
||||
//
|
||||
// The three record types share ONE transaction on purpose: they fold into one memory_version, so a
|
||||
// half-applied replace would make the materialized bank disagree with the hash the snapshot pins it by.
|
||||
// Voice/address rows carry no id linkage at all — they reference a character by the stable (src, sense).
|
||||
func (s *Store) ReplaceBank(bookID string, entries []GlossaryEntry, voices []VoiceProfile, pairs []AddressPair) error {
|
||||
ctx, cancel := opContext()
|
||||
defer cancel()
|
||||
tx, err := s.w.BeginTx(ctx, nil)
|
||||
|
|
@ -136,11 +155,10 @@ func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error {
|
|||
}
|
||||
rows.Close()
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM glossary_aliases WHERE book_id = ?`, bookID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM glossary WHERE book_id = ?`, bookID); err != nil {
|
||||
return err
|
||||
for _, table := range []string{"glossary_aliases", "glossary", "voice_profiles", "address_pairs"} {
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE book_id = ?`, bookID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
|
|
@ -179,6 +197,28 @@ func (s *Store) ReplaceGlossary(bookID string, entries []GlossaryEntry) error {
|
|||
}
|
||||
}
|
||||
}
|
||||
for _, v := range voices {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO voice_profiles (
|
||||
book_id, src, sense, register, self_ref, address_default,
|
||||
lexicon_markers, ng_lexicon, exemplars, brightness, since_ch, until_ch
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
bookID, v.Src, v.Sense, v.Register, v.SelfRef, v.AddressDefault,
|
||||
v.LexiconMarkers, v.NGLexicon, v.Exemplars, v.Brightness, v.SinceCh, v.UntilCh); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, p := range pairs {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO address_pairs (
|
||||
book_id, speaker_src, speaker_sense, addressee_src, addressee_sense,
|
||||
register, form, closeness, since_ch, until_ch
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?)`,
|
||||
bookID, p.SpeakerSrc, p.SpeakerSense, p.AddresseeSrc, p.AddresseeSense,
|
||||
p.Register, p.Form, p.Closeness, p.SinceCh, p.UntilCh); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
|
|
@ -265,8 +305,9 @@ func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
|
|||
embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
||||
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail,
|
||||
n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail,
|
||||
n_unverified_shown, n_unverified_followed, updated_at
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
|
||||
n_unverified_shown, n_unverified_followed,
|
||||
n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail, updated_at
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
|
||||
ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET
|
||||
snapshot_id = excluded.snapshot_id,
|
||||
n_exact_hits = excluded.n_exact_hits,
|
||||
|
|
@ -288,13 +329,18 @@ func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
|
|||
banknote_truncated = excluded.banknote_truncated,
|
||||
n_unverified_shown = excluded.n_unverified_shown,
|
||||
n_unverified_followed = excluded.n_unverified_followed,
|
||||
n_voice_flags = excluded.n_voice_flags,
|
||||
voice_detail = excluded.voice_detail,
|
||||
n_spoiler_leaks = excluded.n_spoiler_leaks,
|
||||
spoiler_leak_detail = excluded.spoiler_leak_detail,
|
||||
updated_at = excluded.updated_at`,
|
||||
rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID,
|
||||
rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted,
|
||||
rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs,
|
||||
rs.NStyleFlags, rs.StyleDetail, rs.NTrustGatedSuppress, rs.TrustGateDetail,
|
||||
rs.NBanknoteLines, rs.BanknoteParseFail, rs.BanknoteTruncated, rs.BanknoteDetail,
|
||||
rs.NUnverifiedShown, rs.NUnverifiedFollowed)
|
||||
rs.NUnverifiedShown, rs.NUnverifiedFollowed,
|
||||
rs.NVoiceFlags, rs.VoiceDetail, rs.NSpoilerLeaks, rs.SpoilerLeakDetail)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -308,14 +354,16 @@ func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*Retrie
|
|||
n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
||||
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail,
|
||||
n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail,
|
||||
n_unverified_shown, n_unverified_followed
|
||||
n_unverified_shown, n_unverified_followed,
|
||||
n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail
|
||||
FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`,
|
||||
bookID, chapter, chunkIdx).Scan(
|
||||
&rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked,
|
||||
&rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
|
||||
&rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail,
|
||||
&rs.NBanknoteLines, &rs.BanknoteParseFail, &rs.BanknoteTruncated, &rs.BanknoteDetail,
|
||||
&rs.NUnverifiedShown, &rs.NUnverifiedFollowed)
|
||||
&rs.NUnverifiedShown, &rs.NUnverifiedFollowed,
|
||||
&rs.NVoiceFlags, &rs.VoiceDetail, &rs.NSpoilerLeaks, &rs.SpoilerLeakDetail)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
@ -333,7 +381,8 @@ func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error)
|
|||
n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
||||
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail,
|
||||
n_banknote_lines, banknote_parse_fail, banknote_truncated, banknote_detail,
|
||||
n_unverified_shown, n_unverified_followed
|
||||
n_unverified_shown, n_unverified_followed,
|
||||
n_voice_flags, voice_detail, n_spoiler_leaks, spoiler_leak_detail
|
||||
FROM retrieval_state WHERE book_id = ?
|
||||
ORDER BY chapter, chunk_idx`,
|
||||
func(rows *sql.Rows) (RetrievalState, error) {
|
||||
|
|
@ -343,7 +392,8 @@ func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error)
|
|||
&rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
|
||||
&rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail,
|
||||
&rs.NBanknoteLines, &rs.BanknoteParseFail, &rs.BanknoteTruncated, &rs.BanknoteDetail,
|
||||
&rs.NUnverifiedShown, &rs.NUnverifiedFollowed)
|
||||
&rs.NUnverifiedShown, &rs.NUnverifiedFollowed,
|
||||
&rs.NVoiceFlags, &rs.VoiceDetail, &rs.NSpoilerLeaks, &rs.SpoilerLeakDetail)
|
||||
return rs, err
|
||||
}, bookID)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ var migrations = []string{
|
|||
type TEXT NOT NULL DEFAULT '', -- name|term|title|… alias-graph node type (B3)
|
||||
sense TEXT NOT NULL DEFAULT '', -- A3 polysemy disambiguator (part of the uniqueness key)
|
||||
gender TEXT NOT NULL DEFAULT '', -- male|female|hidden|'' (C2; hidden = gender-avoidant rendering until reveal)
|
||||
speech TEXT NOT NULL DEFAULT '', -- ты/вы register note (series-bible-lite; the transition JOURNAL is deferred)
|
||||
speech TEXT NOT NULL DEFAULT '', -- DEPRECATED (pack-19): superseded by the address_pairs journal (v13); never read, never hashed
|
||||
decl TEXT NOT NULL DEFAULT '', -- JSON {"invariant":bool,"forms":[...]}: the dst declension forms the post-check accepts (E1/E2 safety, decl = grammaticality mechanism, not a market feature)
|
||||
translit_policy TEXT NOT NULL DEFAULT '', -- B5 western-name-via-katakana (field only in v1; validator = Phase 2)
|
||||
first_person TEXT NOT NULL DEFAULT '', -- D7 first_person field (field only)
|
||||
|
|
@ -344,37 +344,101 @@ var migrations = []string{
|
|||
ALTER TABLE retrieval_state ADD COLUMN n_unverified_shown INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE retrieval_state ADD COLUMN n_unverified_followed INTEGER NOT NULL DEFAULT 0;
|
||||
`,
|
||||
// v13 (pack-19 / D39.55): the two D21 record types the bank was missing — the per-character VOICE
|
||||
// profile and the ordered ADDRESS pair. They are TABLES rather than glossary columns for one hard
|
||||
// reason: a character's term row and its voice profile share (src, sense, since_ch, until_ch), which
|
||||
// is the glossary UNIQUE key, so the profile could not coexist with the term it describes without
|
||||
// rebuilding that constraint — forbidden by the append-only discipline above.
|
||||
//
|
||||
// They are nevertheless BANK CONTENT, not a store beside the bank: their rows are replaced in the SAME
|
||||
// transaction as the glossary and fold into the SAME memory_version (conditionally — see
|
||||
// membank.ComputeVersionScopedIn). Giving them a snapshot key of their own would classify a signature
|
||||
// as moveOther and cost a whole edit wave instead of the touched units (pipeline/repin.go).
|
||||
//
|
||||
// address_pairs is the MATERIALIZATION of the ты/вы transition journal (D7-Amendment-1, D21 п.2), not a
|
||||
// second source of truth: a row IS a journal entry (register valid from since_ch to until_ch) and the
|
||||
// "current register at chapter N" is the projection membank computes, never a stored state. A ты↔вы
|
||||
// switch is a second row with a non-overlapping window, exactly like a spoiler handoff.
|
||||
//
|
||||
// Both reference characters by the STABLE key (src, sense) rather than glossary.id: that id is a fresh
|
||||
// autoincrement on every replace (see v5), so an id link would silently re-point after any re-seed.
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS voice_profiles (
|
||||
id INTEGER PRIMARY KEY,
|
||||
book_id TEXT NOT NULL,
|
||||
src TEXT NOT NULL, -- character identity, half 1 (the glossary term's src)
|
||||
sense TEXT NOT NULL DEFAULT '', -- character identity, half 2
|
||||
register TEXT NOT NULL DEFAULT '', -- one line: register + tone
|
||||
self_ref TEXT NOT NULL DEFAULT '', -- the character's own self-designation (D7 first_person)
|
||||
address_default TEXT NOT NULL DEFAULT '', -- informal|formal|'' — the T/V default outside the pair registry
|
||||
lexicon_markers TEXT NOT NULL DEFAULT '', -- JSON []string: characteristic words
|
||||
ng_lexicon TEXT NOT NULL DEFAULT '', -- JSON []string: words this character never says
|
||||
exemplars TEXT NOT NULL DEFAULT '', -- JSON []string: 3-5 approved target replies
|
||||
brightness TEXT NOT NULL DEFAULT '', -- injection priority hint (D21 п.1в: hypothesis, not a rule)
|
||||
since_ch INTEGER NOT NULL DEFAULT 0, -- manner-change window, same axis as the glossary spoiler window
|
||||
until_ch INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE (book_id, src, sense, since_ch, until_ch)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS voice_profiles_book_idx ON voice_profiles (book_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS address_pairs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
book_id TEXT NOT NULL,
|
||||
speaker_src TEXT NOT NULL,
|
||||
speaker_sense TEXT NOT NULL DEFAULT '',
|
||||
addressee_src TEXT NOT NULL,
|
||||
addressee_sense TEXT NOT NULL DEFAULT '',
|
||||
register TEXT NOT NULL, -- informal|formal — ABSTRACT; the surfaces are target data
|
||||
form TEXT NOT NULL DEFAULT '', -- the address form («молодой господин», a title)
|
||||
closeness TEXT NOT NULL DEFAULT '', -- one word of hierarchy/closeness (why the register is what it is)
|
||||
since_ch INTEGER NOT NULL DEFAULT 0, -- a ты<->вы switch is a STORY event: a second row, new window
|
||||
until_ch INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE (book_id, speaker_src, speaker_sense, addressee_src, addressee_sense, since_ch, until_ch)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS address_pairs_book_idx ON address_pairs (book_id);
|
||||
`,
|
||||
// v14 (pack-19): the per-chunk observability of the two new $0 flaggers, riding the same
|
||||
// recomputed-every-run row as the glossary post-check and the cheap style gates.
|
||||
//
|
||||
// • n_voice_flags / voice_detail — the deterministic T/V + voice-marker flagger (gates.voice);
|
||||
// • n_spoiler_leaks / spoiler_leak_detail — a rendering the spoiler window REJECTED for this
|
||||
// chapter that appeared in the output anyway (the reveal half of D21 п.3).
|
||||
//
|
||||
// Neither is ever a disposition, and both are re-derived deterministically each run (self-healing on
|
||||
// resume), so they are observability exactly like their siblings. Additive with defaults ⇒ every prior
|
||||
// row and the WIRE are unchanged.
|
||||
`
|
||||
ALTER TABLE retrieval_state ADD COLUMN n_voice_flags INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE retrieval_state ADD COLUMN voice_detail TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE retrieval_state ADD COLUMN n_spoiler_leaks INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE retrieval_state ADD COLUMN spoiler_leak_detail TEXT NOT NULL DEFAULT '';
|
||||
`,
|
||||
}
|
||||
|
||||
// DESIGN NOTE (D21.10 — reserved memory-bank v2 record types; Phase 2, NOT a migration, NOT code).
|
||||
// DESIGN NOTE (D21.10 → BUILT by pack-19 / D39.55; kept as the record of what each type is FOR).
|
||||
//
|
||||
// research/15 (D21) ratified three future memory MECHANISMS whose storage will extend the bank. This
|
||||
// note reserves their shape so a later Phase 2 migration lands cleanly against the existing determinism/
|
||||
// snapshot discipline. It is DELIBERATELY not appended to `migrations` above: the mechanisms are Phase 2
|
||||
// (D21.1–3), gated on the pilot and on D15.2 landing — reserving a column now would be dead schema.
|
||||
// research/15 (D21) ratified three memory MECHANISMS extending the bank. Two are now schema (v13); the
|
||||
// third turned out to need none.
|
||||
//
|
||||
// • voice_profile (D21.1) — per-character voice exemplars + descriptors, injected DRAFT-side
|
||||
// (cap ≤300 tok/chunk; injection priority CONFIRMED-glossary > address_pair > voice_profile >
|
||||
// reveal-aliases). FROZEN per-job until D15.2 lands (self-populating exemplars = mid-run memory
|
||||
// append = book re-pay under D8/D15.1). Likely a `voice_profiles` table keyed (book_id, term_id)
|
||||
// with exemplar/descriptor columns + a since_ch/until_ch window mirroring the glossary spoiler
|
||||
// axis. Injected content rides content_hash; any live verdict rides verdictSnapshotID (D15.2).
|
||||
// • voice_profile (D21.1) — BUILT as `voice_profiles` (v13), keyed by the STABLE (src, sense) rather
|
||||
// than the reserved (book_id, term_id): glossary.id is a fresh autoincrement per replace. Still
|
||||
// FROZEN per-job (self-populating exemplars = mid-run memory append = book re-pay under D8/D15.1);
|
||||
// the condition for lifting it — D15.2 landing — has NOT happened (verdictSnapshotID/guard_hash do
|
||||
// not exist in this engine, only in comments).
|
||||
//
|
||||
// • address_pair (D21.2) — the ты/вы register for an ORDERED character pair. It is the
|
||||
// MATERIALIZATION of the D7-Amendment-1 ты/вы transition JOURNAL, NOT a second store: the single
|
||||
// source of truth is the journal (a ты↔вы switch is a STORY event), and address_pair is only its
|
||||
// resolved current-state PROJECTION (recomputable, self-healing). Deterministic flagger
|
||||
// unconditionally; injection only after the hard-pairs probe (lift on easy pairs = 0 — a
|
||||
// flagger-only outcome is legitimate). Likely `address_pairs` (book_id, term_id_a, term_id_b,
|
||||
// register, since_ch) DERIVED from a `speech_transitions` journal — must not duplicate the
|
||||
// journal's authority (glossary.speech today is a static note; the journal supersedes it in Phase 2).
|
||||
// • address_pair (D21.2) — BUILT as `address_pairs` (v13). The reserved shape guessed a derived
|
||||
// projection over a separate `speech_transitions` journal; the built one collapses the two, because a
|
||||
// windowed row IS a journal entry and the "register at chapter N" projection is computed, never
|
||||
// stored. So there is still exactly one source of truth, with one table instead of two.
|
||||
// `glossary.speech` (the static v5 note) is superseded and DEPRECATED — see its column comment.
|
||||
//
|
||||
// • reveal_ch + pre/post-reveal aliases (D21.3) — a reveal window (`reveal_ch`) plus alias sets
|
||||
// valid BEFORE vs AFTER the reveal (gender/identity twist: a post-reveal dst must not leak before
|
||||
// reveal_ch — same self-inflicted-spoiler logic as since_ch/until_ch windows). Likely a `reveal_ch`
|
||||
// column on glossary + alias rows tagged phase=pre|post, so injection selects by current_ch vs reveal_ch.
|
||||
//
|
||||
// None of these is a column today; Phase 2 implements them behind their own migration + tests.
|
||||
// • reveal_ch + pre/post-reveal aliases (D21.3) — NOT built, and deliberately: the pre/post rendering
|
||||
// pair is already expressible as two glossary rows with NON-OVERLAPPING windows (the UNIQUE key
|
||||
// admits them, membank.windowsOverlap permits them, spoilerBlocked picks exactly one), and aliases
|
||||
// inherit their row's window, so a phase tag would be a third way to say what two already say. What
|
||||
// was genuinely missing is the LEAK CHECK — a rejected rendering appearing in the output anyway —
|
||||
// which is membank.SpoilerLeaks over the selection's Rejected set, no schema at all. Scope amended
|
||||
// by D39.55 (form, not intent).
|
||||
|
||||
// migrate runs all pending migrations on the write pool, one transaction per
|
||||
// step, recording each in schema_version.
|
||||
|
|
|
|||
74
backend/internal/store/voice.go
Normal file
74
backend/internal/store/voice.go
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
package store
|
||||
|
||||
import "database/sql"
|
||||
|
||||
// voice.go: durable storage for the two D21 bank record types added by pack-19 (schema v13) — the
|
||||
// per-character VOICE profile and the ordered ADDRESS pair. Dumb storage like glossary.go: this layer
|
||||
// replaces and reads back, the pipeline owns validation, projection and matching.
|
||||
//
|
||||
// Both are BANK CONTENT: they are replaced in the same transaction as the glossary (ReplaceBank) and
|
||||
// fold into the same memory_version, so a signature on either is a bank-only snapshot move and stays
|
||||
// re-pinnable per unit (pipeline/repin.go) instead of re-buying an edit wave.
|
||||
|
||||
// VoiceProfile is one character's voice record for a chapter window (research/15 §1.5). The character is
|
||||
// identified by the STABLE (Src, Sense) key of its glossary term — never by glossary.id, which is a fresh
|
||||
// autoincrement on every replace. The list-valued columns are stored as JSON; membank owns the encoding.
|
||||
type VoiceProfile struct {
|
||||
Src string
|
||||
Sense string
|
||||
Register string
|
||||
SelfRef string
|
||||
AddressDefault string // informal|formal|"" — the T/V default outside the pair registry
|
||||
LexiconMarkers string // JSON []string
|
||||
NGLexicon string // JSON []string
|
||||
Exemplars string // JSON []string
|
||||
Brightness string
|
||||
SinceCh int
|
||||
UntilCh int
|
||||
}
|
||||
|
||||
// AddressPair is one ORDERED (speaker → addressee) register record for a chapter window — one entry of
|
||||
// the ты/вы transition journal (D7-Amendment-1). Register is ABSTRACT (informal|formal); which target
|
||||
// surfaces realise it is target data, so a target without a T/V distinction simply matches nothing.
|
||||
type AddressPair struct {
|
||||
SpeakerSrc string
|
||||
SpeakerSense string
|
||||
AddresseeSrc string
|
||||
AddresseeSense string
|
||||
Register string
|
||||
Form string
|
||||
Closeness string
|
||||
SinceCh int
|
||||
UntilCh int
|
||||
}
|
||||
|
||||
// VoiceProfilesForBook returns a book's voice profiles in a DETERMINISTIC order — the stable
|
||||
// materialization the bank version hashes over.
|
||||
func (s *Store) VoiceProfilesForBook(bookID string) ([]VoiceProfile, error) {
|
||||
return queryAll(s.r, `
|
||||
SELECT src, sense, register, self_ref, address_default,
|
||||
lexicon_markers, ng_lexicon, exemplars, brightness, since_ch, until_ch
|
||||
FROM voice_profiles WHERE book_id = ?
|
||||
ORDER BY src, sense, since_ch, until_ch`,
|
||||
func(rows *sql.Rows) (VoiceProfile, error) {
|
||||
var v VoiceProfile
|
||||
err := rows.Scan(&v.Src, &v.Sense, &v.Register, &v.SelfRef, &v.AddressDefault,
|
||||
&v.LexiconMarkers, &v.NGLexicon, &v.Exemplars, &v.Brightness, &v.SinceCh, &v.UntilCh)
|
||||
return v, err
|
||||
}, bookID)
|
||||
}
|
||||
|
||||
// AddressPairsForBook returns a book's address-register journal in a DETERMINISTIC order.
|
||||
func (s *Store) AddressPairsForBook(bookID string) ([]AddressPair, error) {
|
||||
return queryAll(s.r, `
|
||||
SELECT speaker_src, speaker_sense, addressee_src, addressee_sense,
|
||||
register, form, closeness, since_ch, until_ch
|
||||
FROM address_pairs WHERE book_id = ?
|
||||
ORDER BY speaker_src, speaker_sense, addressee_src, addressee_sense, since_ch, until_ch`,
|
||||
func(rows *sql.Rows) (AddressPair, error) {
|
||||
var p AddressPair
|
||||
err := rows.Scan(&p.SpeakerSrc, &p.SpeakerSense, &p.AddresseeSrc, &p.AddresseeSense,
|
||||
&p.Register, &p.Form, &p.Closeness, &p.SinceCh, &p.UntilCh)
|
||||
return p, err
|
||||
}, bookID)
|
||||
}
|
||||
116
backend/internal/store/voice_test.go
Normal file
116
backend/internal/store/voice_test.go
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// voice_test.go: the pack-19 schema, on the two axes that can only be checked by execution — that an
|
||||
// EXISTING database upgrades onto it, and that the bank's three record types are replaced as one unit.
|
||||
|
||||
// The migration that matters is not the fresh one (every test creates that) but the UPGRADE: a stand
|
||||
// database already at an older version must gain the pack-19 tables and columns without losing a row.
|
||||
func TestMigrationUpgradesAnOlderDatabase(t *testing.T) {
|
||||
const olderVersion = 12 // the schema head before pack-19 (v13 tables + v14 retrieval-state columns)
|
||||
if len(migrations) <= olderVersion {
|
||||
t.Fatalf("this test assumes pack-19 appended to a %d-step schema, got %d", olderVersion, len(migrations))
|
||||
}
|
||||
path := t.TempDir() + "/old.db"
|
||||
// Build a database at the OLD head by applying only the old steps through a raw connection.
|
||||
raw, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := raw.Exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < olderVersion; i++ {
|
||||
if _, err := raw.Exec(migrations[i]); err != nil {
|
||||
t.Fatalf("apply old migration %d: %v", i+1, err)
|
||||
}
|
||||
if _, err := raw.Exec(`INSERT INTO schema_version (version) VALUES (?)`, i+1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A row written under the OLD schema must survive the upgrade untouched.
|
||||
if _, err := raw.Exec(`INSERT INTO glossary (book_id, src, dst, status) VALUES ('b','旧','старое','approved')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Premise check: at the OLD head the pack-19 schema must genuinely be absent, or this test would
|
||||
// prove nothing about upgrading.
|
||||
if _, qerr := raw.Query(`SELECT n_voice_flags FROM retrieval_state`); qerr == nil {
|
||||
t.Fatal("the old schema already has the pack-19 columns — the upgrade is not being exercised")
|
||||
}
|
||||
if _, qerr := raw.Query(`SELECT 1 FROM voice_profiles`); qerr == nil {
|
||||
t.Fatal("the old schema already has voice_profiles — the upgrade is not being exercised")
|
||||
}
|
||||
if err := raw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s, err := Open(path) // runs the pending migrations
|
||||
if err != nil {
|
||||
t.Fatalf("an existing database must upgrade onto the pack-19 schema: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
rows, err := s.GlossaryForBook("b")
|
||||
if err != nil || len(rows) != 1 || rows[0].Dst != "старое" {
|
||||
t.Fatalf("the pre-existing row must survive the upgrade: %v %+v", err, rows)
|
||||
}
|
||||
// The new tables and columns are usable, not merely present.
|
||||
if err := s.ReplaceBank("b", rows,
|
||||
[]VoiceProfile{{Src: "旧", SelfRef: "ваша служанка"}},
|
||||
[]AddressPair{{SpeakerSrc: "旧", AddresseeSrc: "новое", Register: "formal"}}); err != nil {
|
||||
t.Fatalf("write to the upgraded schema: %v", err)
|
||||
}
|
||||
if v, err := s.VoiceProfilesForBook("b"); err != nil || len(v) != 1 {
|
||||
t.Fatalf("voice_profiles unusable after the upgrade: %v %+v", err, v)
|
||||
}
|
||||
if err := s.UpsertRetrievalState(RetrievalState{
|
||||
BookID: "b", Chapter: 1, ChunkIdx: 0, SnapshotID: "s", NVoiceFlags: 3, VoiceDetail: `{"replies":2}`,
|
||||
NSpoilerLeaks: 1, SpoilerLeakDetail: `[{"src":"旧"}]`,
|
||||
}); err != nil {
|
||||
t.Fatalf("write the new retrieval-state columns after the upgrade: %v", err)
|
||||
}
|
||||
got, err := s.GetRetrievalState("b", 1, 0)
|
||||
if err != nil || got == nil || got.NVoiceFlags != 3 || got.NSpoilerLeaks != 1 || got.VoiceDetail == "" {
|
||||
t.Fatalf("the new columns did not round-trip after the upgrade: %v %+v", err, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bank is replaced as ONE unit: a re-seed that drops a profile must drop it from the store too, or a
|
||||
// stale row would keep feeding the flagger a character the seed no longer signs.
|
||||
func TestReplaceBankIsAFullReplaceOfAllThreeRecordTypes(t *testing.T) {
|
||||
s, _ := openTemp(t)
|
||||
terms := []GlossaryEntry{{BookID: "b", Src: "方源", Dst: "Фан Юань", Status: "approved"}}
|
||||
if err := s.ReplaceBank("b", terms,
|
||||
[]VoiceProfile{{Src: "方源", SelfRef: "первый"}, {Src: "白凝冰", SelfRef: "второй"}},
|
||||
[]AddressPair{{SpeakerSrc: "方源", AddresseeSrc: "白凝冰", Register: "formal"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, _ := s.VoiceProfilesForBook("b"); len(v) != 2 {
|
||||
t.Fatalf("setup: want 2 profiles, got %d", len(v))
|
||||
}
|
||||
// Re-seed with ONE profile and no pairs at all.
|
||||
if err := s.ReplaceBank("b", terms, []VoiceProfile{{Src: "方源", SelfRef: "первый"}}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := s.VoiceProfilesForBook("b")
|
||||
if err != nil || len(v) != 1 || v[0].Src != "方源" {
|
||||
t.Fatalf("the dropped profile lingered: %v %+v", err, v)
|
||||
}
|
||||
if p, _ := s.AddressPairsForBook("b"); len(p) != 0 {
|
||||
t.Fatalf("the dropped address pairs lingered: %+v", p)
|
||||
}
|
||||
// Another book's content is untouched by a replace scoped to this one.
|
||||
if err := s.ReplaceBank("other", nil, []VoiceProfile{{Src: "X", SelfRef: "чужой"}}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ReplaceBank("b", terms, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o, _ := s.VoiceProfilesForBook("other"); len(o) != 1 {
|
||||
t.Fatalf("a replace must be scoped to its book, got %+v", o)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
# Журнал прогресса
|
||||
|
||||
> **⟶ ТЕКУЩЕЕ СОСТОЯНИЕ** (на 2026-07-26: пак-20 закрыт ЦЕЛИКОМ D39.41–53 — терминолог · флаговый стоп · двухсекционная инъекция · точечная ре-редактура · языковой экран; жанровый словарь отменён D39.47; полигон-пакеты 6/7/8 закрыты). **Источник истины по РЕШЕНИЯМ — `architecture/05-decisions-log.md` (D1–D39.53); этот файл — ЖУРНАЛ.**
|
||||
> - **Фазы/курс:** Ф0 ✅ · Ф1-инфра ✅ (D20–D28; golden = инвариант №8) · арка «качество-первым» D26→D38.5 закрыта · **арх-ресет D39 исполнен ЦЕЛИКОМ** (7 слоёв → программа D39.1–D39.10 → паки 11–16, D39.13–D39.24; статус слоёв — шапка-таблица `architecture/09-target-architecture.md`) · пере-прогон rerun2 прочитан (D39.20: операционка живьём, $0-резюм ×3 арма, $6.63<$15; планка ≤2 НЕ пройдена; анти-корреляция гладкость↔верность ×3 → mistral-редактор вон) · эксп ЗАКРЫТ (D39.22), итерация №2 отложена · **курс = разработка бэкенда: пак-17 «канал B вживую» ЗАКРЫТ ЦЕЛИКОМ (дизайн D39.26 · стройка D39.27 · дельта `allow` D39.28, 25.07); вокабуляр = ПАРА `violence`/`sexually-explicit`, остаток — ФАКТЫ ToS в зоне полигона (fail-closed до них)** (D39.25: в движке НЕТ понятия «18+» — generic content-labels × provider-capabilities; лейблы вне хешей · резолв до валидации · один хоп `chain[0]` · гранулярность = книга) → **КУРС АМЕНДИРОВАН ВЛАДЕЛЬЦЕМ 25.07 (D39.33): МАСШТАБ — ПОСЛЕДНИМ, после достройки алгоритмического идеала; доказательство эффективности крупных изменений — МИНИ-ПРОГОНЫ ~10 глав (детерминированные чекеры и скан остатка первыми, судья — только когда от него зависит решение).** **ОЧЕРЕДЬ (D39.40/51/53, всё до неё исполнено):** пак-19 «голос/состояние» → ХОЛОДНЫЙ мини-прогон (пустой сид, отдельная БД) → пак-21 «чекеры» → свип гипотез → добор идеала (4 вопроса границы D39.33 у владельца) → МАСШТАБ целой книги (7,78 млн симв., ~2284 раздела) → пилот Ф2.5. Хроника треков A/B, exp15/16 и стройки паков — архивы ниже + D-лог.
|
||||
> - **Фазы/курс:** Ф0 ✅ · Ф1-инфра ✅ (D20–D28; golden = инвариант №8) · арка «качество-первым» D26→D38.5 закрыта · **арх-ресет D39 исполнен ЦЕЛИКОМ** (7 слоёв → программа D39.1–D39.10 → паки 11–16, D39.13–D39.24; статус слоёв — шапка-таблица `architecture/09-target-architecture.md`) · пере-прогон rerun2 прочитан (D39.20: операционка живьём, $0-резюм ×3 арма, $6.63<$15; планка ≤2 НЕ пройдена; анти-корреляция гладкость↔верность ×3 → mistral-редактор вон) · эксп ЗАКРЫТ (D39.22), итерация №2 отложена · **курс = разработка бэкенда: пак-17 «канал B вживую» ЗАКРЫТ ЦЕЛИКОМ (дизайн D39.26 · стройка D39.27 · дельта `allow` D39.28, 25.07); вокабуляр = ПАРА `violence`/`sexually-explicit`, остаток — ФАКТЫ ToS в зоне полигона (fail-closed до них)** (D39.25: в движке НЕТ понятия «18+» — generic content-labels × provider-capabilities; лейблы вне хешей · резолв до валидации · один хоп `chain[0]` · гранулярность = книга) → **КУРС АМЕНДИРОВАН ВЛАДЕЛЬЦЕМ 25.07 (D39.33): МАСШТАБ — ПОСЛЕДНИМ, после достройки алгоритмического идеала; доказательство эффективности крупных изменений — МИНИ-ПРОГОНЫ ~10 глав (детерминированные чекеры и скан остатка первыми, судья — только когда от него зависит решение).** **ОЧЕРЕДЬ (D39.40/51/53, актуализация 31.07):** пак-19 «голос/состояние» ЗАКРЫТ ЦЕЛИКОМ (D39.54–56, вкл. малую пачку 14а+14б) → ХОЛОДНЫЙ мини-прогон (пустой сид, отдельная БД; + первое измерение осей голоса `gates.voice`) → пак-21 «чекеры» → свип гипотез → добор идеала (4 вопроса границы D39.33 у владельца) → МАСШТАБ целой книги (7,78 млн симв., ~2284 раздела) → пилот Ф2.5. Хроника треков A/B, exp15/16 и стройки паков — архивы ниже + D-лог.
|
||||
> - **Стек:** draft deepseek-v4-flash (thinking ON, +банкнота `translator-banknote.md`) → **терминолог** (роль пар-пака, deepseek-v4-flash, батчи, языковой экран `target_script` — D39.42/45/51/53) → **editor deepseek-v4-pro БИЛИНГВ ИНТЕРИМ (D39.22; glm-5 резерв)**, промпт v3-discourse, инъекция двухсекционная (D39.45) → судья gemini-3.1-pro-preview (Ф2, полигон); канал B Mistral+grok; 7 ключей; ~$0.85/ранобэ (D30.4).
|
||||
> - **ЕДИНЫЙ БЭКЛОГ — секция «Бэклог» ниже (введён 26.07 по решению владельца: одна таблица вместо наслоений; ревью-пасс оркестратора сверил её с коммитами и обсуждённым, добавил 7 строк). Норма прежняя: каждая петля обязана иметь диспозицию (решено / отложено-с-записью / отклонено); ведёт оркестратор. Историческая развёртка петель до 26.07 — в git-истории этого файла и в D-логе.**
|
||||
> - Архивы хроники: `archive/PROGRESS-2026-07-04-10.md` (D31) · `archive/PROGRESS-2026-07-10-13.md` (D39.6-гигиена) · `archive/PROGRESS-2026-07-13-25.md` (слайс 25.07: стройка паков 11–16, rerun2). Записи ниже — живой хвост.
|
||||
|
|
@ -27,9 +27,7 @@
|
|||
| 12 | `postcheck_gate` в hard-gate — требует пере-замера recall (recall чекеров теперь измерен и низок) | владелец | когда-нибудь (после пака-21) | отдельное решение | D39.34(2), D24.4+D28.1, D39.39 |
|
||||
| 13 | Включение платной петли ремонта + per-class исходы ремонта (схемное решение принимается вместе) — вернуться при расширении классов детекторов и ненулевом остатке | владелец | когда-нибудь | отдельное решение | D39.38, D39.39(а), CURRENT-STATE |
|
||||
| 13а | Веса рубрики K1–K12 не выбраны (Q1: «по данным фазы B»; фаза B прошла на BWS без весов — выбор нужен к агрегатной приёмке качества книги) | владелец | когда-нибудь (к пилоту) | Ф2.5 / отдельное решение | D39.44 Q1, D39.46 |
|
||||
| **— ПАК-19 (текущий шаг очереди) —** | | | | | |
|
||||
| 14 | Пак-19 «голос/состояние» (D21, урезанный скоуп) — выдан, первый в очереди D39.53; малая пачка 14а+14б ЗАКРЫТА D39.54 | бэкенд | блокер-очереди | пак-19 | D39.38, D39.40, D39.53 |
|
||||
| 15 | Нарративная гендер-аннотация (since_ch-смены + перспектива, не булево поле) — вход пака-19 | бэкенд | блокер-очереди | пак-19 | D39.34(8) |
|
||||
| 13б | Шипить ли `configs/langpacks/zh/speech-cue.txt` (формат+загрузчик готовы; байты двигают `LangpackVersion` = перекупка ОБЕИХ волн всех zh-книг; пока не шипнут — `SourceReplies`=0, знаменатель не измерен) | владелец | скоро (к холодному мини-прогону) | отдельное решение | D39.56, PACK19_BUILD §7.2 |
|
||||
| **— ХОЛОДНЫЙ МИНИ-ПРОГОН (после пака-19) —** | | | | | |
|
||||
| 16 | Холодный старт с пустым банком — единственная неизмеренная цифра цены/покрытия | бэкенд/полигон | блокер-очереди | холодный мини-прогон | D39.50 п.7, D39.51, POLYGON_PREMEASURE §5 |
|
||||
| 17 | Совместный recall майнер∪банкнота против сида живьём (замер 1 пакета-8 оказался неизмерим — цензура черновиков) | полигон | блокер-очереди | холодный мини-прогон | D39.48, D39.51, POLYGON_PREMEASURE §1.2 |
|
||||
|
|
@ -74,6 +72,7 @@
|
|||
| 48а | Серийный шаринг банка — РЕАЛИЗАЦИЯ (экспорт подписанного глоссария как сид-YAML следующей книги серии; спойлер-окна плющатся в since_ch=0); контракт round-trip держится тестом, самой фичи нет | бэкенд | когда-нибудь (перед второй книгой серии) | отдельное решение | D39.42 задел (а), PACK20_BANK_BUILD З-а |
|
||||
| 48б | Импорт «оригинал + чужой перевод» как режим данных терминолога (target-агностичный вход есть контрактом; сам режим + провенанс источника + юр-сторона в юр-пакете) | бэкенд/владелец | когда-нибудь | отдельное решение + Ф2.5 (юр) | D39.42 задел (б), PACK20_BANK_BUILD З-б |
|
||||
| 49 | Этапы Б+В спеки D15.2 (content-addressed resume / `guard_hash` — D39.31 сознательно не строил) | бэкенд | когда-нибудь | отдельное решение | D39.34(4), D33 п.5 |
|
||||
| 49а | ALTER-шаги миграций v8–v14 не идемпотентны вопреки шапке `migrate.go:9-11` (полу-применённая БД не сходится; счётчик версий скрывает) — находка критика полноты пака-19 | бэкенд | когда-нибудь | отдельное решение | D39.56, PACK19_BUILD §6.9 |
|
||||
| 50 | F3-остаток идемпотентности | бэкенд | когда-нибудь | отдельное решение | D39.34(4) |
|
||||
| 51 | `human_override` LOCK (D29 п.1г, в коде 0 вхождений) | бэкенд | когда-нибудь | отдельное решение | D39.34(4) |
|
||||
| 52 | Ф2-гейты вне идеала: морфо-гейты канцелярита + L1-лемматизация (python-сайдкар) · полный OpenCC · опциональный пре-перевод гейт | бэкенд | когда-нибудь | отдельное решение | D39.34(4) |
|
||||
|
|
@ -99,6 +98,10 @@
|
|||
| 69 | Gemini API «под-18» обязательство на конечный продукт (независимо от лейблов) | владелец | когда-нибудь | Ф3 / отдельное решение | D39.32, D39.34(7) |
|
||||
| 70 | Action-security gate перед выдачей tools/webfetch (D25 п.5) | бэкенд | когда-нибудь | Ф3 | D39.34(7) |
|
||||
| 71 | Планы research/22: epub-tag-rewrite · Q7-леджер | бэкенд | когда-нибудь | Ф3 | D39.34(7) |
|
||||
## Оркестратор №9 — СТРОЙКА ПАКА-19 ПРИНЯТА И ЗАЛЕНДЕНА (D39.56), ПАК-19 ЗАКРЫТ ЦЕЛИКОМ, 31.07
|
||||
|
||||
Фаза 2 сдана и принята короткой приёмкой (норма 30.07): сьют `-race` перегнан — зелёный; `repin.go`/`snapshot.go`/golden не тронуты (запреты (д)/(и) держатся); мутации сессии 23/23. Построено: таблицы v13/v14 + условный фолд (ни байта при выключенной инъекции) · ты/вы-флаггер A–C + D-индикатор · флаггер спойлер-утечки · данные V5 (5 категорий `target-ru` + опциональный `speech-cue`). Инъекция голоса — на бумаге до полигон-эксперимента (D21 п.2). Ратифицированное отклонение: атрибуция целевая, не исходная. Бэклог: 14/15 закрыты; +13б (`speech-cue.txt` — владельцу, к холодному прогону), +49а (миграции неидемпотентны). Очередь: ХОЛОДНЫЙ мини-прогон — теперь несёт и первое измерение осей голоса (`gates.voice` ON, строка 24). Отчёт: `archive/reports/PACK19_BUILD_2026-07-30.md` (ревью-шапка).
|
||||
|
||||
## Оркестратор №9 — ДИЗАЙН ПАКА-19 РАТИФИЦИРОВАН (D39.55), стройка ждёт «идём» владельца, 30.07
|
||||
|
||||
Фаза 1 принята: 82 клейма — 71 CONFIRMED / 9 PARTIAL / 2 REFUTED (полнота, не факты). Ратифицировано: типы записей банка + условный фолд в `memory_version` · CONFIRMED-only голос/обращения · свой гейт `gates.voice` без фолда · reveal сокращён до флаггера утечки (амендмент D21 п.3) · слот инъекции не выбран (амендмент D21 п.1г, решит полигон) · D21 п.8 model-bound. Девять новых «ровно так» фазы 2 — в D39.55. Отчёт: `archive/reports/PACK19_DESIGN_2026-07-30.md`.
|
||||
|
|
|
|||
|
|
@ -1176,3 +1176,7 @@ API-529-долг закрыт: 8-осевой refute-by-default воркфлоу
|
|||
## D39.55 — ДИЗАЙН ПАКА-19 «голос/состояние» РАТИФИЦИРОВАН С ПОПРАВКАМИ (фаза 1, $0, стройка после «идём» владельца): три сущности = ТИПЫ ЗАПИСЕЙ БАНКА с УСЛОВНЫМ фолдом в `memory_version` (наследование точечной ре-редактуры подтверждено исполнением) · голос/обращения CONFIRMED-only, порядок инъекции «обязывающее перед необязывающим» · флаггер = свой гейт `gates.voice` БЕЗ фолда (версия логируется с прогоном) · оси A–C полноценно, ось D (регистр пары) отдельным счётчиком-индикатором · reveal-скоуп СОКРАЩЁН до флаггера спойлер-утечки без `reveal_ch` (амендмент ФОРМЫ D21 п.3, интент сохранён `spoilerBlocked`) · D21 п.1г амендирован: слот инъекции НЕ выбран, оба провода, решит полигон-эксперимент · D21 п.8 помечен model-bound (действующий порядок: драфт ~$0.04, редактор ~$0.07–0.10) (30.07, оркестратор №9). ✅
|
||||
|
||||
Приёмка: воркфлоу 7 верификаторов, 82 клейма = 71 CONFIRMED / 9 PARTIAL / 2 REFUTED (оба — полнота отчёта; поправки фактов — в ревью-шапке отчёта). Диспозиции пингов: (1) сокращение reveal ратифицировано; (2) `glossary.speech` → deprecated (журнал-строки вместо статической матрицы), `first_person` → переиспользуется полем профиля; (3) цифра D21 п.8 контрактно привязана к модели; (4) «драфт-слот дефолт» снят; (5) структурная неполнота оси D зафиксирована ДО фазы 2. **Новые «РОВНО ТАК» фазы 2 (из проб верификации):** (а) бюджет инъекции входит в денежный потолок/reserve — дозакрыть в отчёте стройки; (б) критик полноты в отчёте фазы 2 обязателен; (в) `Materialize` получает фильтр типа записи — строки голоса/обращений НЕ попадают в AC-матчер/глоссарную инъекцию как обычные строки (`memory.go:264-270` матчит всё с непустым Dst); (г) условный voice-тег копирует порядок «исключение скоупом ДО подсчёта условия» (`memory.go:352-357`); (д) `voiceSnap`/отдельный ключ снапшота — ЗАПРЕЩЁН (ломает `repin.go:72`); (е) `address_pair` хранится по стабильному ключу (src+sense), не по autoincrement `glossary.id`; (ж) sticky-carry, отклонённый окном, добавляется в `Selection.Rejected` (`memory.go:495-497` — сейчас молчаливый дроп); (з) ВКЛЮЧЕНИЕ инъекции = полный `--resnapshot` книги через consent-гейт, bank-only-класс на самом флипе не ожидается (omitempty защищает только добавление поля, не флип 0→300); (и) v1 переиспользует `GlossaryTokenBudget` в сигнатуре отбора — отдельный voice-бюджет потребовал бы зеркала в `repin.go:186`. Опционально фазе 2: линт «gap в окнах одного src» (сид грузится молча с дырой). Отчёт: `archive/reports/PACK19_DESIGN_2026-07-30.md` (ревью-шапка).
|
||||
|
||||
## D39.56 — СТРОЙКА ПАКА-19 ПРИНЯТА И ЗАЛЕНДЕНА ($0, +672/−71, 16+7 файлов; ПАК-19 ЗАКРЫТ ЦЕЛИКОМ): голос/обращения = таблицы v13/v14 с УСЛОВНЫМ фолдом в `memory_version` (при выключенной инъекции не пишется НИ БАЙТА — golden байт-идентичен, ни одна книга не двигается) · ты/вы-флаггер осей A–C + D-индикатор в `gates.voice` (версия не фолдится, едет с прогоном) · флаггер спойлер-утечки поверх `Selection.Rejected` (вне гейта — стоячий механизм C1) · девять «ровно так» D39.55 исполнены, запреты (д)/(и) держатся (`repin.go`/`snapshot.go` не тронуты ни строкой) · ратифицированное отклонение: атрибуция ЦЕЛЕВАЯ, не исходная (выравнивания src↔tgt в движке нет; целевой хвост атрибуции матчится банком через `speech_verb`+`decl`) · `speech-cue.txt` zh НЕ шипнут (байты двигают `LangpackVersion` = перекупка обеих волн всех zh-книг — решение владельца, бэклог 13б) (31.07, оркестратор №9). ✅
|
||||
|
||||
Приёмка короткая (норма 30.07): сьют `-race` перегнан оркестратором — зелёный; vet/gofmt чисто; диффы repin/snapshot/testdata пусты; записка D21.10 сверена (переписана в «BUILT», стабильный ключ и заморозка в ней). Мутации сессии 23/23 красные (две выжившие первого круга = дыры тестов, закрыты с премис-проверками). Критик полноты честен: оси до холодного прогона дают 0 срабатываний («не измерено», не «чисто») · ось A может ловить цитату · ось B precision-over-recall · качество осей = функция полноты `decl` сида · SourceReplies без выравнивания не сопоставим с Replies. Хвосты: бэклог 13б (`speech-cue.txt` — владельцу) · 49а (ALTER-миграции v8–v14 не идемпотентны вопреки шапке `migrate.go:9-11`) · golden-счётчики голоса — кандидат следующего ратифицированного пере-капчера. Первое измерение = `gates.voice` ON на холодном мини-прогоне (бэклог 24). Отчёт: `archive/reports/PACK19_BUILD_2026-07-30.md` (ревью-шапка).
|
||||
|
|
|
|||
172
docs/archive/reports/PACK19_BUILD_2026-07-30.md
Normal file
172
docs/archive/reports/PACK19_BUILD_2026-07-30.md
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
# ПАК-19 «голос и состояние» — ФАЗА 2 (стройка). Отчёт бэкенд-сессии, 2026-07-30
|
||||
|
||||
**Аддендумы получены релеем:** директива фазы 2 (оркестратор №9, 30.07) + D39.55 + ревью-шапка `PACK19_DESIGN_2026-07-30.md`. Сессия НЕ коммитит; `git mv` не было.
|
||||
|
||||
> **Ревью-шапка (оркестратор №9, 31.07): ПРИНЯТО И ЗАЛЕНДЕНО, D39.56.** Короткая приёмка по норме 30.07:
|
||||
> полный сьют `-race -count=1` перегнан оркестратором — зелёный, vet чист; `repin.go`/`snapshot.go`/`testdata`
|
||||
> не тронуты ни строкой (диффы пусты — запреты (д)/(и) и «golden байт-идентичен» подтверждены исполнением);
|
||||
> записка D21.10 в `migrate.go` сверена — фиксирует стабильный ключ `(src, sense)` и неснятую заморозку.
|
||||
> Диспозиции пингов §7: (1) ЦЕЛЕВАЯ атрибуция принята — выравнивания src↔tgt в движке нет, скоуп не менялся;
|
||||
> (2) `speech-cue.txt` zh НЕ шипить до решения владельца — бэклог 13б; (3) golden не расширять — верно по
|
||||
> директиве, счётчики голоса = кандидат следующего ратифицированного пере-капчера; (4) флаггер утечки вне
|
||||
> гейта — принят с записью (стоячий механизм C1, `retrieval_state` вне хешей); (5) записка сверена. Находка
|
||||
> критика полноты §6.9 (ALTER-миграции не идемпотентны вопреки шапке) — бэклог 49а. Первое измерение осей —
|
||||
> включение `gates.voice` на холодном мини-прогоне (бэклог 24).
|
||||
|
||||
**Гейты:** `go build` ✅ · `go vet ./...` ✅ · `gofmt -l` чист по всем моим файлам (флагнут только `internal/llm/llm.go` — предсуществующий нит бэклога-53, мной не тронут) · **полный сьют `-race -count=1` зелёный** · **golden: capture байт-идентичен** (`git diff --stat testdata/` пуст) · **мутации 23/23 красные** · диффстат **+672/−71, 16 файлов + 7 новых**.
|
||||
|
||||
---
|
||||
|
||||
## 1. Эхо-блок
|
||||
|
||||
1. Шесть пунктов директивы построены; инъекции нет — кода за §3.V3 не написано.
|
||||
2. Три сущности = отдельные таблицы, фолд — внутрь `memory_version` (запрет (д) соблюдён).
|
||||
3. Фолд условный и сегодня НИКОГДА не пишется ⇒ хеш каждой книги байт-идентичен, golden пуст.
|
||||
4. (в) закрыт компиляцией, а не фильтром: voice/address — отдельные Go-типы.
|
||||
5. Утечка считается только по несработавшему-по-окну при СРАБОТАВШЕМ ключе; sticky-реджекты (ж) в счёт не входят.
|
||||
6. Регистр — абстрактный (`informal`/`formal`), поверхности — целевые данные.
|
||||
7. `gates.voice` не фолдится; `VoiceCheckVersion` едет с прогоном и в отчёт.
|
||||
8. `speech-cue.txt` для zh НЕ шипнут — его байты двигают `LangpackVersion`. Решение владельцу (§7).
|
||||
9. Одно отклонение от буквы дизайна (механизм атрибуции — целевой, не исходный) — не молчу, §7 пинг 1.
|
||||
10. Не делаю: инъекцию · Annotator · слой-2 · ru-target долг · платные прогоны · коммиты.
|
||||
|
||||
---
|
||||
|
||||
## 2. Что построено (по пунктам директивы)
|
||||
|
||||
| # | Пункт | Как сделано | Улика |
|
||||
|---|---|---|---|
|
||||
| 1 | Схема трёх типов + сид + одна транзакция + условный фолд | миграция v13 (`voice_profiles`, `address_pairs`), v14 (4 колонки `retrieval_state`); `ReplaceBank` заменяет ВСЕ три типа в одной tx; `BankInput`/`MaterializeBank`/`ComputeVersionScopedIn` | `store/migrate.go` v13/v14 · `store/glossary.go:ReplaceBank` · `membank/memory.go:ComputeVersionScopedIn` |
|
||||
| 1а | `speech` → deprecated, `first_person` → поле профиля | колонка помечена в схеме; `self_ref` живёт в `voice_profiles` | `migrate.go:184` · `store/voice.go` |
|
||||
| 2 | Флаггер спойлер-утечки | `Bank.SpoilerLeaks(rejected, output)` поверх `Selection.Rejected` + `dstFormPresent`; НОЛЬ схемных изменений | `membank/mempostcheck.go` |
|
||||
| 3 | Ты/вы-флаггер | `internal/checks/voice.go`, оси A–C в `Total()`, ось D — отдельное поле; свой гейт `gates.voice`, версия не фолдится | `checks/voice.go` · `config/pipeline.go:VoiceGate` |
|
||||
| 4 | Данные V5 | 5 новых целевых категорий в `target-ru.txt`; опциональный ИСХОДНЫЙ `speech-cue.txt` (формат+загрузчик+тест) | `lang/data/target-ru.txt` · `lang/langpack.go:parseSpeechCue` |
|
||||
| 5 | Инъекция не проводится | `BankInput.InjectVoice` не имеет конфиг-ключа и всегда `false` | `pipeline/seeding.go` (комментарий у `bankIn`) |
|
||||
| 6 | Фикс-строка D39.54 | комментарий поля `SHA256` → «canonical (comment-stripped)» | `pipeline/render.go` |
|
||||
|
||||
**Что НЕ построено сознательно:** колонка `reveal_ch` и фаза на алиасах (D39.55 сократил скоуп; записка D21.10 в `migrate.go` переписана и объясняет, почему две строки с непересекающимися окнами уже выражают pre/post).
|
||||
|
||||
---
|
||||
|
||||
## 3. Девять «РОВНО ТАК» — исполнение
|
||||
|
||||
| | Требование | Исполнено | Улика / мутация |
|
||||
|---|---|---|---|
|
||||
| (а) | бюджет инъекции в потолке/reserve | §5 отдельным абзацем | — |
|
||||
| (б) | критик полноты | §6 отдельной секцией | — |
|
||||
| (в) | `Materialize` не пускает голос в AC/инъекцию | **сильнее фильтра — компиляцией**: `store.VoiceProfile`/`AddressPair` не присваиваемы в `[]store.GlossaryEntry`, матчер видит только `BankInput.Rows` | M23 (искусственно вливаю профили в rows) → красный `TestVoiceProfileNeverReachesTheMatcherOrTheInjection` |
|
||||
| (г) | порядок «исключение скоупом ДО подсчёта условия» | фолд собирает `inScope` в строчном цикле, затем voice-цикл фильтрует ПЕРЕД `hasVoice = true` | M3 (переставить) → красный `TestVoiceFoldRespectsTheMinedScopeBeforeCountingIt` |
|
||||
| (д) | свой ключ снапшота запрещён | ни одного нового ключа payload; `repin.go` не тронут ни строкой | `git diff` по `repin.go` пуст; golden payload не двигается |
|
||||
| (е) | `address_pair` по стабильному ключу | колонки `speaker_src/speaker_sense/addressee_src/addressee_sense`; `glossary.id` нигде не участвует | `store/voice.go`, `migrate.go` v13 |
|
||||
| (ж) | sticky-carry, отклонённый окном → `Rejected` | одна строка в `Select` | M4 → красный `TestStickyCarryBlockedByWindowIsRecordedAsRejected` |
|
||||
| (з) | включение инъекции = полный `--resnapshot` | `InjectVoice` входит в `memory_version` ⇒ флип двигает ОБА wave-снапшота; bank-only на флипе не обещаю | M1/M2 → красные `TestVoiceRowsDoNotMoveTheBankVersionWhileUninjected` / `TestInjectedVoiceRowsMoveTheBankVersion` |
|
||||
| (и) | v1 переиспользует `GlossaryTokenBudget` | отдельного voice-бюджета не заведено; `repin.go:186` не тронут | — |
|
||||
| опц. | линт «gap в окнах одного src» | `VoiceWindowGaps`, WARN при сиде, никогда не фатал | `TestUnknownVoiceCharactersAndWindowGaps` |
|
||||
|
||||
**Уточнение по (ж), которое важнее самой правки.** Ветка недостижима через прод-драйвер: `precomputeSticky` сбрасывает окно на каждой границе главы (`wave.go:82-85`), поэтому carry всегда приходит в ТУ ЖЕ главу, где сработал, и `spoilerBlocked(e, chapter)` даёт тот же ответ. То есть это починка на границе ЭКСПОРТИРОВАННОГО API, а не смена поведения — и именно поэтому golden остался байт-идентичным. Тест бьёт по `Select` напрямую.
|
||||
|
||||
---
|
||||
|
||||
## 4. Найдено и починено ИСПОЛНЕНИЕМ (то, чего дизайн не видел)
|
||||
|
||||
1. **Отсчёт исходных реплик двоил.** `“来吧。”方源说道。他走了。“不。”` давал 2 атрибутированные реплики на одном кие: окно «до открывающей кавычки» дотягивалось до кия ПРЕДЫДУЩЕЙ реплики. Исправлено потреблением: кий атрибутирует не более одной реплики (`countSourceReplies`, `consumed`). Мутация M13 возвращает дефект → красный.
|
||||
2. **Две мои же мутации выжили в первом круге** — обе оказались реальными дырами тестов, не формальностью:
|
||||
- **M12** (снять предусловие «на сцене ровно двое»): мой «третий персонаж» стоял В АТРИБУЦИИ, поэтому кейс на самом деле проверял неоднозначную атрибуцию, а не эвристику адресата. Перенёс третье имя в НАРРАТИВ + добавил утверждение `Attributed == 1`, чтобы кейс не мог снова уехать не на то правило.
|
||||
- **M14** (убрать инертный путь без T/V-данных): мой «инертный» кейс не имел и маркеров реплик, так что T/V-гард был замаскирован гардом сегментации. Добавлен кейс «цель РАЗМЕЧАЕТ диалог, но T/V-различения не имеет» (en-подобная цель) — он изолирует именно T/V-гард.
|
||||
Обе дыры закрыты, второй круг: **23/23 красные**.
|
||||
3. **Тест апгрейда схемы был бы вакуумным** без премис-проверки: добавил два `SELECT`, которые ОБЯЗАНЫ упасть на старой БД до `Open` — иначе тест «апгрейда» ничего не доказывает.
|
||||
|
||||
---
|
||||
|
||||
## 5. Бюджет инъекции и денежный потолок — закрытие REFUTED-1 (ровно так (а))
|
||||
|
||||
**Сегодня вопрос закрыт ПО ПОСТРОЕНИЮ, и это надо сказать прямо, а не выдать за проектную заслугу:** инъекции нет, `InjectVoice` не имеет конфиг-ключа, ни один байт голоса не попадает в `msgs`, поэтому промпт-оценка не меняется, `EstimateTokens` (`render.go:349`) считает те же строки, `ledger.EstimateUSD` получает тот же `promptEst`, и `Store.Reserve` (`stagerun.go:450-452`) резервирует ту же сумму. Потолок не ослеплён, потому что резервировать нечего.
|
||||
|
||||
**Что механизм обязан будет сделать, когда инъекция поедет** (записываю, чтобы полигон и будущая сессия не искали это заново):
|
||||
|
||||
- Инъекция — ОТДЕЛЬНОЕ system-сообщение (`render.go:245`), и `promptEst` считается по СОБРАННЫМ `msgs`, а не по шаблону. Значит voice-блок попадёт в оценку автоматически, БЕЗ правки резервации — это свойство того, что оценка живёт после сборки сообщений, а не до неё. Проверяемое утверждение, не обещание: если добавить блок и не тронуть `stagerun.go`, резерв вырастет.
|
||||
- Ослепить потолок может ровно один сценарий: блок, добавленный в `msgs` ПОСЛЕ вычисления `promptEst`. В текущем коде такого пути нет (сборка → хеш → оценка → резерв, один порядок в `stagerun.go`), и сохранить этот порядок — обязательное «ровно так» пака инъекции.
|
||||
- Порядок величины (пере-мер §V6 дизайна по действующему прайсу): драфт-слот ≈ **$0.042/книга-500** при 300 ток./чанк и $0.14/M; редакторский ≈ **$0.065** при $0.435/M. Резерв на вызов растёт на ≤$0.0002 — на три порядка ниже флора `min_max_tokens`, то есть в потолок не упирается.
|
||||
- `gates.terminology.budget_usd`-подобный отдельный потолок голосу НЕ нужен: инъекция не является отдельным КЛАССОМ ВЫЗОВОВ (у неё нет своих чекпойнтов), она удорожает существующие вызовы, а их уже накрывает книжный потолок.
|
||||
|
||||
---
|
||||
|
||||
## 6. Критик полноты — закрытие REFUTED-2 (ровно так (б))
|
||||
|
||||
Спрашиваю себя: что в этом паке НЕ покрыто, НЕ измерено или измерено слабее, чем звучит.
|
||||
|
||||
1. **Оси B и D держатся на данных, которых на стенде нет.** Профилей и реестра ни у одной стендовой книги нет, `gates.voice` выключен, `speech-cue.txt` не шипнут. Значит на реальном тексте флаггер сегодня даёт ровно ноль срабатываний — и это НЕ «чисто», а «не измерено». Первое настоящее измерение — холодный мини-прогон (бэклог-24 уже держит строку «срабатывания голос-флаггера»); до него точность осей не известна.
|
||||
2. **Ось A может ловить цитату.** Реплика, внутри которой персонаж ЦИТИРУЕТ чужие слова с другим регистром, засчитается противоречием. Вето по множественности этого не снимает. Класс не закрыт и не обойдён — назван.
|
||||
3. **Ось B узка по построению.** Она срабатывает только когда в реплике есть ОБЩЕЕ местоимение и нет заявленного самоназвания. Реплика, где служанка не называет себя вовсе, — молчание, а не сигнал. Это осознанный precision-over-recall, но «доля сохранённых маркеров» из research/15 этим НЕ меряется.
|
||||
4. **Атрибуция ловит только ту форму имени, что есть в `decl`.** Мой же тест это и показал: без «Фан Юаню» неоднозначная атрибуция читается как однозначная. Значит качество осей B/C/D — функция ПОЛНОТЫ склонений в сиде, а не только кода. Для полигона это прямое следствие: неполный `decl` тихо снижает `Attributed`, а не флаги.
|
||||
5. **Сегментация реплик — построчная.** Реплика, разорванная переносом строки внутри одного хода, разъедется на две; реплика без маркера (косвенная речь) не видна вовсе.
|
||||
6. **`SourceReplies` меряет исходник, но сопоставить с целевым `Replies` нельзя** — выравнивания нет. Это два независимых счётчика; читать разницу как «потеряли реплики» без ручной сверки нельзя.
|
||||
7. **Схема есть, потребителя-инъекции нет.** `register`, `lexicon_markers`, `exemplars`, `brightness`, `address_default`, `form`, `closeness` хранятся, валидируются и (при включённой инъекции) хешируются — но сегодня их не читает НИКТО. Это ратифицированное состояние (D21 п.2), но честное имя ему — «зарезервировано», а не «работает».
|
||||
8. **Не измерено: холодный старт голоса.** Как поведёт себя флаггер на книге, где банк пуст, а профили есть, — не проверено ни фикстурой, ни прогоном.
|
||||
9. **Пре-существующее, не моё, но рядом:** ALTER-шаги миграций (v8–v14) не идемпотентны, вопреки шапке `migrate.go:9-11`, которая обещает сходимость полу-применённой БД. Счётчик версий это скрывает. Мой апгрейд-тест показывает, что путь v12→v14 работает; про полу-применённый шаг не утверждаю ничего.
|
||||
|
||||
---
|
||||
|
||||
## 7. Пинги оркестратору
|
||||
|
||||
1. **Отклонение от буквы дизайна, названное вслух: атрибуция ЦЕЛЕВАЯ, а не исходная.** Фаза 1 набросала исходный пре-гейт «имя + 说/道 у кавычки». При стройке выяснилось то, что на бумаге не видно: кий называет говорящего в ИСХОДНИКЕ, а проверять надо реплику в ЦЕЛИ, и выравнивания src↔tgt в движке нет — перенести спикера через границу нечем. Целевой стороне выравнивание не нужно: хвост атрибуции русской тире-строки уже называет спикера словами, которые банк умеет матчить (`speech_verb` + `decl`-формы). Поэтому оси A–D работают на целевой стороне, а исходный `speech-cue.txt` построен и используется ТОЛЬКО как знаменатель `SourceReplies`. Скоуп не расширен и не сужен; изменился механизм внутри пункта.
|
||||
2. **Решение владельца: шипить ли `configs/langpacks/zh/speech-cue.txt`.** Файл опциональный, формат и загрузчик готовы, но его байты входят в `Version()` пакета (`langpack.go:188-191`) ⇒ `LangpackVersion` (`snapshot.go:376`) ⇒ **пере-покупка ОБЕИХ волн всех zh-книг**. Цена ≠ ноль, польза = один знаменатель. Поэтому не шипнул: тот же аргумент, которым `terminology.txt` не шипят при совпадении с дефолтом (`langpack.go:245-247`). Пока файла нет, `SourceReplies` = 0 = «не измерено».
|
||||
3. **golden сознательно НЕ расширен** новыми счётчиками. Директива требует пустой маскированный дифф; добавление полей в capture — это ратифицированный пере-капчер, которого в директиве нет. Новые счётчики закрыты сквозными тестами через настоящий драйвер. Если оркестратор хочет их в golden — это отдельная строка «сдвинутой оси».
|
||||
4. **Флаггер утечки НЕ гейтится `gates.voice`** (сознательно): спойлер-окно — стоячий механизм безопасности C1, а не опциональный замер. Следствие: у книги с окнами счётчик `n_spoiler_leaks` начнёт заполняться без всякого включения. На провод и на снапшот это не влияет (`retrieval_state` пересчитывается каждый прогон), но это ИЗМЕНЕНИЕ наблюдаемого поведения без флага — называю строкой, а не прячу.
|
||||
5. **Записка D21.10 в `migrate.go` переписана** (была «reserved, not code»): теперь она фиксирует, что два типа построены, третий не нужен, и почему. Это зона бэкенда, но записка была ссылкой D-лога — сверьте формулировку.
|
||||
|
||||
---
|
||||
|
||||
## 8. Сдвинутые оси (п.6 «ровно так»)
|
||||
|
||||
Одна, и она нулевая по байтам: `memory_version` получил УСЛОВНОЕ слагаемое, которое при `InjectVoice=false` не пишется. Проверено исполнением с двух сторон — байт-идентичность (`TestVoiceRowsDoNotMoveTheBankVersionWhileUninjected`, плюс сверка с `ComputeVersion` старой формы) и golden байт-в-байт. Ни одна книга не двигается этим паком.
|
||||
|
||||
Наблюдаемое поведение, изменившееся без флага: `n_spoiler_blocked` теперь считает и sticky-реджекты (ветка недостижима в проде, §3), `n_spoiler_leaks` заполняется всегда (§7 пинг 4). Оба — `retrieval_state`, самозалечивающиеся, вне хешей.
|
||||
|
||||
---
|
||||
|
||||
## 9. Мутации (23/23 красные)
|
||||
|
||||
| | Мутация | Убита тестом |
|
||||
|---|---|---|
|
||||
| M1 | фолд игнорирует условие инъекции (фолдит всегда) | `TestVoiceRowsDoNotMoveTheBankVersionWhileUninjected` |
|
||||
| M2 | voice-строки не фолдятся даже при инъекции | `TestInjectedVoiceRowsMoveTheBankVersion` |
|
||||
| M3 | тег ставится ДО скоуп-фильтра (нарушение (г)) | `TestVoiceFoldRespectsTheMinedScopeBeforeCountingIt` |
|
||||
| M4 | sticky-реджект снова тихо дропается | `TestStickyCarryBlockedByWindowIsRecordedAsRejected` |
|
||||
| M5 | утечка считает и sticky-реджекты (скоуп расширен) | `TestSpoilerLeakFlagger` |
|
||||
| M6 | утечка не проверяет присутствие рендеринга | `TestSpoilerLeakFlagger` |
|
||||
| M7 | ось A без вето по множественности | `TestVoiceAxisATVContradictionInOneReply` |
|
||||
| M8 | ось B срабатывает при ПРИСУТСТВУЮЩЕМ самоназвании | `TestVoiceAxisBSelfReferenceFlattened` |
|
||||
| M9 | ось D втянута в `Total()` | `TestVoiceAxisDIsAnIndicatorOutsideTheCount` |
|
||||
| M10 | атрибуция принимает неоднозначного спикера | `TestVoiceAttributionRequiresOneSpeakerAndASpeechVerb` |
|
||||
| M11 | атрибуция без глагола речи | то же |
|
||||
| M12 | ось D без предусловия «на сцене ровно двое» | `TestVoiceAxisDIsAnIndicatorOutsideTheCount` |
|
||||
| M13 | кий атрибутирует несколько реплик | `TestVoiceSourceReplyDenominator` |
|
||||
| M14 | флаггер работает без целевых T/V-данных | `TestVoiceFlaggerIsInertWithoutTargetData` |
|
||||
| M15 | цитата принимается за реплику (нет глагола/вето) | `TestVoiceQuotedRepliesNeedASpeechVerbAndRespectTheInnerVeto` |
|
||||
| M16 | драйвер игнорирует гейт | `TestVoiceFlaggerIsSilentWhenTheGateIsOff` |
|
||||
| M17 | гард «персонажа нет в банке» обезврежен | `TestSeedRefusesAVoiceProfileForAnUnknownCharacter` |
|
||||
| M18 | пересекающиеся окна профиля принимаются | `TestBankSeedRefusesMalformedVoiceSections` |
|
||||
| M19 | `ReplaceBank` не пишет пары | `TestVoiceContentRoundTripsThroughTheBankReplace` |
|
||||
| M20 | проекция игнорирует окно профиля | `TestVoiceProjectionRespectsWindows` |
|
||||
| M21 | `setVoiceState` не пишет в строку | `TestVoiceFlaggerReachesTheStoreAndTheReport` |
|
||||
| M22 | противоречие регистров в сиде принимается | `TestBankSeedRefusesMalformedVoiceSections` |
|
||||
| M23 | содержимое профиля вливается в глоссарные строки | `TestVoiceProfileNeverReachesTheMatcherOrTheInjection` |
|
||||
|
||||
Харнесс: `scratchpad/pack19/mutate.py` (правка → именованный тест → восстановление файла; `.mutbak` не остались, `git status` чист).
|
||||
|
||||
---
|
||||
|
||||
## 10. Тесты (26 новых `^func Test`, исполнено)
|
||||
|
||||
`checks/voice_test.go` 9 · `membank/memvoice_test.go` 10 · `pipeline/voicerun_test.go` 5 · `store/voice_test.go` 2. Сквозные — через настоящий драйвер (`TranslateBook` с мок-провайдером): флаггер доезжает до `retrieval_state` и `QualityReport` и НЕ флагает юнит; гейт выключен ⇒ ноль байт; утечка всплывает без гейта; сид с профилем неизвестного персонажа останавливает прогон ДО вызовов; голос/пары round-trip через `ReplaceBank`. Схема: апгрейд БД v12→v14 с премис-проверкой + полная замена трёх типов.
|
||||
|
||||
---
|
||||
|
||||
## 11. Что осталось владельцу/оркестратору
|
||||
|
||||
- решение по `speech-cue.txt` для zh (§7 пинг 2);
|
||||
- включение `gates.voice` на холодном мини-прогоне — первое настоящее измерение точности осей (§6 п.1);
|
||||
- инъекция — отдельное решение после полигон-эксперимента D21 п.2; при включении см. §5.
|
||||
|
||||
**СТОП. Не коммичу.**
|
||||
Loading…
Add table
Reference in a new issue