Land track-A pack-1: trust-gated glossary suppressor with loud telemetry, export-contract normalization, pair-keyed prompt seam, role-injection registry, target-gated readability, quality report

This commit is contained in:
Claude (backend session) 2026-07-16 21:54:00 +03:00
parent ce3e17faa0
commit d3f6b34ad4
23 changed files with 1409 additions and 361 deletions

View file

@ -102,11 +102,20 @@ func report(cfgPath string) error {
// Чтения — коллбеками (interleaved с печатью, как исторически): провал чтения
// посреди аудита оставляет уже напечатанную часть на stdout (см. renderReport).
return renderReport(os.Stdout,
if err := renderReport(os.Stdout,
func() ([]store.RequestLogView, error) { return r.Store.RequestLogRows(r.Book.BookID) },
func() ([]store.ChunkStatus, error) { return r.Store.ChunkStatusesForBook(r.Book.BookID) },
func() ([]store.RetrievalState, error) { return r.Store.RetrievalStatesForBook(r.Book.BookID) },
func() (float64, float64, error) { return r.Store.SpentUSD(r.Book.BookID) })
func() (float64, float64, error) { return r.Store.SpentUSD(r.Book.BookID) }); err != nil {
return err
}
// Per-run quality-report (D39 слой 5): aggregate the deterministic quality signals into a readable
// summary — the "telemetry from day one" the owner asked for (H5). Read-only ($0), never a gate.
q, err := r.QualityReport()
if err != nil {
return err
}
return renderQuality(os.Stdout, q)
}
// status prints the READ-ONLY progress projection (D15.3): 0 LLM, 0 replay. --json emits the

View file

@ -210,6 +210,24 @@ func errTail(degraded, errText string) string {
return s
}
// renderQuality prints the deterministic per-run quality-report (D39 слой 5): the claim-1 structural
// KPI (mean sentences per narrative paragraph) plus the aggregated deterministic signals
// (dialogue-dash, glossary consistency, number drift, CJK-leak / echo rates, trust-gated). Pure
// observability — a summary section, never an exit code. Kept SEPARATE from renderReport so that
// function's frozen callback contract (render_test.go) is untouched.
func renderQuality(w io.Writer, q *pipeline.QualityReport) error {
fmt.Fprintf(w, "\n=== КАЧЕСТВО (per-run, детерминированные сигналы — наблюдаемость, не гейт) ===\n")
fmt.Fprintf(w, "чанков всего=%d · дошло до финала=%d · с экспорт-текстом=%d\n", q.TotalChunks, q.ProcessedChunks, q.TextChunks)
// Claim-1: рубленые абзацы. ≈1.0 предл./абзац = рублено (претензия владельца); выше = слитая проза.
fmt.Fprintf(w, "СТРУКТУРА (претензия-1 «рубленые абзацы»): предл/нарратив-абзац=%.2f (предложений=%d / нарратив-абзацев=%d)\n",
q.MeanSentPerNarrPara, q.NarrativeSentences, q.NarrativeParagraphs)
fmt.Fprintf(w, "СИГНАЛЫ: диалог-тире=%d · глоссарий-промахов=%d · число-дрейф=%d · trust-gated(сид)=%d\n",
q.DialogueDashFlags, q.GlossaryMisses, q.NumberDriftFlags, q.TrustGated)
fmt.Fprintf(w, "УТЕЧКИ: CJK-strip чанков=%d (%.1f%%) · echo(cjk_artifact) чанков=%d (%.1f%%)\n",
q.CJKLeakChunks, 100*q.CJKLeakRate, q.EchoChunks, 100*q.EchoRate)
return nil
}
// renderStatusJSON emits the StatusReport as indented JSON (stable enums — the
// ratified CI/IDE contract, D12/D15.3) and maps flagged>0 to the exit-2 sentinel.
func renderStatusJSON(w io.Writer, rep *pipeline.StatusReport) error {

View file

@ -35,7 +35,12 @@ stages:
- name: draft
role: translator
model: deepseek-v4-flash # интерим — переводчик по exp13 (draft качеством не мерен, D30.6); флип при развязке бейк-оффа
prompt: ../prompts/translator.md
# Слой 2 (D39): промпт резолвится из pair-keyed пакета конвенций по паре книги (Book.LangPair()),
# НЕ из фикс-пути. Наполнена только zh→ru (сегодня); ja/en книга без своей пары — FAIL-LOUD в
# loadTemplates (закрывает латентный баг «ja едет через китайский паратаксис»). Язык промпта —
# свойство пакета. Прод zh→ru резолвит тот же translator.md → тот же SHA (поведение-нейтрально).
prompts:
zh-ru: ../prompts/translator.md
# D30.2 reflow: из translator.md убрано «Сохраняй разбивку на абзацы» (построчная
# вёрстка исходника деструктивна для ru — корень нечитаемости exp12 §2.5-S). Bump
# v0-draft→v1-reflow: сменил SHA промпта, осознанный --resnapshot.
@ -54,7 +59,9 @@ stages:
# верность 5.0, 0 утечек преамбул; grok reasoning-off из редакторских ролей СНЯТ — no-op).
# gemini-3.1-pro — премиум-эскалация редактора ТОЛЬКО за санитайзером (утечка преамбул 6/6).
model: glm-5
prompt: ../prompts/editor.md
# Слой 2 (D39): pair-keyed пакет конвенций (см. draft-стадию) — наполнена zh→ru.
prompts:
zh-ru: ../prompts/editor.md
# Bump v2-bilingual-reflow→v3-discourse-reflow: в editor.md вшито валидированное P1a-ядро
# ДИСКУРС-ПЕРЕВЁРСТКИ + few-shot (exp14/D37 §2а — претензия-1 «рубленые абзацы» = рычаг ПРОМПТА,
# не модель; P1a реверстает дефект у всех семей). Меняет SHA промпта → осознанный --resnapshot.

View file

@ -23,7 +23,8 @@ stages:
- name: draft
role: translator
model: deepseek-v4-flash
prompt: ../prompts/translator.md
prompts:
zh-ru: ../prompts/translator.md
prompt_version: v1-reflow # D30.2 reflow (тот же translator.md, что в C1) — label-SHA дисциплина
temperature: 0.8 # C2: кандидаты сэмплируются горячее (Р2: T=0.61.0)
reasoning: "off"
@ -34,7 +35,8 @@ stages:
- name: select
role: judge
model: glm-5
prompt: ../prompts/judge-selector.md
prompts:
zh-ru: ../prompts/judge-selector.md
prompt_version: v0-skeleton
temperature: 0
reasoning: "off"
@ -45,7 +47,8 @@ stages:
# по D30.1 — glm-5 билингв (grok reasoning-off из редакторов СНЯТ — no-op). (Стадию select
# выше — judge/glm-5, Gemini-слот Ф2 — НЕ трогаем.)
model: glm-5
prompt: ../prompts/editor.md
prompts:
zh-ru: ../prompts/editor.md
# Бамп v2-bilingual-reflow→v3-discourse-reflow (label-SHA дисциплина: тот же editor.md, что в C1 —
# вшито P1a-ядро ДИСКУРС-ПЕРЕВЁРСТКИ, D37 §2а; лейбл обязан следовать за новым SHA файла).
prompt_version: v3-discourse-reflow

View file

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"gopkg.in/yaml.v3"
@ -60,7 +61,17 @@ type Stage struct {
Name string `yaml:"name"`
Role string `yaml:"role"`
Model string `yaml:"model"`
Prompt string `yaml:"prompt"` // path to the versioned template file
// Prompt is the LEGACY single, pair-AGNOSTIC template path (kept for tests / simple configs).
// Prompts is the pair-KEYED convention pack (D39 слой 2): src→tgt → template path, resolved by the
// book's LangPair() at load with a FAIL-LOUD on a missing pair (never a silent fall-through to
// another pair's conventions — the latent bug where a ja book rode the zh-parataxis prompt,
// L4-prompts-not-per-pair-zh-baked). The pack is where a per-pair prompt (and its LANGUAGE — a
// property of the pack, not a hardcode) lives; today only zh-ru is populated. Exactly ONE of
// Prompt/Prompts per stage (validated in LoadPipeline). The pair→prompt binding is snapshot-folded
// transitively: the resolved template's content rides PromptSHA256 and the book's pair rides
// BriefHash (source_lang/target_lang), so a pair or content change is a loud --resnapshot.
Prompt string `yaml:"prompt"`
Prompts map[string]string `yaml:"prompts"`
PromptVersion string `yaml:"prompt_version"`
Temperature float64 `yaml:"temperature"`
Reasoning string `yaml:"reasoning"` // "", off, low, medium, high
@ -161,10 +172,25 @@ type Fanout struct {
Candidates int `yaml:"candidates"`
}
// gatesEnabled reports whether any QA-gate is on. Эскалация запускается только
// по провалу гейта, поэтому это сигнал «эскалация вообще достижима» (Фаза 1+).
func (p *Pipeline) gatesEnabled() bool {
return p.Gates.Coverage.Enabled
// PromptPathFor resolves the effective prompt template path for this stage given the book's language
// pair (D39 слой 2). A pair-keyed `prompts` pack resolves the entry for `pair`, failing LOUD if the
// pair is absent (never a silent fall-through to another pair's conventions — the latent bug where a
// ja book rode the zh-parataxis prompt). A legacy single `prompt` is pair-agnostic (returned as-is).
// Callers pass Book.LangPair(). Paths are already resolved to absolute in LoadPipeline.
func (st Stage) PromptPathFor(pair string) (string, error) {
if len(st.Prompts) == 0 {
return st.Prompt, nil
}
if p, ok := st.Prompts[pair]; ok {
return p, nil
}
have := make([]string, 0, len(st.Prompts))
for k := range st.Prompts {
have = append(have, k)
}
sort.Strings(have)
return "", fmt.Errorf("stage %q: no prompt for language pair %q — the pair-keyed pack has [%s]; add a %q prompt (conventions authored in the right language) or fix the book's source_lang/target_lang (D39 слой 2: никогда молча не подставляем чужую пару)",
st.Name, pair, strings.Join(have, ", "), pair)
}
// CheckRunnable rejects a config whose mechanics the Phase-0 runner does NOT
@ -228,9 +254,16 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) {
// Prompt paths resolve relative to the pipeline file's directory, so a
// project works from any CWD.
dir := filepath.Dir(path)
resolvePrompt := func(pr string) string {
if pr != "" && !filepath.IsAbs(pr) {
return filepath.Join(dir, pr)
}
return pr
}
for i := range p.Stages {
if pr := p.Stages[i].Prompt; pr != "" && !filepath.IsAbs(pr) {
p.Stages[i].Prompt = filepath.Join(dir, pr)
p.Stages[i].Prompt = resolvePrompt(p.Stages[i].Prompt)
for pair, pr := range p.Stages[i].Prompts {
p.Stages[i].Prompts[pair] = resolvePrompt(pr)
}
}
@ -275,11 +308,31 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) {
if _, ok := models.Models[st.Model]; !ok {
bad("stage %q: model %q is not defined in models.yaml", st.Name, st.Model)
}
if st.Prompt == "" {
bad("stage %q: prompt template path is required", st.Name)
} else if _, err := os.Stat(st.Prompt); err != nil {
// Prompt (legacy single, pair-agnostic) XOR Prompts (pair-keyed pack, D39 слой 2) — exactly one.
switch {
case st.Prompt != "" && len(st.Prompts) > 0:
bad("stage %q: set either `prompt` (single) OR `prompts` (pair-keyed pack), not both", st.Name)
case st.Prompt == "" && len(st.Prompts) == 0:
bad("stage %q: a prompt is required — `prompt` (single) or a pair-keyed `prompts` pack (D39 слой 2)", st.Name)
case st.Prompt != "":
if _, err := os.Stat(st.Prompt); err != nil {
bad("stage %q: prompt template %s is not readable: %v", st.Name, st.Prompt, err)
}
default: // pair-keyed pack: every declared pair's template must be readable (deterministic order)
pairs := make([]string, 0, len(st.Prompts))
for pair := range st.Prompts {
pairs = append(pairs, pair)
}
sort.Strings(pairs)
for _, pair := range pairs {
pr := st.Prompts[pair]
if pr == "" {
bad("stage %q: prompt pack pair %q has an empty path", st.Name, pair)
} else if _, err := os.Stat(pr); err != nil {
bad("stage %q: prompt pack %q → %s is not readable: %v", st.Name, pair, pr, err)
}
}
}
switch st.Reasoning {
case "", "off", "low", "medium", "high":
default:

View file

@ -0,0 +1,140 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// writeTmp writes a file under a temp path and returns it.
func writeTmp(t *testing.T, path, body string) string {
t.Helper()
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return path
}
// miniModels loads a minimal valid models.yaml with a single `fake` model.
func miniModels(t *testing.T) *Models {
t.Helper()
dir := t.TempDir()
body := "prices_checked: " + time.Now().UTC().Format("2006-01-02") + `
default_model: fake
providers:
p: { kind: openai, base_url: http://x }
models:
fake: { provider: p, price: { input_per_m: 1, output_per_m: 2 } }
`
m, err := LoadModels(writeTmp(t, filepath.Join(dir, "models.yaml"), body))
if err != nil {
t.Fatalf("miniModels: %v", err)
}
return m
}
// prompt_pack_test.go pins the pair-keyed prompt seam (D39 слой 2, L4-prompts-not-per-pair-zh-baked):
// a stage resolves its prompt from a pair-keyed pack by the book's LangPair(), FAIL-LOUD on a missing
// pair; a legacy single `prompt` stays pair-agnostic; prod zh→ru resolves the same file (SHA-neutral).
func TestPromptPathForResolvesPair(t *testing.T) {
st := Stage{Name: "edit", Prompts: map[string]string{"zh-ru": "/x/editor.md", "ja-ru": "/y/editor.md"}}
got, err := st.PromptPathFor("zh-ru")
if err != nil || got != "/x/editor.md" {
t.Fatalf("PromptPathFor(zh-ru) = %q, %v; want /x/editor.md, nil", got, err)
}
}
func TestPromptPathForFailsLoudOnMissingPair(t *testing.T) {
st := Stage{Name: "edit", Prompts: map[string]string{"zh-ru": "/x/editor.md"}}
_, err := st.PromptPathFor("ja-ru")
if err == nil {
t.Fatal("PromptPathFor(ja-ru) must FAIL LOUD when the pack has no ja-ru (the latent ja-runs-zh-prompt bug)")
}
// The error must name the missing pair AND list what the pack has (operator-actionable).
if !strings.Contains(err.Error(), "ja-ru") || !strings.Contains(err.Error(), "zh-ru") {
t.Errorf("fail-loud error must name the missing pair and the pack's pairs, got: %v", err)
}
}
func TestPromptPathForLegacySingleIsPairAgnostic(t *testing.T) {
st := Stage{Name: "draft", Prompt: "/x/translator.md"}
for _, pair := range []string{"zh-ru", "ja-ru", "en-ru"} {
got, err := st.PromptPathFor(pair)
if err != nil || got != "/x/translator.md" {
t.Errorf("legacy single prompt must resolve for %q: got %q, %v", pair, got, err)
}
}
}
func TestLoadPipelineRejectsAmbiguousPromptForms(t *testing.T) {
dir := t.TempDir()
writeTmp(t, filepath.Join(dir, "translator.md"), "sys\n---USER---\n{{text}}")
models := miniModels(t)
// Both `prompt` and `prompts` set → rejected.
both := writeTmp(t, filepath.Join(dir, "both.yaml"), `
core: C1
stages:
- name: draft
role: translator
model: fake
prompt: translator.md
prompts: { zh-ru: translator.md }
prompt_version: v1
`)
if _, err := LoadPipeline(both, models); err == nil || !strings.Contains(err.Error(), "not both") {
t.Errorf("both prompt+prompts must be rejected, got: %v", err)
}
// Neither set → rejected.
neither := writeTmp(t, filepath.Join(dir, "neither.yaml"), `
core: C1
stages:
- name: draft
role: translator
model: fake
prompt_version: v1
`)
if _, err := LoadPipeline(neither, models); err == nil || !strings.Contains(err.Error(), "prompt is required") {
t.Errorf("neither prompt nor prompts must be rejected, got: %v", err)
}
}
// TestBoevoyPipelineC1ResolvesZhRu proves the migrated prod config is behaviour-neutral for zh→ru:
// the draft/edit stages resolve the same translator.md/editor.md the legacy path did (same file →
// same PromptSHA256), and a NON-zh pair fails loud.
func TestBoevoyPipelineC1ResolvesZhRu(t *testing.T) {
models, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
if err != nil {
t.Fatalf("load models: %v", err)
}
p, err := LoadPipeline(filepath.Join("..", "..", "configs", "pipeline-c1.yaml"), models)
if err != nil {
t.Fatalf("load pipeline-c1: %v", err)
}
for _, st := range p.Stages {
zh, err := st.PromptPathFor("zh-ru")
if err != nil {
t.Errorf("stage %q must resolve zh-ru: %v", st.Name, err)
}
if !strings.HasSuffix(zh, ".md") {
t.Errorf("stage %q zh-ru prompt should be a .md template, got %q", st.Name, zh)
}
// A book of a pair the pack does not carry must fail loud (closes the ja-runs-zh-prompt bug).
if _, err := st.PromptPathFor("ja-ru"); err == nil {
t.Errorf("stage %q must fail loud for ja-ru (pack is zh-ru only)", st.Name)
}
}
// The draft stage must resolve to the shared translator.md (the zh→ru pack content is unchanged).
for _, st := range p.Stages {
if st.Role == "translator" {
zh, _ := st.PromptPathFor("zh-ru")
if !strings.HasSuffix(zh, filepath.Join("prompts", "translator.md")) {
t.Errorf("draft zh-ru must be prompts/translator.md, got %q", zh)
}
}
}
}

View file

@ -24,38 +24,45 @@ import (
// re-pins loudly). The coverage gate runs ONLY on the TRANSLATOR role's output (vs the
// original source): a monolingual editor legitimately restructures sentences, so gating
// it against the source would false-flag a correct edit (see the role check below).
func (r *Runner) classifyOutput(role, source, output, finish string, isFinal bool) classification {
// classifyOutput returns the disposition AND, for a cosmetic sanitizer strip, the resolved EXPORT
// text (the strip remainder) computed ONCE here (T3.4 / L8-stripcosmetic-recomputed-twice): the
// second return travels back to runStage as the authoritative export, so no site re-derives
// stripCosmetic and the "non-empty & clean" invariant is asserted over the very bytes that ship.
// The string is "" for every disposition except FlagSanitizerStripped.
func (r *Runner) classifyOutput(role, source, output, finish string, isFinal bool) (classification, string) {
cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang})
if !cls.ok() {
return cls
return cls, ""
}
// Output-sanitizer gate (D30.3): a deterministic verdict-axis check for "instant
// unreadability" defects no other gate catches — a leaked service preamble, a trailing
// note/edit block, a markdown ### header, a Latin-script insertion, a broken word form.
// Opt-in (Gates.Sanitizer). Runs ONLY on the FINAL stage's output (isFinal) — the text
// that SHIPS: an intermediate translator draft is legitimately rough and the EDITOR's job
// is to clean it, so sanitizing the draft would wrongly skip the recovering editor and drop
// the chunk to a placeholder (адверсариальное ревью). Runs on the completion text like the
// intrinsic classifier, so a resumed checkpoint re-derives the identical verdict for free;
// its rule version folds into the snapshot only when enabled (sanitizerSnapshot).
if isFinal && r.Pipeline.Gates.Sanitizer.Enabled {
// Output-sanitizer gate (D30.3): a deterministic verdict-axis check for "instant unreadability"
// defects no other gate catches — a leaked service preamble, a trailing note/edit block, a markdown
// ### header, a Latin-script insertion, a contentless CJK-leak run. Opt-in (Gates.Sanitizer). Runs
// ONLY on the FINAL stage's output (isFinal) — the text that SHIPS: an intermediate translator draft
// is legitimately rough and the EDITOR's job is to clean it, so sanitizing the draft would wrongly
// skip the recovering editor and drop the chunk to a placeholder (адверсариальное ревью). It is a
// READABILITY gate whose patterns bake in Cyrillic/Russian conventions, so it is gated behind a
// Russian target (D39 слой 7, L8-readability-gates-target-blind) — a no-op for any other pair, which
// would false-flag (prod zh→ru is unaffected). Runs on the completion text like the intrinsic
// classifier, so a resumed checkpoint re-derives the identical verdict; its rule version folds into
// the snapshot only when enabled (sanitizerSnapshot).
if isFinal && r.Pipeline.Gates.Sanitizer.Enabled && isRuTarget(r.Book.TargetLang) {
if san := sanitizeOutput(output); san.total() > 0 {
// Cosmetic-only leak (leading markdown header and/or CJK-leak) → strip it and export
// the remainder flagged, not drop the whole chunk (D35.4a). The guard is load-bearing:
// we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that is now
// clean, so runStage/resume can trust that the derived export text is usable; anything
// else falls through to the substantive skip. Both are pure over `output`, so a resumed
// checkpoint re-derives the identical verdict.
// Cosmetic-only leak (leading markdown header and/or contentless CJK-leak) → strip it and
// export the remainder flagged, not drop the whole chunk (D35.4a). The guard is
// load-bearing: we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that
// is now clean, and we RETURN that stripped text so runStage/resume ship the exact bytes the
// guard validated. Anything else falls through to the substantive skip. Pure over `output`,
// so a resumed checkpoint re-derives the identical verdict.
if san.cosmeticOnly() {
if stripped := stripCosmetic(output); strings.TrimSpace(stripped) != "" && sanitizeOutput(stripped).total() == 0 {
return classification{FlagSanitizerStripped, san.summary()}
return classification{FlagSanitizerStripped, san.summary()}, stripped
}
}
return classification{FlagSanitizerDefect, san.summary()}
return classification{FlagSanitizerDefect, san.summary()}, ""
}
}
if !r.Pipeline.Gates.Coverage.Enabled {
return cls
return cls, ""
}
// The coverage gate compares the output against the ORIGINAL source, which is a
// source→target translation only for the TRANSLATOR role. A monolingual editor
@ -64,12 +71,12 @@ func (r *Runner) classifyOutput(role, source, output, finish string, isFinal boo
// (self-review finding). Editor/other-role fidelity is a Phase-2 concern (a
// bilingual judge over the draft, §04 mode 2), not this excision gate.
if role != roleTranslator {
return cls
return cls, ""
}
if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() {
return cov.cls
return cov.cls, ""
}
return cls
return cls, ""
}
// translateChunk drives ONE chunk through the stage list. Stages run in order,
@ -94,17 +101,26 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
// regression guard's draft→final comparison (== final in a single-stage pipeline).
draftText := ""
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and
// serialize the translator injection block. Recomputed on every run (resumed chunks
// included) so the sticky window and the injected bytes are resume-stable.
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and serialize the
// per-role injection blocks via the role→renderer registry (D39 слой 7). Recomputed on every run
// (resumed chunks included) so the sticky window and the injected bytes are resume-stable.
var memSel memorySelection
var translatorInjection, editorInjection string
injectionByRole := map[string]string{}
activeIDs := map[string]injectionDisposition{}
if r.memory != nil {
memSel = r.memory.Select(ch.Text, ch.Chapter, stickyPrev, r.Pipeline.Context.GlossaryTokenBudget)
translatorInjection = renderGlossaryBlock(memSel.injected)
editorInjection = renderEditorConstraintBlock(memSel.injected)
for role, render := range roleInjectionRenderers {
injectionByRole[role] = render(memSel.injected) // pure → per-role result is map-order-independent
}
activeIDs = memSel.activeIDs
// The disposition-gated suppressor refused to let a lower-trust longer key eat a higher-trust
// nested one (D39 слой 4): the approved term survives, but surface the seed-hygiene collision
// loudly so an operator can reconcile the draft-nesting-over-approved seed (research/13 §7).
if len(memSel.trustGated) > 0 {
r.Log.WarnContext(ctx, "memory: lower-trust longer key refused from suppressing a higher-trust nested key (approved term preserved; reconcile the seed)",
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "trust_gated", len(memSel.trustGated),
"first", memSel.trustGated[0].Suppressor+"⊃"+memSel.trustGated[0].Protected)
}
}
for stageIdx, st := range r.Pipeline.Stages {
@ -125,19 +141,12 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
continue
}
// Inject the glossary per role. The TRANSLATOR gets the src→dst block (its keys are
// matched against the source chunk); the monolingual EDITOR gets the CONFIRMED dst
// forms as target-consistency constraints (no source — decisions-log D1); every other
// stage gets none. Empty injection → the plain 2-message layout. The injection is a
// message, so it enters this stage's request_hash automatically (a resumed chunk
// reproduces it deterministically from the frozen bank).
injection := ""
switch st.Role {
case roleTranslator:
injection = translatorInjection
case roleEditor:
injection = editorInjection
}
// Resolve the per-role memory injection from the registry (D39 слой 7): the TRANSLATOR gets the
// src→dst glossary block (keys matched against the source chunk); the BILINGUAL editor (D30.1)
// gets the CONFIRMED dst forms as target-consistency constraints; every other role gets none
// (empty string → the plain 2-message layout). The injection is a message, so it enters this
// stage's request_hash automatically (a resumed chunk reproduces it deterministically).
injection := injectionByRole[st.Role]
sr, err := r.runStage(ctx, st, stageIdx, snapID, ch, prev, injection)
if err != nil {
return out, activeIDs, err
@ -182,9 +191,12 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
// Cheap deterministic style/number flaggers on the FINAL text (cheapgates.go): observability
// only, never a disposition. Runs on the same fresh-or-resumed output as the post-check, so it
// is resume-reproducible; skipped when a stage flagged (no usable output). Source is ch.Text
// (the 万/億 magnitude gate compares source↔output).
// (the 万/億 magnitude gate compares source↔output). These are READABILITY flaggers whose rules
// bake in Russian conventions (em-dash dialogue, ёфикатор, ru magnitude words), so they are gated
// behind a Russian target (D39 слой 7, L8-readability-gates-target-blind) — a no-op for any other
// pair, which would false-flag. Prod zh→ru is unaffected (target=ru).
var cheap cheapGateResult
if !flagged && prev != "" {
if !flagged && prev != "" && isRuTarget(r.Book.TargetLang) {
cheap = runCheapGates(ch.Text, draftText, prev, r.cheapGateConfig())
if cheap.total() > 0 {
r.Log.InfoContext(ctx, "cheap style gates flagged the chunk (observability, not a gate)",
@ -204,19 +216,35 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
}
}
// The export contract (D39 слой 6): NORMALISE every final text before it ships — clean OR
// flagged-stripped — so a cosmetic Unicode artifact (fullwidth glyph, U+3000 indent, stray CJK
// punctuation, a leaked combining stress mark) is fixed uniformly, not only on a flagged chunk
// (L5-normalization-fused-to-flag-path). A cosmetic sanitizer strip exports its cleaned remainder
// (recovered != ""); every other flag exports "" (the contaminated output never reaches TM/export,
// D2 / D35.4a), and exportNormalize("") == "". Pure ephemeral projection — no wire/checkpoint touch.
if flagged {
out.Disposition = DispFlagged
out.FlagReason = flagReason
// A cosmetic sanitizer strip exports its cleaned remainder (recovered != ""); every other
// flag (garbage/refusal/glossary-miss/substantive-sanitizer) exports "" — the contaminated
// output never reaches TM/export (D2 / D35.4a).
out.FinalText = recovered
out.FinalText = exportNormalize(recovered)
} else {
out.FinalText = prev
out.FinalText = exportNormalize(prev)
}
return out, activeIDs, nil
}
// injectionRenderer serializes a chunk's selected memory records into a role's injection message.
type injectionRenderer func(injected []pickedEntry) string
// roleInjectionRenderers maps a stage ROLE to the memory-injection renderer it consumes — a registry
// instead of a hand-edited switch (D39 слой 7, L8-role-injection-hardcoded-switch), so a new Ф2
// consumer role (annotator, voice-layer — README) is DATA + one renderer, not surgery on a switch.
// A role absent from the map gets no injection. Renderers are pure, so the per-role result is
// deterministic regardless of map iteration order (no map-order in OUTPUT — invariant #6).
var roleInjectionRenderers = map[string]injectionRenderer{
roleTranslator: renderGlossaryBlock, // src→dst glossary block
roleEditor: renderEditorConstraintBlock, // CONFIRMED dst forms (target-consistency constraints)
}
// persistRetrievalState writes the per-chunk observability record from the deterministic
// selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED
// records (what the model saw); spoiler/eviction are the dropped-and-logged totals;
@ -243,6 +271,15 @@ func (r *Runner) persistRetrievalState(snapID string, ch Chunk, sel memorySelect
}
rs.NSpoilerBlocked = len(sel.rejected)
rs.NEvicted = len(sel.evicted)
// Loud record of the disposition-gated suppressor (D39 слой 4): a longer lower-trust key refused
// from eating a nested higher-trust one — the term-drift code root, now visible instead of a
// silent drop (research/13 §7). Detail carries the refused suppressor→protected pairs for a human.
rs.NTrustGatedSuppress = len(sel.trustGated)
if len(sel.trustGated) > 0 {
if b, err := json.Marshal(sel.trustGated); err == nil {
rs.TrustGateDetail = string(b)
}
}
if outputChecked {
// n_postcheck_miss is the CONFIRMED-miss count (the actionable consistency-failure
// signal, external-review #1); the detail carries ALL misses (confirmed AND the

View file

@ -299,6 +299,15 @@ func isCJKTarget(lang string) bool {
return false
}
// isRuTarget reports whether the translation TARGET is Russian — the guard for the READABILITY gate
// set (output-sanitizer + cheap style flaggers), whose rules bake in Cyrillic/Russian conventions
// (D39 слой 7, L8-readability-gates-target-blind). They are a no-op for any other target, which would
// otherwise false-flag (a →en final reads as a "Latin insertion"). Mirrors the pair-aware
// coverage/classify seam. Prod zh→ru is unaffected (target=ru).
func isRuTarget(lang string) bool {
return strings.EqualFold(strings.TrimSpace(lang), "ru")
}
// --- degeneration detector (n-gram loop) — runs BEFORE the length retry (D2.3) ---
// degenerateLoop reports whether the text is a repetition loop: the word-level

View file

@ -32,7 +32,7 @@ func TestFewShotSplitAndAssembly(t *testing.T) {
}
// TestFewShotAbsentIsNoop: a template with no ---FEWSHOT--- block leaves FewShot empty and the
// toggle a no-op, so every existing prompt (translator, analyst, golden stub) is untouched.
// toggle a no-op, so every existing prompt (translator, golden stub) is untouched.
func TestFewShotAbsentIsNoop(t *testing.T) {
tpl := writeTemplate(t, "Только система.\n---USER---\n{{text}}")
if tpl.FewShot != "" {

View file

@ -245,10 +245,10 @@ func captureGolden(t *testing.T, label string, r *Runner, res *BookResult, wireB
return rss[i].ChunkIdx < rss[j].ChunkIdx
})
for _, rs := range rss {
w("ch%d/%d snap_match=%t exact=%d sticky=%d ambiguous=%d spoiler=%d evicted=%d postcheck_miss=%d style_flags=%d",
w("ch%d/%d snap_match=%t exact=%d sticky=%d ambiguous=%d spoiler=%d evicted=%d postcheck_miss=%d style_flags=%d trust_gated=%d",
rs.Chapter, rs.ChunkIdx, rs.SnapshotID == snapID, rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged,
rs.NSpoilerBlocked, rs.NEvicted, rs.NPostcheckMiss, rs.NStyleFlags)
w(" injected_ids=%s postcheck_detail=%s style_detail=%s", rs.InjectedIDs, rs.PostcheckDetail, rs.StyleDetail)
rs.NSpoilerBlocked, rs.NEvicted, rs.NPostcheckMiss, rs.NStyleFlags, rs.NTrustGatedSuppress)
w(" injected_ids=%s postcheck_detail=%s style_detail=%s trust_gate_detail=%s", rs.InjectedIDs, rs.PostcheckDetail, rs.StyleDetail, rs.TrustGateDetail)
}
w("-- wire bodies (call order) --")

View file

@ -38,7 +38,13 @@ import (
// recomputing it status-based (a collision-prone AMBIGUOUS carry no longer silently
// upgrades to CONFIRMED), and a spaced-phonetic (Latin/Cyrillic) source key is validated
// against a word boundary (approved "rose" no longer fires inside "roseanne").
const memoryMatchVersion = "memmatch-v3-perlang-minkey+collision-ambiguous+longest+spoiler+sticky-inherit-disp+phonetic-srcboundary+budget+postcheck-declaware"
// v4 (D39 слой 4): the longest-match suppressor is now DISPOSITION-GATED (suppressContained) —
// a longer but LOWER-trust key (draft/ambiguous) no longer deletes a nested HIGHER-trust key
// (approved/confirmed), closing the term-drift code root the D38.5 seed-promote only patched by
// data (L3-suppressor-disposition-blind). A refused suppression is recorded loudly (trustGated).
// The injection SELECTION changes for a draft-longer-over-approved-shorter chunk (the approved term
// now survives), so a loud --resnapshot + golden re-capture (invariant №8).
const memoryMatchVersion = "memmatch-v4-perlang-minkey+collision-ambiguous+longest-trustgated+spoiler+sticky-inherit-disp+phonetic-srcboundary+budget+postcheck-declaware"
// minKeyLenHan / minKeyLenPhonetic are the per-language single-key floors (A3, registry
// "min_key_len per-язык"). An ideographic (Han) key is semantically distinct at 2 chars
@ -133,11 +139,29 @@ type pickedEntry struct {
disp injectionDisposition
}
// trustGateEvent records a longest-match suppression the DISPOSITION-gate REFUSED (D39 слой 4):
// a longer but LOWER-trust key (draft/ambiguous, e.g. draft 四代族长) that would have deleted a nested
// HIGHER-trust key (approved/confirmed, e.g. approved 族长). The draft is editor-excluded (the block
// is CONFIRMED-only), so suppressing the approved term would have left the reader with NOTHING — the
// term-drift code root (L3). The nested higher-trust match survives and injects; this record turns
// the near-silent drop into a loud, operator-visible signal (research/13 §7). A non-empty list means
// the seed has a draft term nesting over an approved one — reconcile the seed so it stops recurring.
type trustGateEvent struct {
Suppressor string `json:"suppressor"` // the longer, lower-trust key blocked from suppressing
SuppressorDisp string `json:"suppressor_disp"` // its best injection disposition (ambiguous)
Protected string `json:"protected"` // the nested, higher-trust key it would have eaten
ProtectedDisp string `json:"protected_disp"` // its best injection disposition (confirmed)
}
// memorySelection is the hot path's output for one chunk.
type memorySelection struct {
injected []pickedEntry // in budget priority order (what the model sees)
rejected []pickedEntry // spoiler-window rejects (C1, logged)
evicted []pickedEntry // dropped by the token budget (F2, logged)
// trustGated are the longest-match suppressions the disposition-gate refused (a lower-trust
// longer key would have eaten a higher-trust nested one) — the term-drift code root, surfaced
// loudly instead of a silent drop (research/13 §7). Deterministic (ms order, deduped by pair).
trustGated []trustGateEvent
// activeIDs are the exact-matched ids (NOT sticky) → the next chunk's sticky_prev,
// each mapped to the DISPOSITION it fired with. Sticky carries this disposition
// forward instead of recomputing it (D16.2): a sticky carry has no firing key, so it
@ -305,10 +329,12 @@ func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]inj
// Closes the source-vs-target boundary asymmetry (the target side already has
// containsWholeWord). Han/kana are NOT boundary-checked (no word segmentation) — deliberate.
occ = b.suppressUnboundedPhonetic(occ, ntext)
// Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when
// the longer key belongs to a spoiler-VALID entry at this chapter (self-review #6). A
// spoiler-blocked longer key must not suppress a valid shorter name nested inside it.
occ = b.suppressContained(occ, chapter)
// Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when the
// longer key belongs to a spoiler-VALID entry at this chapter (self-review #6) AND is at least
// as TRUSTWORTHY as the nested match it would delete (D39 слой 4 — a lower-trust longer key must
// not eat a higher-trust nested one). Refused suppressions are returned for a loud record.
var trustGated []trustGateEvent
occ, trustGated = b.suppressContained(occ, chapter)
// Map surviving key occurrences → entries, recording the LONGEST firing key per entry.
matchedVia := map[int]string{}
@ -321,7 +347,7 @@ func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]inj
}
}
sel := memorySelection{activeIDs: map[string]injectionDisposition{}}
sel := memorySelection{activeIDs: map[string]injectionDisposition{}, trustGated: trustGated}
hits := map[string]pickedEntry{} // id → picked (exact)
// Iterate entries in their frozen (ORDER BY-stable) order — never map order.
@ -466,22 +492,26 @@ func renderGlossaryBlock(injected []pickedEntry) string {
return glossaryBlockHeader + "\n" + strings.Join(lines, "\n")
}
// editorConstraintHeader introduces the monolingual editor's dst-constraint block (D1).
// The editor never sees the source, so it gets ONLY the approved target renderings as
// consistency constraints — it must normalise any divergent rendering in the draft to
// these canonical forms (inflecting for context) and touch nothing else.
// editorConstraintHeader introduces the editor's dst-constraint block. It gives the editor the
// approved TARGET renderings as consistency constraints — it must normalise any divergent rendering
// in the draft to these canonical forms (inflecting for context) and touch nothing else.
const editorConstraintHeader = "КАНОНИЧЕСКИЕ ПЕРЕВОДЫ имён и терминов (в черновике эти сущности должны быть переданы ИМЕННО этими формами — приводи к ним любые расхождения, склоняя по контексту; не вводи иных вариантов и не меняй ничего другого):"
// renderEditorConstraintBlock serializes the selected records' CONFIRMED dst renderings into
// the monolingual editor's injection (D1). Unlike renderGlossaryBlock it emits the TARGET
// forms only — no "src → dst" (the editor never sees the source: decisions-log D1). It is
// CONFIRMED-only on purpose: an AMBIGUOUS record is a candidate the translator MAY have
// legitimately rejected, so forcing the editor to rewrite the draft toward it would corrupt
// a correct translation (mirrors the gate's CONFIRMED-only discipline, external-review #1).
// Deduped by dst (an alias and its main entry share a rendering), in the budget priority
// order fixed by Select. Returns "" when nothing CONFIRMED renders → the editor gets NO
// injection (the plain draft-only layout). The block is a subset of the already
// budget-limited memSel.injected, so it needs no separate token budget.
// renderEditorConstraintBlock serializes the selected records' CONFIRMED dst renderings into the
// editor's injection. Unlike renderGlossaryBlock it emits the TARGET forms only — bare canonical
// Russian, not "src → dst". It is CONFIRMED-only on purpose: an AMBIGUOUS record is a candidate the
// translator MAY have legitimately rejected, so forcing the editor to rewrite the draft toward it
// would corrupt a correct translation (mirrors the gate's CONFIRMED-only discipline, external-review
// #1). NOTE (D30.1 supersede of D1): the editor is now BILINGUAL — editor.md feeds it the source, so
// the earlier "monolingual editor never sees the source" rationale for target-forms-only is stale
// (the CONFIRMED-only discipline above is the LIVE reason and is unchanged here). Whether a bilingual
// editor should receive the src→dst mapping (like the translator) so it can bind each canonical form
// to its source term, rather than bare Russian, is an OPEN design question the D30.1 flip left
// unclosed (flagged to the orchestrator; behaviour deliberately unchanged in this pack). Deduped by
// dst (an alias and its main entry share a rendering), in the budget priority order fixed by Select.
// Returns "" when nothing CONFIRMED renders → the editor gets NO injection (the plain draft-only
// layout). The block is a subset of the already budget-limited memSel.injected, so it needs no
// separate token budget.
func renderEditorConstraintBlock(injected []pickedEntry) string {
var lines []string
seen := map[string]bool{}
@ -604,40 +634,115 @@ func (ac *ahoCorasick) matches(text []rune) []acMatch {
return out
}
// suppressContained drops an occurrence fully contained inside a STRICTLY longer one
// (longest-match, whole-entity replacement — A3), where the longer one belongs to a
// spoiler-VALID entry at this chapter. Gating the SUPPRESSOR on spoiler validity closes
// self-review #6: a spoiler-blocked longer key (e.g. 林动的父亲, until_ch=3, read at ch10)
// must not silently drop a valid shorter name nested inside it (林动) — the shorter name
// survives and injects, the longer one is separately spoiler-rejected+logged. Equal-length
// overlaps are both kept (genuine surface ambiguity, surfaced by the injectivity check).
// O(n²) over the few matches in a chunk.
func (b *MemoryBank) suppressContained(ms []acMatch, chapter int) []acMatch {
validSuppressor := func(m acMatch) bool {
for _, ei := range b.keyOwners[b.ac.keys[m.keyIdx]] {
if !spoilerBlocked(&b.entries[ei], chapter) {
return true
// trustRank orders injection dispositions by how much the reader may TRUST the record: CONFIRMED
// (approved, authoritative) outranks AMBIGUOUS (draft/auto/collision-prone), which outranks REJECT
// (spoiler-blocked / unset). It decides the longest-match containment contest below.
func trustRank(d injectionDisposition) int {
switch d {
case memConfirmed:
return 2
case memAmbiguous:
return 1
default: // memReject / unset
return 0
}
}
return false
// matchTrust returns the BEST (highest-trust) disposition an occurrence of the given key could
// inject with at this chapter — the max over its spoiler-VALID owner entries — and whether ANY
// valid owner exists (this second value subsumes the old validSuppressor: no valid owner ⇒ the
// occurrence injects nothing). CONFIRMED beats AMBIGUOUS. It is the trust the disposition-gate
// decides a containment contest on (D39 слой 4).
func (b *MemoryBank) matchTrust(m acMatch, chapter int) (disp injectionDisposition, hasValid bool) {
key := b.ac.keys[m.keyIdx]
for _, ei := range b.keyOwners[key] {
e := &b.entries[ei]
if spoilerBlocked(e, chapter) {
continue
}
if d := dispositionFor(e, key); !hasValid || trustRank(d) > trustRank(disp) {
disp, hasValid = d, true
}
}
return disp, hasValid
}
// suppressContained applies longest-match / whole-entity replacement (A3), DISPOSITION-GATED (D39
// слой 4). It drops an occurrence fully contained inside a STRICTLY longer one, but ONLY when the
// longer one is BOTH (a) owned by a spoiler-VALID entry at this chapter (self-review #6: a spoiler-
// blocked longer key like 林动的父亲, until_ch=3, read at ch10, must not eat a valid nested 林动) AND
// (b) at least as TRUSTWORTHY as the nested match it would delete. A longer but LOWER-trust key
// (a draft/ambiguous 四代族长) must NOT suppress a nested HIGHER-trust key (an approved 族长): the draft
// is editor-excluded (renderEditorConstraintBlock is CONFIRMED-only), so eating the approved term
// leaves the reader with nothing AND the drop is invisible to post-check/retrieval-state — the exact
// term-drift code root the D38.5 seed-promote only patched by data (L3-recurrence). The whole-entity
// longest-match is preserved when the longer key is ≥ trust (approved 四代族长 still beats approved
// 族长; draft 四代族长 still eats a nested draft 族长). Each REFUSED suppression is recorded as a
// trustGateEvent so the near-silent drop is surfaced loudly (research/13 §7). Equal-length overlaps
// are both kept (genuine surface ambiguity). Deterministic (ms is sorted; events deduped by pair in
// ms order). O(n²) over the few matches in a chunk.
func (b *MemoryBank) suppressContained(ms []acMatch, chapter int) ([]acMatch, []trustGateEvent) {
var kept []acMatch
var gated []trustGateEvent
for i, m := range ms {
conDisp, conValid := b.matchTrust(m, chapter)
contained := false
var refused *trustGateEvent // a lower-trust valid longer key that WANTED to suppress m
for j, o := range ms {
if i == j {
continue
}
if o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start) && validSuppressor(o) {
contained = true
// o must STRICTLY contain m's span to be a longest-match suppressor.
if !(o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start)) {
continue
}
supDisp, supValid := b.matchTrust(o, chapter)
if !supValid {
continue // spoiler-blocked longer key — never a valid suppressor (self-review #6)
}
if !conValid || trustRank(supDisp) >= trustRank(conDisp) {
contained = true // a valid, ≥-trust container wins the whole-entity longest-match
break
}
// supValid && conValid && supDisp < conDisp: a lower-trust longer key wants to eat a
// higher-trust nested one — REFUSE (the L3 fix). Remember the FIRST such refusal for the
// loud record; keep scanning in case a higher-trust container legitimately suppresses m.
if refused == nil {
refused = &trustGateEvent{
Suppressor: b.ac.keys[o.keyIdx], SuppressorDisp: string(supDisp),
Protected: b.ac.keys[m.keyIdx], ProtectedDisp: string(conDisp),
}
}
}
if contained {
continue // m suppressed by the whole-entity longest-match; any refusal above was moot
}
if !contained {
kept = append(kept, m)
if refused != nil {
gated = append(gated, *refused) // m survived AND a lower-trust key tried to eat it — loud
}
}
return kept
return kept, dedupeTrustGate(gated)
}
// dedupeTrustGate collapses identical (suppressor→protected) events to a distinct set, preserving
// first-occurrence (ms) order so the count and detail are deterministic and count distinct term
// collisions rather than raw occurrences.
func dedupeTrustGate(evs []trustGateEvent) []trustGateEvent {
if len(evs) < 2 {
return evs
}
seen := map[[2]string]bool{}
var out []trustGateEvent
for _, e := range evs {
k := [2]string{e.Suppressor, e.Protected}
if seen[k] {
continue
}
seen[k] = true
out = append(out, e)
}
return out
}
// suppressUnboundedPhonetic drops an occurrence of a SPACED-SCRIPT phonetic key (Latin or

View file

@ -0,0 +1,126 @@
package pipeline
import (
"strings"
"testing"
"textmachine/backend/internal/store"
)
// memory_trustgate_test.go pins the DISPOSITION-GATED longest-match suppressor (D39 слой 4,
// L3-suppressor-disposition-blind): a longer but LOWER-trust key (draft/ambiguous 四代族长) must NOT
// suppress a nested HIGHER-trust key (approved 族长) — the term-drift code root. The whole-entity
// longest-match is preserved when the longer key is ≥ trust. Each refused suppression is recorded
// loudly (trustGated). Han keys are used so normalizeSourceKey is (near) identity — the stored
// event keys equal the raw src.
// trustGateKeys collects the (suppressor→protected) pairs of a selection's trust-gate events.
func trustGateKeys(sel memorySelection) map[string]string {
m := map[string]string{}
for _, e := range sel.trustGated {
m[e.Suppressor] = e.Protected
}
return m
}
// TestSuppressorTrustGateFires is the FIX: a longer draft key nested over an approved shorter key no
// longer eats it. BEFORE the gate the approved 族长 was dropped from Select (never injected, invisible
// to post-check) and the draft 四代族长 injected only AMBIGUOUS (editor-excluded) → the reader got
// NOTHING. AFTER: the approved 族长 survives as CONFIRMED (editor gets it) and the collision is logged.
func TestSuppressorTrustGateFires(t *testing.T) {
zuzhang := gl("族长", "глава клана", "title", "approved")
zuzhang.Decl = declJSON(false, "глава клана", "главы клана", "главе клана")
sidai := gl("四代族长", "четвёртый глава клана", "title", "draft") // longer, LOWER trust
b := bankFrom([]store.GlossaryEntry{zuzhang, sidai})
sel := b.Select("四代族长 вошёл в зал.", 1, nil, 0)
m := injMap(sel)
if m["族长"] != memConfirmed {
t.Fatalf("approved 族长 must survive as CONFIRMED (the fix), got %q; injected=%v", m["族长"], m)
}
if m["四代族长"] != memAmbiguous {
t.Errorf("draft 四代族长 must still inject AMBIGUOUS alongside, got %q", m["四代族长"])
}
// Editor constraint block: the approved term is now present (was empty before the fix).
if blk := renderEditorConstraintBlock(sel.injected); !strings.Contains(blk, "глава клана") {
t.Errorf("editor constraint block must carry the approved «глава клана», got %q", blk)
}
// Loud record of the refused suppression.
if got := trustGateKeys(sel); got["四代族长"] != "族长" {
t.Errorf("expected a trust-gate event 四代族长⊃族长, got %v", got)
}
if len(sel.trustGated) != 1 {
t.Errorf("expected exactly 1 trust-gate event, got %d", len(sel.trustGated))
}
}
// TestSuppressorApprovedOverApprovedStillSuppresses is the regression-safety guard: an APPROVED
// longer key legitimately eats a nested APPROVED shorter one (whole-entity longest-match, unchanged).
// This also spot-verifies the D38.5 promote: once 四代族长 is promoted draft→approved the whole entity
// wins — but the FIX above means the promote is no longer LOAD-BEARING for correctness (a draft
// 四代族长 no longer silently drops 族长).
func TestSuppressorApprovedOverApprovedStillSuppresses(t *testing.T) {
zuzhang := gl("族长", "глава клана", "title", "approved")
zuzhang.Decl = declJSON(false, "глава клана")
sidai := gl("四代族长", "четвёртый глава клана", "title", "approved") // longer, EQUAL trust
b := bankFrom([]store.GlossaryEntry{zuzhang, sidai})
sel := b.Select("四代族长 вошёл в зал.", 1, nil, 0)
m := injMap(sel)
if _, present := m["族长"]; present {
t.Errorf("approved 族长 nested in an EQUAL-trust approved 四代族长 must be suppressed (whole-entity), got injected=%v", m)
}
if m["四代族长"] != memConfirmed {
t.Errorf("the whole entity 四代族长 must inject CONFIRMED, got %q", m["四代族长"])
}
if len(sel.trustGated) != 0 {
t.Errorf("an EQUAL-trust suppression is legitimate — no trust-gate event, got %d", len(sel.trustGated))
}
}
// TestSuppressorDraftOverDraftStillSuppresses pins the edge the task calls out: two DRAFT keys
// (equal trust) behave as before — the longer suppresses the nested, no trust-gate event.
func TestSuppressorDraftOverDraftStillSuppresses(t *testing.T) {
zuzhang := gl("族长", "глава клана", "title", "draft")
sidai := gl("四代族长", "четвёртый глава клана", "title", "draft")
b := bankFrom([]store.GlossaryEntry{zuzhang, sidai})
sel := b.Select("四代族长 вошёл в зал.", 1, nil, 0)
m := injMap(sel)
if _, present := m["族长"]; present {
t.Errorf("draft 族长 nested in an equal-trust draft 四代族长 must be suppressed, got %v", m)
}
if m["四代族长"] != memAmbiguous {
t.Errorf("the longer draft must inject AMBIGUOUS, got %q", m["四代族长"])
}
if len(sel.trustGated) != 0 {
t.Errorf("equal-trust suppression records no event, got %d", len(sel.trustGated))
}
}
// TestSuppressorSpoilerBlockedLongDoesNotSuppress preserves self-review #6: a spoiler-BLOCKED longer
// key is not a valid suppressor at all, so the nested valid approved key survives — and this is NOT a
// trust-gate event (the refusal path records only a spoiler-VALID lower-trust suppressor).
func TestSuppressorSpoilerBlockedLongDoesNotSuppress(t *testing.T) {
zuzhang := gl("族长", "глава клана", "title", "approved")
zuzhang.Decl = declJSON(false, "глава клана")
sidai := gl("四代族长", "четвёртый глава клана", "title", "approved")
sidai.SinceCh = 200 // spoiler-blocked before ch200
b := bankFrom([]store.GlossaryEntry{zuzhang, sidai})
sel := b.Select("四代族长 вошёл в зал.", 1, nil, 0) // ch1 → 四代族长 blocked
m := injMap(sel)
if m["族长"] != memConfirmed {
t.Errorf("approved 族长 must survive a spoiler-blocked longer key, got %q", m["族长"])
}
if len(sel.rejected) != 1 || sel.rejected[0].entry.src != "四代族长" {
t.Errorf("the spoiler-blocked 四代族长 must be rejected+logged, got %v", sel.rejected)
}
if len(sel.trustGated) != 0 {
t.Errorf("a spoiler-block is not a trust-gate event, got %d", len(sel.trustGated))
}
}

View file

@ -0,0 +1,197 @@
package pipeline
import (
"encoding/json"
"strings"
)
// quality.go: the DETERMINISTIC per-run quality-report (D39 слой 5, H5-no-in-loop-quality-signal) —
// the online/offline quality telemetry the owner asked for "from day one" (п.25/п.31), which
// research/18 §C1 #10 flagged as a NOW lever but was silently deferred to Ф2. It AGGREGATES signals
// that are ALREADY computed and stored (retrieval_state: glossary post-check misses, cheap style
// flaggers, trust-gated suppressions; chunk_status: echo/CJK/sanitizer flags) plus ONE cheap
// deterministic structural KPI (sentences per narrative paragraph — the "рубленые абзацы" claim-1
// signal) recomputed from the exported final text. It is a PURE READ-ONLY projection like Status: $0,
// no LLM, no snapshot touch, no checkpoint replay — only OBSERVABILITY, never a gate. The semantic
// span-judge (inversion/omission backstop for claim-2) is NOT here — it is research-dependent (пак-2).
// QualityReport is the whole-book per-run quality projection.
type QualityReport struct {
BookID string `json:"book_id"`
TotalChunks int `json:"total_chunks"`
// TextChunks is the number of chunks whose exported final text was available for the structural
// KPI (done or cosmetically-stripped); a flagged-empty chunk contributes no prose.
TextChunks int `json:"text_chunks"`
// ProcessedChunks is the number of chunks that REACHED the final stage (a final-stage row exists:
// ok, cosmetic-strip, or skipped-because-upstream-flagged) — the rate denominator, so echo/CJK
// rates are a bounded [0,1] fraction of processed chunks (an echo chunk is a skipped final row,
// disjoint from the exported-text set, which is why dividing by TextChunks alone could exceed 1).
ProcessedChunks int `json:"processed_chunks"`
// Claim-1 structural KPI (рубленые абзацы). MeanSentPerNarrPara ≈ 1 is choppy (one sentence per
// paragraph — the owner's exact complaint); higher is merged discourse prose. Aggregated as
// total sentences / total narrative paragraphs across the book, so it is a true book-wide mean.
NarrativeSentences int `json:"narrative_sentences"`
NarrativeParagraphs int `json:"narrative_paragraphs"`
MeanSentPerNarrPara float64 `json:"mean_sentences_per_narrative_paragraph"`
// Deterministic signal aggregates (all observability, never a disposition).
DialogueDashFlags int `json:"dialogue_dash_flags"` // Rosenthal dialogue-dash inconsistencies
GlossaryMisses int `json:"glossary_misses"` // CONFIRMED post-check misses (D10 consistency)
NumberDriftFlags int `json:"number_drift_flags"` // reflow number drift + 万/億 magnitude drift
TrustGated int `json:"trust_gated"` // lower-trust suppressions refused (seed hygiene)
CJKLeakChunks int `json:"cjk_leak_chunks"` // chunks the sanitizer stripped a CJK leak from
EchoChunks int `json:"echo_chunks"` // chunks flagged cjk_artifact (untranslated echo)
// Rates over ProcessedChunks (0..1) for the two "instant unreadability" families, so the numbers
// read as a bounded fraction of processed chunks rather than a bare count.
CJKLeakRate float64 `json:"cjk_leak_rate"`
EchoRate float64 `json:"echo_rate"`
Chunks []ChunkQuality `json:"chunks,omitempty"`
}
// ChunkQuality is one chunk's per-chunk quality row (the "where did quality slip" signal).
type ChunkQuality struct {
Chapter int `json:"chapter"`
ChunkIdx int `json:"chunk_idx"`
NarrativeSentences int `json:"narrative_sentences"`
NarrativeParagraphs int `json:"narrative_paragraphs"`
DialogueDashFlags int `json:"dialogue_dash_flags"`
GlossaryMisses int `json:"glossary_misses"`
NumberDriftFlags int `json:"number_drift_flags"`
TrustGated int `json:"trust_gated"`
}
// narrativeStructure computes the claim-1 structural KPI over a FINAL Russian text: the count of
// NARRATIVE paragraphs (non-empty lines that are NOT a dialogue turn opened with a dash «—»/«–»/«-»)
// and the total sentences within them (the oracle-parity splitSentences). Dialogue turns are excluded
// (they are legitimately one short line). Deterministic and pure — the same signal the reflow lever
// is supposed to move (exp14 mean-sentences-per-paragraph), reused here as in-loop observability.
func narrativeStructure(final string) (sentences, paragraphs int) {
for _, line := range strings.Split(final, "\n") {
t := strings.TrimSpace(line)
if t == "" {
continue
}
if rs := []rune(t); rs[0] == '—' || rs[0] == '' || rs[0] == '-' {
continue // a dialogue turn — not a narrative paragraph
}
paragraphs++
sentences += len(splitSentences(t))
}
return sentences, paragraphs
}
// QualityReport builds the read-only per-run quality projection. It opens no jobs, reserves nothing,
// makes no LLM call — it reads the persisted chunk_status / retrieval_state and, for each chunk with
// an exported final text, the $0 final checkpoint to recompute the structural KPI. Safe to run
// whenever `report`/`status` are (the same exclusive-lock rule). Deterministic over the store.
// CAVEAT (same class as Status's config-drift): the exported-text signals key on the CURRENT config's
// final-stage NAME; if a config edit renamed the final stage since the run, the stored rows use the
// old name and the structural KPI / echo / CJK rates read 0 (the underlying spend/verdict rows are
// untouched — surface `status` shows the drift). A run under the same config reads correctly.
func (r *Runner) QualityReport() (*QualityReport, error) {
statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
if err != nil {
return nil, err
}
states, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
if err != nil {
return nil, err
}
rep := &QualityReport{BookID: r.Book.BookID}
byChunk := map[chunkKey]*ChunkQuality{}
order := []chunkKey{}
chunkOf := func(k chunkKey) *ChunkQuality {
if q := byChunk[k]; q != nil {
return q
}
q := &ChunkQuality{Chapter: k.chapter, ChunkIdx: k.chunkIdx}
byChunk[k] = q
order = append(order, k)
return q
}
// Aggregate the stored retrieval-state signals (glossary consistency, style breakdown, trust-gated).
for _, rs := range states {
q := chunkOf(chunkKey{rs.Chapter, rs.ChunkIdx})
q.GlossaryMisses += rs.NPostcheckMiss
q.TrustGated += rs.NTrustGatedSuppress
rep.GlossaryMisses += rs.NPostcheckMiss
rep.TrustGated += rs.NTrustGatedSuppress
if rs.NStyleFlags > 0 && rs.StyleDetail != "" {
var cg cheapGateResult
if json.Unmarshal([]byte(rs.StyleDetail), &cg) == nil {
dash := cg.DialogueDash
drift := cg.NumberDrift + cg.NumberMagnitude
q.DialogueDashFlags += dash
q.NumberDriftFlags += drift
rep.DialogueDashFlags += dash
rep.NumberDriftFlags += drift
}
}
}
// The exported-text chunks (the FINAL stage's row): echo/CJK flag rates + the structural KPI.
lastStage := ""
if n := len(r.Pipeline.Stages); n > 0 {
lastStage = r.Pipeline.Stages[n-1].Name
}
seen := map[chunkKey]bool{}
for _, cs := range statuses {
k := chunkKey{cs.Chapter, cs.ChunkIdx}
if !seen[k] {
seen[k] = true
rep.TotalChunks++
}
if cs.Stage != lastStage {
continue // the exported text and the final verdict live on the final stage's row
}
// Every chunk that REACHED the final stage has exactly one lastStage row (ok, cosmetic-strip,
// or skipped-because-an-upstream-stage-flagged). This is the common rate denominator so the
// echo/CJK rates are a bounded fraction of processed chunks (an echo chunk is a skipped final
// row, disjoint from the exported-text set — dividing by TextChunks alone could exceed 100%).
rep.ProcessedChunks++
if cs.FlagReason == string(FlagCJKArtifact) {
rep.EchoChunks++
chunkOf(k) // ensure the chunk row exists even if it has no retrieval-state signals
}
if cs.FlagReason == string(FlagSanitizerStripped) {
rep.CJKLeakChunks++ // a stripped chunk carried a markdown/CJK cosmetic leak (detail says which)
}
// Structural KPI: recompute over the exported final text (ok, or the cosmetic-stripped export).
if cs.FinalHash == "" {
continue
}
if cs.Disposition != string(DispOK) && cs.FlagReason != string(FlagSanitizerStripped) {
continue // a dropped chunk exported nothing
}
cp, cperr := r.Store.GetCheckpoint(cs.FinalHash)
if cperr != nil {
return nil, cperr
}
if cp == nil || strings.TrimSpace(cp.ResponseText) == "" {
continue
}
sent, para := narrativeStructure(exportNormalize(cp.ResponseText))
q := chunkOf(k)
q.NarrativeSentences, q.NarrativeParagraphs = sent, para
rep.NarrativeSentences += sent
rep.NarrativeParagraphs += para
rep.TextChunks++
}
if rep.NarrativeParagraphs > 0 {
rep.MeanSentPerNarrPara = float64(rep.NarrativeSentences) / float64(rep.NarrativeParagraphs)
}
if rep.ProcessedChunks > 0 {
rep.CJKLeakRate = float64(rep.CJKLeakChunks) / float64(rep.ProcessedChunks)
rep.EchoRate = float64(rep.EchoChunks) / float64(rep.ProcessedChunks)
}
for _, k := range order {
rep.Chunks = append(rep.Chunks, *byChunk[k])
}
return rep, nil
}

View file

@ -0,0 +1,85 @@
package pipeline
import (
"context"
"testing"
)
// quality_test.go pins the per-run quality-report (D39 слой 5, H5): the claim-1 structural KPI and
// the aggregation of the already-stored deterministic signals. Pure read-only projection.
// TestNarrativeStructure pins the claim-1 KPI: narrative paragraphs (non-dialogue lines) and their
// sentence count. A choppy text (one sentence per line) reads ≈1.0; merged prose reads higher;
// dialogue turns are excluded.
func TestNarrativeStructure(t *testing.T) {
cases := []struct {
name string
text string
wantSent int
wantPara int
}{
// Choppy: 3 one-sentence narrative lines → 3 sentences / 3 paragraphs = 1.0 (претензия-1).
{"choppy", "Он вышел.\nНебо серело.\nОн вздохнул.", 3, 3},
// Merged: one paragraph, 3 sentences → 3 / 1 = 3.0 (the reflow fix).
{"merged", "Он вышел за дверь. Небо серело. Он вздохнул, глядя вдаль.", 3, 1},
// Dialogue turns excluded; only the narrative line counts.
{"dialogue-excluded", "Дядя нахмурился.\n— Время летит. Садись, — сказал он.\n— Спасибо.", 1, 1},
// Blank lines skipped; a two-sentence narrative paragraph plus a one-sentence one.
{"mixed", "Первый абзац. Ещё предложение.\n\nВторой абзац.", 3, 2},
}
for _, c := range cases {
sent, para := narrativeStructure(c.text)
if sent != c.wantSent || para != c.wantPara {
t.Errorf("%s: narrativeStructure = (%d sent, %d para), want (%d, %d)", c.name, sent, para, c.wantSent, c.wantPara)
}
}
}
// TestQualityReportAggregates runs a real 1-chunk book and asserts the read-only quality projection
// aggregates the structural KPI from the exported final text plus the stored signals.
func TestQualityReportAggregates(t *testing.T) {
rec := &reqRec{}
// The editor (final stage) emits a KNOWN 3-line final: 2 narrative paragraphs (3 sentences total)
// + 1 dialogue turn (excluded) → mean 1.5 sentences/narrative-paragraph.
editorFinal := "Судзуки шёл по коридору. Он думал о вчерашней лекции.\nБиблиотека была тихой.\n— Доброе утро, — сказал он."
srv := newJSONProvider(rec, func(body string) (string, string) {
if isEditBody(body) {
return editorFinal, "stop"
}
return "Судзуки шёл по коридору.", "stop"
})
defer srv.Close()
bookPath := setupProjectOpts(t, srv.URL, projectOpts{})
ctx := context.Background()
r := newRunner(t, bookPath)
defer r.Close()
if _, err := r.TranslateBook(ctx); err != nil {
t.Fatal(err)
}
q, err := r.QualityReport()
if err != nil {
t.Fatalf("QualityReport: %v", err)
}
if q.TotalChunks != 1 || q.TextChunks != 1 || q.ProcessedChunks != 1 {
t.Fatalf("chunk counts: total=%d processed=%d text=%d, want 1/1/1", q.TotalChunks, q.ProcessedChunks, q.TextChunks)
}
// Rates are bounded [0,1] over ProcessedChunks (the disjoint-set bug fix).
if q.EchoRate < 0 || q.EchoRate > 1 || q.CJKLeakRate < 0 || q.CJKLeakRate > 1 {
t.Errorf("rates must be within [0,1], got echo=%.3f cjk=%.3f", q.EchoRate, q.CJKLeakRate)
}
if q.NarrativeSentences != 3 || q.NarrativeParagraphs != 2 {
t.Errorf("structural KPI = %d sent / %d para, want 3/2", q.NarrativeSentences, q.NarrativeParagraphs)
}
if q.MeanSentPerNarrPara != 1.5 {
t.Errorf("mean sentences/narrative-paragraph = %.3f, want 1.5", q.MeanSentPerNarrPara)
}
// A clean run: no glossary misses, no echo, no CJK strip, no trust-gated, no dialogue-dash issue.
if q.GlossaryMisses != 0 || q.EchoChunks != 0 || q.CJKLeakChunks != 0 || q.TrustGated != 0 || q.DialogueDashFlags != 0 {
t.Errorf("clean run must have zero defect signals, got %+v", q)
}
if len(q.Chunks) != 1 || q.Chunks[0].NarrativeParagraphs != 2 {
t.Errorf("per-chunk row missing/wrong: %+v", q.Chunks)
}
}

View file

@ -22,8 +22,9 @@ import (
// translation — the only role the excision coverage-gate is meaningful on.
const roleTranslator = "translator"
// roleEditor is the monolingual editor stage (D1): it receives the draft plus the approved
// glossary's dst forms as target-consistency constraints, never the source.
// roleEditor is the BILINGUAL editor stage (D30.1 supersede of D1): editor.md feeds it BOTH the
// source and the draft, plus the approved glossary's CONFIRMED dst forms as target-consistency
// constraints. (The stale "monolingual, never sees the source" framing is D1, fully superseded.)
const roleEditor = "editor"
// Runner executes a book's pipeline.
@ -145,8 +146,16 @@ func openRunner(bookPath string, logger *slog.Logger, forWrite bool) (*Runner, e
func (r *Runner) Close() error { return r.Store.Close() }
func (r *Runner) loadTemplates() error {
pair := r.Book.LangPair()
for _, st := range r.Pipeline.Stages {
tpl, err := LoadPromptTemplate(st.Prompt)
// Resolve the stage's prompt from the pair-keyed pack by the book's language pair (D39 слой 2):
// a legacy single `prompt` is pair-agnostic; a `prompts` pack fails LOUD on a missing pair here
// (a ja book against a zh-ru-only pack stops instead of silently running Chinese conventions).
promptPath, err := st.PromptPathFor(pair)
if err != nil {
return err
}
tpl, err := LoadPromptTemplate(promptPath)
if err != nil {
return err
}

View file

@ -0,0 +1,91 @@
package pipeline
import (
"fmt"
"path/filepath"
"strings"
"testing"
"time"
"textmachine/backend/internal/obs"
)
// runner_pairpack_test.go is the end-to-end proof of the pair-keyed prompt seam (D39 слой 2): a book
// whose LangPair() is NOT in a stage's pair pack FAILS LOUD at loadTemplates (closing the latent bug
// where a ja book rode the zh-parataxis prompt), while a book of the populated pair resolves cleanly.
// setupPairPackProject writes a minimal project whose stages use a pair-keyed `prompts` pack carrying
// ONLY zh-ru, with the book's source_lang set to srcLang. Returns the book.yaml path.
func setupPairPackProject(t *testing.T, srcLang string) string {
t.Helper()
dir := t.TempDir()
writeFile(t, filepath.Join(dir, "prompts", "translator.md"), "Переводи.\n---USER---\n{{text}}")
writeFile(t, filepath.Join(dir, "prompts", "editor.md"), "Редактируй.\n---USER---\n{{text}}\n{{draft}}")
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
prices_checked: %q
default_model: fake-model
providers:
fake: { kind: openai, base_url: http://127.0.0.1:0 }
models:
fake-model: { provider: fake, price: { input_per_m: 1.0, output_per_m: 2.0 } }
`, time.Now().UTC().Format("2006-01-02")))
// Pair-keyed pack — ONLY zh-ru is populated.
writeFile(t, filepath.Join(dir, "pipeline.yaml"), `
core: C1
version: 1
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
stages:
- name: draft
role: translator
model: fake-model
prompts: { zh-ru: prompts/translator.md }
prompt_version: v-test
reasoning: "off"
- name: edit
role: editor
model: fake-model
prompts: { zh-ru: prompts/editor.md }
prompt_version: v-test
reasoning: "off"
`)
writeFile(t, filepath.Join(dir, "source.txt"), "四代族长 вошёл в зал.")
writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(`
book_id: pairpack
title: Тест
source_lang: %s
target_lang: ru
genre: ранобэ
audience: тест
venuti: 0.5
honorifics: keep
transcription: polivanov
pipeline: pipeline.yaml
models: models.yaml
source_file: source.txt
ceilings: { book_usd: 1.0, day_usd: 2.0 }
`, srcLang))
return filepath.Join(dir, "book.yaml")
}
func TestPairPackJaBookFailsLoud(t *testing.T) {
// A ja→ru book against a zh-ru-only pack must STOP at loadTemplates, not silently run zh prompts.
_, err := NewRunner(setupPairPackProject(t, "ja"), obs.NewLogger())
if err == nil {
t.Fatal("a ja book against a zh-ru-only prompt pack must fail loud (D39 слой 2)")
}
if !strings.Contains(err.Error(), "ja-ru") {
t.Errorf("fail-loud error must name the missing pair ja-ru, got: %v", err)
}
}
func TestPairPackZhBookResolves(t *testing.T) {
// A zh→ru book of the populated pair resolves the pack cleanly (prod path, behaviour-neutral).
r, err := NewRunner(setupPairPackProject(t, "zh"), obs.NewLogger())
if err != nil {
t.Fatalf("a zh book must resolve the zh-ru pack: %v", err)
}
defer r.Close()
if len(r.templates) != 2 {
t.Errorf("expected both stage templates loaded, got %d", len(r.templates))
}
}

View file

@ -6,6 +6,9 @@ import (
"sort"
"strings"
"unicode"
"golang.org/x/text/unicode/norm"
"golang.org/x/text/width"
)
// sanitizer.go: the output-sanitizer (D30.3) — a deterministic verdict-axis gate on the
@ -66,7 +69,16 @@ import (
// («»→«123», not deleted) and ideographic comma/period, space-replaces dropped ideographs (no
// word-merge), removes an emptied bracket pair, and no longer reformats a legit «2 : 1» — all shift
// a stripped chunk's EXPORT text → a loud --resnapshot + golden re-capture (invariant №8).
const sanitizerVersion = "sanitizer-v4"
// v5 (D39 слой 6, export-contract): the detector is now a PURE detector over the export-normalised
// text — the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «»→«123», ideographic space
// U+3000, «、»→«,» «。»→«.») are folded by the shared recoverableFold (reusing the x/text width.Fold
// primitive instead of the hand-rolled «r-0xFEE0» arithmetic, closing L5-stripcosmetic-duplicates-nfkc;
// deliberately NOT full NFKC, which would clobber «…»/«№») and therefore NO LONGER FIRE the CJK-leak
// class — only genuinely contentless CJK (Han/kana/CJK brackets & symbols) does. stripCosmetic
// delegates that fold too. This shifts which chunks flag and a stripped chunk's export → a loud
// --resnapshot + golden re-capture (invariant №8). [exportNormalize itself is an EPHEMERAL export
// projection, not a persisted input; only stripCosmetic's output rides a checkpoint, versioned here.]
const sanitizerVersion = "sanitizer-v5"
// sanitizer tuning constants (versioned by sanitizerVersion).
const (
@ -355,37 +367,44 @@ func detectBrokenWords(text string) (int, []string) {
// --- 6. CJK-leak in the final --------------------------------------------------
// isCJKLeakRune reports whether r is a stray CJK glyph that has NO place in a Russian final: a
// Han ideograph, kana, a fullwidth/halfwidth form (U+FF00FFEF — «,()!» etc.), or a CJK
// symbol/punctuation (U+3001303F — «、。「」»). The ideographic SPACE U+3000 is deliberately
// EXCLUDED (it is an indent artifact, not a content leak — research/18 §C#4 "draft-21 includes a
// U+3000 indent → 9 real leaks"): stripCosmetic normalises it to a plain space, but it does not
// alone FIRE the class, so a legitimately-indented chunk is not flag-stormed. In clean Russian
// prose every one of these is count-0, so the class is high precision. The intrinsic classify()
// already flags a ≥15%-CJK output as an ECHO (cjk_artifact, substantive) BEFORE the sanitizer
// runs, so this class only ever sees a SPARSE leak in otherwise-Russian text.
// isCJKLeakRune reports whether r is a genuinely CONTENTLESS CJK glyph that has NO place in a Russian
// final AND cannot be recovered by the export normaliser: a Han ideograph, kana, or a CJK
// symbol/bracket/punctuation (U+3001303F — «「」『』【】〜※»). It is ALWAYS consulted on
// recoverableFold-ed text (detectCJKLeak/stripCosmetic fold first), so the RECOVERABLE wrong-glyph
// forms are already gone before this runs: fullwidth ASCII «!1» folded to «!1», the ideographic
// space U+3000 folded to a plain space, and «、»/«。» mapped to «,»/«.». Those must NOT fire — the
// export contract silently fixes them (D39 слой 6); only the contentless class remains here. ACCEPTED
// NARROWING (precision over recall): the old code fired on the WHOLE U+FF00FFEF block; width.Fold now
// recovers the common fullwidth-ASCII/symbol forms to real glyphs (so they no longer flag), but a few
// RARE legacy forms in that block (halfwidth hangul jamo, halfwidth arrows/symbols, the katakana
// middle dot) fold to non-CJK glyphs this predicate does not catch and ship un-flagged — an
// extremely improbable artifact in zh/ja→ru output, traded for not over-flagging the common case. In
// clean Russian prose every one of these is count-0, so the class is high precision. The intrinsic
// classify() already flags a ≥15%-CJK output as an ECHO (cjk_artifact, substantive) BEFORE the
// sanitizer runs, so this class only ever sees a SPARSE leak in otherwise-Russian text.
func isCJKLeakRune(r rune) bool {
switch {
case unicode.Is(unicode.Han, r):
return true
case unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r):
return true
case r >= 0xFF00 && r <= 0xFFEF: // fullwidth & halfwidth forms
return true
case r >= 0x3001 && r <= 0x303F: // CJK symbols & punctuation, EXCLUDING U+3000 (ideographic space)
return true
}
return false
}
// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail
// per distinct run. A single run («却有») is enough to fire — the class is COSMETIC, so the whole
// chunk is not lost: stripCosmetic removes the run and the remainder exports (flagged for review).
// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail per
// distinct run. It runs over the RECOVERABLE-FOLD of the text (D39 слой 6): a fullwidth «123» or a
// «、» is a wrong-glyph rendering the export contract silently fixes, so it is NOT a leak — only a
// genuinely contentless Han/kana/CJK-bracket run is. A single run («却有») is enough to fire — the
// class is COSMETIC, so the whole chunk is not lost: stripCosmetic removes the run and the remainder
// exports (flagged for review).
func detectCJKLeak(text string) (int, []string) {
var det []string
seen := map[string]bool{}
n := 0
rs := []rune(text)
rs := []rune(recoverableFold(text))
for i := 0; i < len(rs); {
if !isCJKLeakRune(rs[i]) {
i++
@ -415,41 +434,98 @@ func detectCJKLeak(text string) (int, []string) {
// after the hashes) does NOT match, so it is left intact (and it never fired the class either).
var leadingMarkdownStripRE = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+([\p{L}\p{Nd}])`)
// --- export-contract normalization (D39 слой 6, D29.1a) --------------------------
// recoverableFold folds the RECOVERABLE wrong-glyph Unicode renderings a Russian final can carry into
// their intended characters — the shared primitive of the export contract. It is a WIDTH fold
// (width.Fold: fullwidth ASCII «!12» → «!12», the ideographic space U+3000 → a plain space,
// halfwidth kana → fullwidth kana) plus the two CJK punctuation glyphs the width fold leaves alone
// («、» → «,», «。» → «.»). It reuses the x/text width.Fold primitive instead of the old hand-rolled
// «r-0xFEE0» arithmetic (L5-stripcosmetic-duplicates-nfkc) — but deliberately NOT full norm.NFKC,
// which would over-fold legitimate Russian typography (NFKC decomposes «…»→«...», «№²½»→…), a
// regression the golden strip contract forbids. Used by exportNormalize (every final), stripCosmetic
// (before dropping the contentless leak) and detectCJKLeak (which flags only what SURVIVES the fold —
// a genuinely contentless Han/kana/CJK-bracket, never a foldable glyph). Pure.
func recoverableFold(text string) string {
return ideographicPunctFold.Replace(width.Fold.String(text))
}
// ideographicPunctFold maps the two CJK punctuation glyphs the width fold does NOT touch to their
// ASCII equivalents (a «、»/«。» in a Russian final is an unambiguous wrong-glyph rendering of «,»/«.»).
var ideographicPunctFold = strings.NewReplacer("、", ",", "。", ".")
// exportNormalize is the export-contract normalization (D29.1a / target arch слой 6) applied to
// EVERY final text before it ships (chunkrun.ChunkOutcome.FinalText), clean OR flagged-stripped — the
// fix for a cosmetic defect being normalised on a flagged chunk but shipped RAW on a clean one
// (L5-normalization-fused-to-flag-path). It folds the recoverable wrong glyphs (recoverableFold),
// strips a stray combining stress mark on a Cyrillic base («источа́ло»→«источало»,
// L5-diacritics-class-lost), and trims trailing horizontal whitespace per line (never a newline, so
// paragraph structure is kept). It is a PURE, IDEMPOTENT projection over the final text: it never
// touches the wire, the stored checkpoint or the disposition — only the exported bytes — so it is
// NOT a snapshot/verdict input (recomputed each run; a change re-projects existing checkpoints for
// free, no re-bill). Deliberately conservative on whitespace: it does NOT collapse internal doubled
// spaces (a clean chunk's spacing is left as the model wrote it).
func exportNormalize(text string) string {
out := recoverableFold(text)
// NFC composes a canonically-decomposed base+mark (a «й» stored as и+U+0306, a «ё» as е+U+0308)
// BEFORE the combining-strip below, so the strip only removes the NON-composable stress accents
// (U+0301 on a vowel), never corrupting «й»/«ё» into «и»/«е». NFC is canonical-only, so it leaves
// «…» and other compatibility characters intact (unlike NFKC).
out = norm.NFC.String(out)
out = stripCombiningOnCyrillic(out)
return trailingHSpaceRE.ReplaceAllString(out, "")
}
// stripCombiningOnCyrillic drops a combining mark (unicode.Mn) that sits directly on a CYRILLIC base
// — the leaked stress-accent defect the exp12 flagman named («источа́ло» → «источало»,
// L5-diacritics-class-lost). Narrow and zero-FP for Russian prose (which never carries combining
// stress marks); a combining mark on a Latin base (a foreign name «é») is preserved. Runs AFTER NFKC
// (recoverableFold), which has already composed any canonically-composable base+mark.
func stripCombiningOnCyrillic(s string) string {
var b strings.Builder
b.Grow(len(s))
prevCyrillic := false
for _, r := range s {
if unicode.Is(unicode.Mn, r) {
if prevCyrillic {
continue // drop the stress accent on a Cyrillic vowel; leave prevCyrillic set
}
b.WriteRune(r)
continue
}
b.WriteRune(r)
prevCyrillic = unicode.Is(unicode.Cyrillic, r)
}
return b.String()
}
// stripCosmetic deterministically removes the two STRIPPABLE cosmetic leak classes — leading
// markdown headers and CJK-leak runs — and tidies the whitespace the removal leaves, so the
// remainder is readable. It is a PURE function (resume re-derives the identical export), invoked
// ONLY on output the sanitizer classified as cosmeticOnly (classifyOutput guarantees the result
// is non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
// markdown headers and CONTENTLESS CJK-leak runs — and tidies the whitespace the removal leaves, so
// the remainder is readable. It is a PURE function (resume re-derives the identical export), invoked
// ONLY on output the sanitizer classified as cosmeticOnly (classifyOutput guarantees the result is
// non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
// sanitizer_stripped). It never crosses newlines when tidying, so paragraph structure is kept.
func stripCosmetic(text string) string {
// 1) markdown header prefixes → keep the heading text, drop the «#### » artifact.
out := leadingMarkdownStripRE.ReplaceAllString(text, "$1")
// 2) neutralise CJK-leak runes. CONTENT-BEARING glyphs are FOLDED, not dropped (adversarial
// review MAJOR): a fullwidth ASCII form is a wrong-glyph rendering of a plain char, so «123»
// folds to «123» (the number is kept, not deleted) and «,」→«,» — the same NFKC fold the
// memory normaliser uses. Only genuinely contentless runes (Han/kana/CJK brackets/symbols) are
// removed, and a removed run leaves a SINGLE SPACE so adjacent Russian words do not merge
// («нашёл特产древний» → «нашёл древний», not «нашёлдревний»). U+3000 → a plain space (indent).
// 2) fold the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «»→«123», U+3000→space,
// «、»→«,» «。»→«.») via the shared export-contract fold — no longer a hand-rolled NFKC
// (L5-stripcosmetic-duplicates-nfkc). After this only genuinely contentless CJK remains to drop.
out = recoverableFold(out)
// 3) drop the contentless CJK-leak runes (Han/kana/CJK brackets & symbols) the fold cannot
// recover, each leaving a SINGLE SPACE so adjacent Russian words do not merge («нашёл特产древний»
// → «нашёл древний», not «нашёлдревний»).
var b strings.Builder
b.Grow(len(out))
for _, r := range out {
switch {
case r == ' ': // U+3000 ideographic space → plain space
if isCJKLeakRune(r) {
b.WriteByte(' ')
case r >= 0xFF01 && r <= 0xFF5E: // fullwidth ASCII form → fold to plain ASCII (→1, →A, ,→,)
b.WriteRune(r - 0xFEE0)
case r == '、': // ideographic comma → plain comma (keep the clause boundary)
b.WriteByte(',')
case r == '。': // ideographic full stop → plain period
b.WriteByte('.')
case isCJKLeakRune(r): // Han / kana / CJK brackets / other symbol → drop, leaving a space
b.WriteByte(' ')
default:
} else {
b.WriteRune(r)
}
}
out = b.String()
// 3) tidy the seams the removal created, per line (horizontal whitespace only — never a
// 4) tidy the seams the removal created, per line (horizontal whitespace only — never a
// newline): collapse doubled spaces, drop a bracket/quote pair the strip emptied («(羅漢拳)» →
// «()» → removed), drop a space now sitting before closing punctuation or after an opening
// bracket/quote (never-correct Russian spacing), and trim trailing blanks. ':' is deliberately

View file

@ -219,8 +219,6 @@ func TestSanitizerCJKLeak(t *testing.T) {
"Судзуки нашёл 特产 древнего клана.", // single Han run
"Разряд资质乙等 остался неясен.", // Han glued to Cyrillic
"Он сказал: すごい, и замолчал.", // kana leak (ja book)
"Цена была золотых.", // fullwidth digits
"Он вошёл в дом、и дверь закрылась.", // ideographic comma (U+3001)
}
for _, s := range fire {
if got := sanitizeOutput(s); got.CJKLeak == 0 {
@ -233,8 +231,14 @@ func TestSanitizerCJKLeak(t *testing.T) {
"На гербе была латинская надпись, но он её не разобрал в темноте.",
"Цена была 123 золотых, и он расплатился не торгуясь.", // ASCII digits — not fullwidth
// U+3000 ideographic space alone is an INDENT artifact, NOT a content leak → must NOT fire
// (research/18 §C#4); stripCosmetic still normalises it, but it does not flag the chunk.
// (research/18 §C#4); the export contract normalises it, but it does not flag the chunk.
" Судзуки медленно шёл по длинному коридору к башне.",
// D39 слой 6: RECOVERABLE wrong-glyph renderings — fullwidth ASCII «123» and the ideographic
// comma «、» — no longer FIRE the leak class. The detector is a pure detector over the folded
// text (fullwidth «»→«123», «、»→«,»), and the export contract fixes them silently. Only
// genuinely contentless CJK (Han/kana above) is flagged.
"Цена была золотых.", // fullwidth digits → silently folded to «123»
"Он вошёл в дом、и дверь закрылась.", // ideographic comma (U+3001) → silently folded to «,»
}
for _, s := range clean {
if got := sanitizeOutput(s); got.CJKLeak != 0 {
@ -305,6 +309,45 @@ func TestStripCosmetic(t *testing.T) {
}
}
// TestExportNormalize pins the export-contract (D39 слой 6, D29.1a): every final text is normalised
// before it ships — a CLEAN chunk (which the sanitizer never flags) now gets its cosmetic Unicode
// artifacts fixed, not shipped raw (L5-normalization-fused-to-flag-path). Width fold + CJK-punct map
// + combining-stress strip; «…»/«№»/«й»/«ё» preserved; idempotent.
func TestExportNormalize(t *testing.T) {
cases := []struct{ in, want string }{
// The finding's exact case: a clean chunk with only a U+3000 indent shipped raw before.
{" Судзуки медленно шёл к башне.", " Судзуки медленно шёл к башне."},
// fullwidth ASCII folded (a clean chunk, never flagged) — the number/letters survive.
{"Цена была золотых, индекс .", "Цена была 123 золотых, индекс ABC."},
// ideographic comma/period → ASCII on a CLEAN chunk (no Han → sanitizer never fired).
{"Он вошёл в дом、дверь закрылась。", "Он вошёл в дом,дверь закрылась."},
// combining stress accent on a Cyrillic base dropped (L5-diacritics-class-lost): «источа́ло».
{"Он источа́ло тепло за́мка.", "Он источало тепло замка."},
// legit typography PRESERVED (the NFKC-would-clobber regression the width fold avoids).
{"Он ждал… но никто не пришёл. № 5, ½ часа.", "Он ждал… но никто не пришёл. № 5, ½ часа."},
// trailing horizontal whitespace trimmed per line; the newline is kept.
{"Первая строка. \nВторая строка.", "Первая строка.\nВторая строка."},
}
for _, c := range cases {
got := exportNormalize(c.in)
if got != c.want {
t.Errorf("exportNormalize(%q) = %q, want %q", c.in, got, c.want)
}
if again := exportNormalize(got); again != got {
t.Errorf("exportNormalize not idempotent for %q: %q → %q", c.in, got, again)
}
}
// A DECOMPOSED «й» (и U+0438 + combining breve U+0306) / «ё» (е U+0435 + diaeresis U+0308) must be
// COMPOSED by NFC, NOT corrupted to «и»/«е» by the combining-stress strip — the load-bearing reason
// NFC runs BEFORE stripCombiningOnCyrillic in exportNormalize.
if got := exportNormalize("\u0438\u0306"); got != "\u0439" { // decomposed й → composed «й»
t.Errorf("decomposed й corrupted by the combining strip: %q (want «й» U+0439)", got)
}
if got := exportNormalize("\u0435\u0308"); got != "\u0451" { // decomposed ё → composed «ё»
t.Errorf("decomposed ё corrupted by the combining strip: %q (want «ё» U+0451)", got)
}
}
// TestSanitizerDeterministic asserts the verdict is a pure function of the text (resume-safe).
func TestSanitizerDeterministic(t *testing.T) {
s := "### Заголовок\nВот отредактированный перевод: текст. Стольь странно.\nПримечание: правка."

View file

@ -244,49 +244,29 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
on := fewShotEnabled(st)
ss.FewShot = &on
}
if prov, ok := r.Models.Providers[r.Models.Models[st.Model].Provider]; ok {
ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = prov.Temperature, prov.MaxTokens, prov.Model
// Fold the PRIMARY model's wire-affecting inputs (prov-triple, extra_body, resolved capability)
// via the shared foldModelWire (D39 слой 7, L8-snapshot-fold-copypaste-tripwire): the same
// helper folds the escalate_to fallback below, so a third model axis (channel B / annotator)
// reuses it and cannot silently drift the snapshot.
primary, perr := r.foldModelWire(st.Model)
if perr != nil {
return "", "", perr
}
if extra := r.Models.Models[st.Model].ExtraBody; len(extra) > 0 {
raw, merr := json.Marshal(extra)
if merr != nil {
return "", "", merr
}
ss.ModelExtra = raw
}
// Резолвнутая каппа — тем же ResolveCapability, что и у клиента, поэтому
// хэшируемое всегда совпадает с отправляемым. json.Marshal сортирует
// ключи карт (off_extra_body) → детерминированный рендер.
capRaw, cerr := json.Marshal(r.Models.ResolveCapability(st.Model))
if cerr != nil {
return "", "", cerr
}
ss.Capability = capRaw
// Fold the single-hop fallback model + its resolved capability (Веха 2.5):
// both are wire-affecting inputs of an escalated call, so changing them is a
// loud --resnapshot, not a silent false-hit on an escalated checkpoint.
// TRIPWIRE (пакет №4): этот блок — зеркало primary-фолда выше (2×, ниже порога
// извлечения ≥3). Когда канал B/annotator добавит ТРЕТЬЮ модельную ось стадии —
// сначала извлечь общий foldModelWire(model) (prov-тройка, extra, capability),
// потом добавлять ось: копипаста здесь тише всего расходится и портит
// идентичность снапшота (false-hit/false-miss на эскалированных чекпоинтах).
ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = primary.provTemp, primary.provMaxTok, primary.provModel
ss.ModelExtra = primary.extra
ss.Capability = primary.capability
// Fold the single-hop fallback model + its wire inputs (Веха 2.5): both are wire-affecting
// inputs of an escalated call, so changing them is a loud --resnapshot, not a silent false-hit
// on an escalated checkpoint. Same fold as the primary — no copy-paste to drift.
if st.EscalateTo != "" {
ss.EscalateTo = st.EscalateTo
escCap, eerr := json.Marshal(r.Models.ResolveCapability(st.EscalateTo))
esc, eerr := r.foldModelWire(st.EscalateTo)
if eerr != nil {
return "", "", eerr
}
ss.EscalateCapability = escCap
if prov, ok := r.Models.Providers[r.Models.Models[st.EscalateTo].Provider]; ok {
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok, ss.EscalateProviderModel = prov.Temperature, prov.MaxTokens, prov.Model
}
if extra := r.Models.Models[st.EscalateTo].ExtraBody; len(extra) > 0 {
raw, merr := json.Marshal(extra)
if merr != nil {
return "", "", merr
}
ss.EscalateExtra = raw
}
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok, ss.EscalateProviderModel = esc.provTemp, esc.provMaxTok, esc.provModel
ss.EscalateExtra = esc.extra
ss.EscalateCapability = esc.capability
}
snap.Stages = append(snap.Stages, ss)
}
@ -297,3 +277,39 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:]), string(data), nil
}
// modelWireFold is a model's snapshot-folded wire-affecting inputs: its provider-level
// temperature/max_tokens/model overrides (local kind, applied AFTER the request-hash), its top-level
// extra_body, and its RESOLVED capability (budget-key/temperature-mode/reasoning-control). Extracted
// (D39 слой 7, L8-snapshot-fold-copypaste-tripwire) so the primary model, the escalate_to fallback,
// and any future third model axis fold the SAME way — the copy-paste the old tripwire warned about.
type modelWireFold struct {
provTemp float64
provMaxTok int
provModel string
extra json.RawMessage // nil when the model has no extra_body → omitted (omitempty)
capability json.RawMessage // always present (ResolveCapability, the same the client sends)
}
// foldModelWire renders one model's wire-fold. ResolveCapability matches exactly what the client
// sends, and json.Marshal sorts map keys (off_extra_body) → deterministic render. Returns an error
// only on a marshal failure (a struct of scalars/known types cannot fail in practice).
func (r *Runner) foldModelWire(model string) (modelWireFold, error) {
var f modelWireFold
if prov, ok := r.Models.Providers[r.Models.Models[model].Provider]; ok {
f.provTemp, f.provMaxTok, f.provModel = prov.Temperature, prov.MaxTokens, prov.Model
}
if extra := r.Models.Models[model].ExtraBody; len(extra) > 0 {
raw, err := json.Marshal(extra)
if err != nil {
return f, err
}
f.extra = raw
}
capRaw, err := json.Marshal(r.Models.ResolveCapability(model))
if err != nil {
return f, err
}
f.capability = capRaw
return f, nil
}

View file

@ -168,13 +168,15 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
if disposition == DispOK {
finalHash = last.reqHash
} else if last.cls.Reason == FlagSanitizerStripped {
// Cosmetic leak (D35.4a): strip it and COMMIT the cleaned remainder as a $0 derived export
// checkpoint, then point final_hash at it — so the standard final_hash→checkpoint.response_text
// export (records.json / exp12_extract) yields the cleaned text instead of empty, while the
// chunk stays flagged for a human. classifyOutput guaranteed the strip is non-empty and clean.
// The derived checkpoint is durably written BEFORE chunk_status references it (the ok path's
// checkpoint-before-pointer discipline), so a resume never dangles final_hash at a missing row.
recovered = stripCosmetic(last.text)
// Cosmetic leak (D35.4a): COMMIT the cleaned remainder as a $0 derived export checkpoint, then
// point final_hash at it — so the standard final_hash→checkpoint.response_text export
// (records.json / exp12_extract) yields the cleaned text instead of empty, while the chunk stays
// flagged for a human. The strip was computed ONCE by classifyOutput and rides on last.stripped
// (T3.4 / L8-stripcosmetic-recomputed-twice): runStage no longer re-derives it, so the exact
// bytes the "non-empty & clean" guard validated are what commit. The derived checkpoint is
// durably written BEFORE chunk_status references it (the ok path's checkpoint-before-pointer
// discipline), so a resume never dangles final_hash at a missing row.
recovered = last.stripped
dh, err := r.commitSanitizedExport(st, ch, job, last, attemptsMade-1, recovered)
if err != nil {
return nil, err
@ -242,6 +244,11 @@ func (r *Runner) commitSanitizedExport(st config.Stage, ch Chunk, job *store.Job
type stageAttempt struct {
reqHash string
cls classification
// stripped is the resolved cosmetic-strip EXPORT text classifyOutput computed ONCE (T3.4):
// non-empty only when cls.Reason == FlagSanitizerStripped. runStage ships it verbatim instead of
// re-deriving stripCosmetic, so the "non-empty & clean" invariant classifyOutput asserts is the
// exact text committed to the derived export checkpoint.
stripped string
text string
usage llm.Usage
finish string
@ -279,7 +286,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
att.finish = cp.FinishReason
att.modelActual = cp.ModelActual
att.cumCost = cp.CostUSD
att.cls = r.classifyOutput(st.Role, ch.Text, cp.ResponseText, cp.FinishReason, isFinal)
att.cls, att.stripped = r.classifyOutput(st.Role, ch.Text, cp.ResponseText, cp.FinishReason, isFinal)
rl := r.baseRequestLog(st, ch, model, reqHash)
rl.ModelActual = cp.ModelActual
rl.TMHit, rl.OK, rl.FinishReason, rl.Degraded = true, att.cls.ok(), cp.FinishReason, degradedTag(att.cls)
@ -445,7 +452,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
// non-empty truncated length draft is now classified (flagged/retried), never
// silently passed downstream as OK; an empty completion is flagged too, not a
// run-crash.
att.cls = r.classifyOutput(st.Role, ch.Text, resp.Text, resp.FinishReason, isFinal)
att.cls, att.stripped = r.classifyOutput(st.Role, ch.Text, resp.Text, resp.FinishReason, isFinal)
rl := r.baseRequestLog(st, ch, model, reqHash)
rl.ModelActual = modelActual

File diff suppressed because one or more lines are too long

View file

@ -74,6 +74,11 @@ type RetrievalState struct {
InjectedIDs string // JSON
NStyleFlags int // cheap style/number gate hits (dialogue-dash, ё, translit-interj, 万/億)
StyleDetail string // JSON breakdown of the style-flag hits
// NTrustGatedSuppress counts refused lower-trust longest-match suppressions (D39 слой 4): a
// longer draft/ambiguous key blocked from eating a nested approved/confirmed one. A nonzero
// count is a loud seed-hygiene signal (a draft term nesting over an approved term to reconcile).
NTrustGatedSuppress int
TrustGateDetail string // JSON of the refused suppressor→protected pairs
}
// ReplaceGlossary replaces a book's ENTIRE glossary (rows + aliases) with the given
@ -239,8 +244,8 @@ func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
book_id, chapter, chunk_idx, snapshot_id,
n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted,
embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
n_style_flags, style_detail, updated_at
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail, updated_at
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET
snapshot_id = excluded.snapshot_id,
n_exact_hits = excluded.n_exact_hits,
@ -254,11 +259,13 @@ func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
injected_ids = excluded.injected_ids,
n_style_flags = excluded.n_style_flags,
style_detail = excluded.style_detail,
n_trust_gated_suppress = excluded.n_trust_gated_suppress,
trust_gate_detail = excluded.trust_gate_detail,
updated_at = excluded.updated_at`,
rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID,
rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted,
rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs,
rs.NStyleFlags, rs.StyleDetail)
rs.NStyleFlags, rs.StyleDetail, rs.NTrustGatedSuppress, rs.TrustGateDetail)
return err
}
@ -270,12 +277,12 @@ func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*Retrie
err := s.r.QueryRowContext(ctx, `
SELECT snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked,
n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
n_style_flags, style_detail
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail
FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`,
bookID, chapter, chunkIdx).Scan(
&rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked,
&rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
&rs.NStyleFlags, &rs.StyleDetail)
&rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
@ -291,7 +298,7 @@ func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error)
return queryAll(s.r, `
SELECT chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged,
n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
n_style_flags, style_detail
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail
FROM retrieval_state WHERE book_id = ?
ORDER BY chapter, chunk_idx`,
func(rows *sql.Rows) (RetrievalState, error) {
@ -299,7 +306,7 @@ func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error)
err := rows.Scan(&rs.Chapter, &rs.ChunkIdx, &rs.SnapshotID, &rs.NExactHits, &rs.NSticky,
&rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed,
&rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
&rs.NStyleFlags, &rs.StyleDetail)
&rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail)
return rs, err
}, bookID)
}

View file

@ -272,6 +272,16 @@ var migrations = []string{
ALTER TABLE retrieval_state ADD COLUMN n_style_flags INTEGER NOT NULL DEFAULT 0;
ALTER TABLE retrieval_state ADD COLUMN style_detail TEXT NOT NULL DEFAULT '';
`,
// v8 (D39 слой 4): the disposition-gated suppressor's LOUD record. n_trust_gated_suppress counts
// the times a longer but LOWER-trust glossary key (draft/ambiguous) was REFUSED from suppressing a
// nested HIGHER-trust key (approved/confirmed) — the term-drift code root, converted from a silent
// drop into a visible signal (research/13 §7). A nonzero count means the seed has a draft term
// nesting over an approved one to reconcile. trust_gate_detail is the JSON of the refused pairs.
// Recomputed deterministically each run (self-heals on resume), NOT wire-load-bearing.
`
ALTER TABLE retrieval_state ADD COLUMN n_trust_gated_suppress INTEGER NOT NULL DEFAULT 0;
ALTER TABLE retrieval_state ADD COLUMN trust_gate_detail TEXT NOT NULL DEFAULT '';
`,
}
// DESIGN NOTE (D21.10 — reserved memory-bank v2 record types; Ф2, NOT a migration, NOT code).