Land track-A pack-1: trust-gated glossary suppressor with loud telemetry, export-contract normalization, pair-keyed prompt seam, role-injection registry, target-gated readability, quality report
This commit is contained in:
parent
ce3e17faa0
commit
d3f6b34ad4
23 changed files with 1409 additions and 361 deletions
|
|
@ -102,11 +102,20 @@ func report(cfgPath string) error {
|
|||
|
||||
// Чтения — коллбеками (interleaved с печатью, как исторически): провал чтения
|
||||
// посреди аудита оставляет уже напечатанную часть на stdout (см. renderReport).
|
||||
return renderReport(os.Stdout,
|
||||
if err := renderReport(os.Stdout,
|
||||
func() ([]store.RequestLogView, error) { return r.Store.RequestLogRows(r.Book.BookID) },
|
||||
func() ([]store.ChunkStatus, error) { return r.Store.ChunkStatusesForBook(r.Book.BookID) },
|
||||
func() ([]store.RetrievalState, error) { return r.Store.RetrievalStatesForBook(r.Book.BookID) },
|
||||
func() (float64, float64, error) { return r.Store.SpentUSD(r.Book.BookID) })
|
||||
func() (float64, float64, error) { return r.Store.SpentUSD(r.Book.BookID) }); err != nil {
|
||||
return err
|
||||
}
|
||||
// Per-run quality-report (D39 слой 5): aggregate the deterministic quality signals into a readable
|
||||
// summary — the "telemetry from day one" the owner asked for (H5). Read-only ($0), never a gate.
|
||||
q, err := r.QualityReport()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return renderQuality(os.Stdout, q)
|
||||
}
|
||||
|
||||
// status prints the READ-ONLY progress projection (D15.3): 0 LLM, 0 replay. --json emits the
|
||||
|
|
|
|||
|
|
@ -210,6 +210,24 @@ func errTail(degraded, errText string) string {
|
|||
return s
|
||||
}
|
||||
|
||||
// renderQuality prints the deterministic per-run quality-report (D39 слой 5): the claim-1 structural
|
||||
// KPI (mean sentences per narrative paragraph) plus the aggregated deterministic signals
|
||||
// (dialogue-dash, glossary consistency, number drift, CJK-leak / echo rates, trust-gated). Pure
|
||||
// observability — a summary section, never an exit code. Kept SEPARATE from renderReport so that
|
||||
// function's frozen callback contract (render_test.go) is untouched.
|
||||
func renderQuality(w io.Writer, q *pipeline.QualityReport) error {
|
||||
fmt.Fprintf(w, "\n=== КАЧЕСТВО (per-run, детерминированные сигналы — наблюдаемость, не гейт) ===\n")
|
||||
fmt.Fprintf(w, "чанков всего=%d · дошло до финала=%d · с экспорт-текстом=%d\n", q.TotalChunks, q.ProcessedChunks, q.TextChunks)
|
||||
// Claim-1: рубленые абзацы. ≈1.0 предл./абзац = рублено (претензия владельца); выше = слитая проза.
|
||||
fmt.Fprintf(w, "СТРУКТУРА (претензия-1 «рубленые абзацы»): предл/нарратив-абзац=%.2f (предложений=%d / нарратив-абзацев=%d)\n",
|
||||
q.MeanSentPerNarrPara, q.NarrativeSentences, q.NarrativeParagraphs)
|
||||
fmt.Fprintf(w, "СИГНАЛЫ: диалог-тире=%d · глоссарий-промахов=%d · число-дрейф=%d · trust-gated(сид)=%d\n",
|
||||
q.DialogueDashFlags, q.GlossaryMisses, q.NumberDriftFlags, q.TrustGated)
|
||||
fmt.Fprintf(w, "УТЕЧКИ: CJK-strip чанков=%d (%.1f%%) · echo(cjk_artifact) чанков=%d (%.1f%%)\n",
|
||||
q.CJKLeakChunks, 100*q.CJKLeakRate, q.EchoChunks, 100*q.EchoRate)
|
||||
return nil
|
||||
}
|
||||
|
||||
// renderStatusJSON emits the StatusReport as indented JSON (stable enums — the
|
||||
// ratified CI/IDE contract, D12/D15.3) and maps flagged>0 to the exit-2 sentinel.
|
||||
func renderStatusJSON(w io.Writer, rep *pipeline.StatusReport) error {
|
||||
|
|
|
|||
|
|
@ -35,7 +35,12 @@ stages:
|
|||
- name: draft
|
||||
role: translator
|
||||
model: deepseek-v4-flash # интерим — переводчик по exp13 (draft качеством не мерен, D30.6); флип при развязке бейк-оффа
|
||||
prompt: ../prompts/translator.md
|
||||
# Слой 2 (D39): промпт резолвится из pair-keyed пакета конвенций по паре книги (Book.LangPair()),
|
||||
# НЕ из фикс-пути. Наполнена только zh→ru (сегодня); ja/en книга без своей пары — FAIL-LOUD в
|
||||
# loadTemplates (закрывает латентный баг «ja едет через китайский паратаксис»). Язык промпта —
|
||||
# свойство пакета. Прод zh→ru резолвит тот же translator.md → тот же SHA (поведение-нейтрально).
|
||||
prompts:
|
||||
zh-ru: ../prompts/translator.md
|
||||
# D30.2 reflow: из translator.md убрано «Сохраняй разбивку на абзацы» (построчная
|
||||
# вёрстка исходника деструктивна для ru — корень нечитаемости exp12 §2.5-S). Bump
|
||||
# v0-draft→v1-reflow: сменил SHA промпта, осознанный --resnapshot.
|
||||
|
|
@ -54,7 +59,9 @@ stages:
|
|||
# верность 5.0, 0 утечек преамбул; grok reasoning-off из редакторских ролей СНЯТ — no-op).
|
||||
# gemini-3.1-pro — премиум-эскалация редактора ТОЛЬКО за санитайзером (утечка преамбул 6/6).
|
||||
model: glm-5
|
||||
prompt: ../prompts/editor.md
|
||||
# Слой 2 (D39): pair-keyed пакет конвенций (см. draft-стадию) — наполнена zh→ru.
|
||||
prompts:
|
||||
zh-ru: ../prompts/editor.md
|
||||
# Bump v2-bilingual-reflow→v3-discourse-reflow: в editor.md вшито валидированное P1a-ядро
|
||||
# ДИСКУРС-ПЕРЕВЁРСТКИ + few-shot (exp14/D37 §2а — претензия-1 «рубленые абзацы» = рычаг ПРОМПТА,
|
||||
# не модель; P1a реверстает дефект у всех семей). Меняет SHA промпта → осознанный --resnapshot.
|
||||
|
|
|
|||
|
|
@ -23,7 +23,8 @@ stages:
|
|||
- name: draft
|
||||
role: translator
|
||||
model: deepseek-v4-flash
|
||||
prompt: ../prompts/translator.md
|
||||
prompts:
|
||||
zh-ru: ../prompts/translator.md
|
||||
prompt_version: v1-reflow # D30.2 reflow (тот же translator.md, что в C1) — label-SHA дисциплина
|
||||
temperature: 0.8 # C2: кандидаты сэмплируются горячее (Р2: T=0.6–1.0)
|
||||
reasoning: "off"
|
||||
|
|
@ -34,7 +35,8 @@ stages:
|
|||
- name: select
|
||||
role: judge
|
||||
model: glm-5
|
||||
prompt: ../prompts/judge-selector.md
|
||||
prompts:
|
||||
zh-ru: ../prompts/judge-selector.md
|
||||
prompt_version: v0-skeleton
|
||||
temperature: 0
|
||||
reasoning: "off"
|
||||
|
|
@ -45,7 +47,8 @@ stages:
|
|||
# по D30.1 — glm-5 билингв (grok reasoning-off из редакторов СНЯТ — no-op). (Стадию select
|
||||
# выше — judge/glm-5, Gemini-слот Ф2 — НЕ трогаем.)
|
||||
model: glm-5
|
||||
prompt: ../prompts/editor.md
|
||||
prompts:
|
||||
zh-ru: ../prompts/editor.md
|
||||
# Бамп v2-bilingual-reflow→v3-discourse-reflow (label-SHA дисциплина: тот же editor.md, что в C1 —
|
||||
# вшито P1a-ядро ДИСКУРС-ПЕРЕВЁРСТКИ, D37 §2а; лейбл обязан следовать за новым SHA файла).
|
||||
prompt_version: v3-discourse-reflow
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
|
@ -60,7 +61,17 @@ type Stage struct {
|
|||
Name string `yaml:"name"`
|
||||
Role string `yaml:"role"`
|
||||
Model string `yaml:"model"`
|
||||
Prompt string `yaml:"prompt"` // path to the versioned template file
|
||||
// Prompt is the LEGACY single, pair-AGNOSTIC template path (kept for tests / simple configs).
|
||||
// Prompts is the pair-KEYED convention pack (D39 слой 2): src→tgt → template path, resolved by the
|
||||
// book's LangPair() at load with a FAIL-LOUD on a missing pair (never a silent fall-through to
|
||||
// another pair's conventions — the latent bug where a ja book rode the zh-parataxis prompt,
|
||||
// L4-prompts-not-per-pair-zh-baked). The pack is where a per-pair prompt (and its LANGUAGE — a
|
||||
// property of the pack, not a hardcode) lives; today only zh-ru is populated. Exactly ONE of
|
||||
// Prompt/Prompts per stage (validated in LoadPipeline). The pair→prompt binding is snapshot-folded
|
||||
// transitively: the resolved template's content rides PromptSHA256 and the book's pair rides
|
||||
// BriefHash (source_lang/target_lang), so a pair or content change is a loud --resnapshot.
|
||||
Prompt string `yaml:"prompt"`
|
||||
Prompts map[string]string `yaml:"prompts"`
|
||||
PromptVersion string `yaml:"prompt_version"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
Reasoning string `yaml:"reasoning"` // "", off, low, medium, high
|
||||
|
|
@ -161,10 +172,25 @@ type Fanout struct {
|
|||
Candidates int `yaml:"candidates"`
|
||||
}
|
||||
|
||||
// gatesEnabled reports whether any QA-gate is on. Эскалация запускается только
|
||||
// по провалу гейта, поэтому это сигнал «эскалация вообще достижима» (Фаза 1+).
|
||||
func (p *Pipeline) gatesEnabled() bool {
|
||||
return p.Gates.Coverage.Enabled
|
||||
// PromptPathFor resolves the effective prompt template path for this stage given the book's language
|
||||
// pair (D39 слой 2). A pair-keyed `prompts` pack resolves the entry for `pair`, failing LOUD if the
|
||||
// pair is absent (never a silent fall-through to another pair's conventions — the latent bug where a
|
||||
// ja book rode the zh-parataxis prompt). A legacy single `prompt` is pair-agnostic (returned as-is).
|
||||
// Callers pass Book.LangPair(). Paths are already resolved to absolute in LoadPipeline.
|
||||
func (st Stage) PromptPathFor(pair string) (string, error) {
|
||||
if len(st.Prompts) == 0 {
|
||||
return st.Prompt, nil
|
||||
}
|
||||
if p, ok := st.Prompts[pair]; ok {
|
||||
return p, nil
|
||||
}
|
||||
have := make([]string, 0, len(st.Prompts))
|
||||
for k := range st.Prompts {
|
||||
have = append(have, k)
|
||||
}
|
||||
sort.Strings(have)
|
||||
return "", fmt.Errorf("stage %q: no prompt for language pair %q — the pair-keyed pack has [%s]; add a %q prompt (conventions authored in the right language) or fix the book's source_lang/target_lang (D39 слой 2: никогда молча не подставляем чужую пару)",
|
||||
st.Name, pair, strings.Join(have, ", "), pair)
|
||||
}
|
||||
|
||||
// CheckRunnable rejects a config whose mechanics the Phase-0 runner does NOT
|
||||
|
|
@ -228,9 +254,16 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) {
|
|||
// Prompt paths resolve relative to the pipeline file's directory, so a
|
||||
// project works from any CWD.
|
||||
dir := filepath.Dir(path)
|
||||
resolvePrompt := func(pr string) string {
|
||||
if pr != "" && !filepath.IsAbs(pr) {
|
||||
return filepath.Join(dir, pr)
|
||||
}
|
||||
return pr
|
||||
}
|
||||
for i := range p.Stages {
|
||||
if pr := p.Stages[i].Prompt; pr != "" && !filepath.IsAbs(pr) {
|
||||
p.Stages[i].Prompt = filepath.Join(dir, pr)
|
||||
p.Stages[i].Prompt = resolvePrompt(p.Stages[i].Prompt)
|
||||
for pair, pr := range p.Stages[i].Prompts {
|
||||
p.Stages[i].Prompts[pair] = resolvePrompt(pr)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -275,11 +308,31 @@ func LoadPipeline(path string, models *Models) (*Pipeline, error) {
|
|||
if _, ok := models.Models[st.Model]; !ok {
|
||||
bad("stage %q: model %q is not defined in models.yaml", st.Name, st.Model)
|
||||
}
|
||||
if st.Prompt == "" {
|
||||
bad("stage %q: prompt template path is required", st.Name)
|
||||
} else if _, err := os.Stat(st.Prompt); err != nil {
|
||||
// Prompt (legacy single, pair-agnostic) XOR Prompts (pair-keyed pack, D39 слой 2) — exactly one.
|
||||
switch {
|
||||
case st.Prompt != "" && len(st.Prompts) > 0:
|
||||
bad("stage %q: set either `prompt` (single) OR `prompts` (pair-keyed pack), not both", st.Name)
|
||||
case st.Prompt == "" && len(st.Prompts) == 0:
|
||||
bad("stage %q: a prompt is required — `prompt` (single) or a pair-keyed `prompts` pack (D39 слой 2)", st.Name)
|
||||
case st.Prompt != "":
|
||||
if _, err := os.Stat(st.Prompt); err != nil {
|
||||
bad("stage %q: prompt template %s is not readable: %v", st.Name, st.Prompt, err)
|
||||
}
|
||||
default: // pair-keyed pack: every declared pair's template must be readable (deterministic order)
|
||||
pairs := make([]string, 0, len(st.Prompts))
|
||||
for pair := range st.Prompts {
|
||||
pairs = append(pairs, pair)
|
||||
}
|
||||
sort.Strings(pairs)
|
||||
for _, pair := range pairs {
|
||||
pr := st.Prompts[pair]
|
||||
if pr == "" {
|
||||
bad("stage %q: prompt pack pair %q has an empty path", st.Name, pair)
|
||||
} else if _, err := os.Stat(pr); err != nil {
|
||||
bad("stage %q: prompt pack %q → %s is not readable: %v", st.Name, pair, pr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch st.Reasoning {
|
||||
case "", "off", "low", "medium", "high":
|
||||
default:
|
||||
|
|
|
|||
140
backend/internal/config/prompt_pack_test.go
Normal file
140
backend/internal/config/prompt_pack_test.go
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// writeTmp writes a file under a temp path and returns it.
|
||||
func writeTmp(t *testing.T, path, body string) string {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// miniModels loads a minimal valid models.yaml with a single `fake` model.
|
||||
func miniModels(t *testing.T) *Models {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
body := "prices_checked: " + time.Now().UTC().Format("2006-01-02") + `
|
||||
default_model: fake
|
||||
providers:
|
||||
p: { kind: openai, base_url: http://x }
|
||||
models:
|
||||
fake: { provider: p, price: { input_per_m: 1, output_per_m: 2 } }
|
||||
`
|
||||
m, err := LoadModels(writeTmp(t, filepath.Join(dir, "models.yaml"), body))
|
||||
if err != nil {
|
||||
t.Fatalf("miniModels: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// prompt_pack_test.go pins the pair-keyed prompt seam (D39 слой 2, L4-prompts-not-per-pair-zh-baked):
|
||||
// a stage resolves its prompt from a pair-keyed pack by the book's LangPair(), FAIL-LOUD on a missing
|
||||
// pair; a legacy single `prompt` stays pair-agnostic; prod zh→ru resolves the same file (SHA-neutral).
|
||||
|
||||
func TestPromptPathForResolvesPair(t *testing.T) {
|
||||
st := Stage{Name: "edit", Prompts: map[string]string{"zh-ru": "/x/editor.md", "ja-ru": "/y/editor.md"}}
|
||||
got, err := st.PromptPathFor("zh-ru")
|
||||
if err != nil || got != "/x/editor.md" {
|
||||
t.Fatalf("PromptPathFor(zh-ru) = %q, %v; want /x/editor.md, nil", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptPathForFailsLoudOnMissingPair(t *testing.T) {
|
||||
st := Stage{Name: "edit", Prompts: map[string]string{"zh-ru": "/x/editor.md"}}
|
||||
_, err := st.PromptPathFor("ja-ru")
|
||||
if err == nil {
|
||||
t.Fatal("PromptPathFor(ja-ru) must FAIL LOUD when the pack has no ja-ru (the latent ja-runs-zh-prompt bug)")
|
||||
}
|
||||
// The error must name the missing pair AND list what the pack has (operator-actionable).
|
||||
if !strings.Contains(err.Error(), "ja-ru") || !strings.Contains(err.Error(), "zh-ru") {
|
||||
t.Errorf("fail-loud error must name the missing pair and the pack's pairs, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptPathForLegacySingleIsPairAgnostic(t *testing.T) {
|
||||
st := Stage{Name: "draft", Prompt: "/x/translator.md"}
|
||||
for _, pair := range []string{"zh-ru", "ja-ru", "en-ru"} {
|
||||
got, err := st.PromptPathFor(pair)
|
||||
if err != nil || got != "/x/translator.md" {
|
||||
t.Errorf("legacy single prompt must resolve for %q: got %q, %v", pair, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineRejectsAmbiguousPromptForms(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeTmp(t, filepath.Join(dir, "translator.md"), "sys\n---USER---\n{{text}}")
|
||||
models := miniModels(t)
|
||||
|
||||
// Both `prompt` and `prompts` set → rejected.
|
||||
both := writeTmp(t, filepath.Join(dir, "both.yaml"), `
|
||||
core: C1
|
||||
stages:
|
||||
- name: draft
|
||||
role: translator
|
||||
model: fake
|
||||
prompt: translator.md
|
||||
prompts: { zh-ru: translator.md }
|
||||
prompt_version: v1
|
||||
`)
|
||||
if _, err := LoadPipeline(both, models); err == nil || !strings.Contains(err.Error(), "not both") {
|
||||
t.Errorf("both prompt+prompts must be rejected, got: %v", err)
|
||||
}
|
||||
|
||||
// Neither set → rejected.
|
||||
neither := writeTmp(t, filepath.Join(dir, "neither.yaml"), `
|
||||
core: C1
|
||||
stages:
|
||||
- name: draft
|
||||
role: translator
|
||||
model: fake
|
||||
prompt_version: v1
|
||||
`)
|
||||
if _, err := LoadPipeline(neither, models); err == nil || !strings.Contains(err.Error(), "prompt is required") {
|
||||
t.Errorf("neither prompt nor prompts must be rejected, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBoevoyPipelineC1ResolvesZhRu proves the migrated prod config is behaviour-neutral for zh→ru:
|
||||
// the draft/edit stages resolve the same translator.md/editor.md the legacy path did (same file →
|
||||
// same PromptSHA256), and a NON-zh pair fails loud.
|
||||
func TestBoevoyPipelineC1ResolvesZhRu(t *testing.T) {
|
||||
models, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
|
||||
if err != nil {
|
||||
t.Fatalf("load models: %v", err)
|
||||
}
|
||||
p, err := LoadPipeline(filepath.Join("..", "..", "configs", "pipeline-c1.yaml"), models)
|
||||
if err != nil {
|
||||
t.Fatalf("load pipeline-c1: %v", err)
|
||||
}
|
||||
for _, st := range p.Stages {
|
||||
zh, err := st.PromptPathFor("zh-ru")
|
||||
if err != nil {
|
||||
t.Errorf("stage %q must resolve zh-ru: %v", st.Name, err)
|
||||
}
|
||||
if !strings.HasSuffix(zh, ".md") {
|
||||
t.Errorf("stage %q zh-ru prompt should be a .md template, got %q", st.Name, zh)
|
||||
}
|
||||
// A book of a pair the pack does not carry must fail loud (closes the ja-runs-zh-prompt bug).
|
||||
if _, err := st.PromptPathFor("ja-ru"); err == nil {
|
||||
t.Errorf("stage %q must fail loud for ja-ru (pack is zh-ru only)", st.Name)
|
||||
}
|
||||
}
|
||||
// The draft stage must resolve to the shared translator.md (the zh→ru pack content is unchanged).
|
||||
for _, st := range p.Stages {
|
||||
if st.Role == "translator" {
|
||||
zh, _ := st.PromptPathFor("zh-ru")
|
||||
if !strings.HasSuffix(zh, filepath.Join("prompts", "translator.md")) {
|
||||
t.Errorf("draft zh-ru must be prompts/translator.md, got %q", zh)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -24,38 +24,45 @@ import (
|
|||
// re-pins loudly). The coverage gate runs ONLY on the TRANSLATOR role's output (vs the
|
||||
// original source): a monolingual editor legitimately restructures sentences, so gating
|
||||
// it against the source would false-flag a correct edit (see the role check below).
|
||||
func (r *Runner) classifyOutput(role, source, output, finish string, isFinal bool) classification {
|
||||
// classifyOutput returns the disposition AND, for a cosmetic sanitizer strip, the resolved EXPORT
|
||||
// text (the strip remainder) computed ONCE here (T3.4 / L8-stripcosmetic-recomputed-twice): the
|
||||
// second return travels back to runStage as the authoritative export, so no site re-derives
|
||||
// stripCosmetic and the "non-empty & clean" invariant is asserted over the very bytes that ship.
|
||||
// The string is "" for every disposition except FlagSanitizerStripped.
|
||||
func (r *Runner) classifyOutput(role, source, output, finish string, isFinal bool) (classification, string) {
|
||||
cls := classify(classifyInput{Source: source, Output: output, Finish: finish, TargetLang: r.Book.TargetLang})
|
||||
if !cls.ok() {
|
||||
return cls
|
||||
return cls, ""
|
||||
}
|
||||
// Output-sanitizer gate (D30.3): a deterministic verdict-axis check for "instant
|
||||
// unreadability" defects no other gate catches — a leaked service preamble, a trailing
|
||||
// note/edit block, a markdown ### header, a Latin-script insertion, a broken word form.
|
||||
// Opt-in (Gates.Sanitizer). Runs ONLY on the FINAL stage's output (isFinal) — the text
|
||||
// that SHIPS: an intermediate translator draft is legitimately rough and the EDITOR's job
|
||||
// is to clean it, so sanitizing the draft would wrongly skip the recovering editor and drop
|
||||
// the chunk to a placeholder (адверсариальное ревью). Runs on the completion text like the
|
||||
// intrinsic classifier, so a resumed checkpoint re-derives the identical verdict for free;
|
||||
// its rule version folds into the snapshot only when enabled (sanitizerSnapshot).
|
||||
if isFinal && r.Pipeline.Gates.Sanitizer.Enabled {
|
||||
// Output-sanitizer gate (D30.3): a deterministic verdict-axis check for "instant unreadability"
|
||||
// defects no other gate catches — a leaked service preamble, a trailing note/edit block, a markdown
|
||||
// ### header, a Latin-script insertion, a contentless CJK-leak run. Opt-in (Gates.Sanitizer). Runs
|
||||
// ONLY on the FINAL stage's output (isFinal) — the text that SHIPS: an intermediate translator draft
|
||||
// is legitimately rough and the EDITOR's job is to clean it, so sanitizing the draft would wrongly
|
||||
// skip the recovering editor and drop the chunk to a placeholder (адверсариальное ревью). It is a
|
||||
// READABILITY gate whose patterns bake in Cyrillic/Russian conventions, so it is gated behind a
|
||||
// Russian target (D39 слой 7, L8-readability-gates-target-blind) — a no-op for any other pair, which
|
||||
// would false-flag (prod zh→ru is unaffected). Runs on the completion text like the intrinsic
|
||||
// classifier, so a resumed checkpoint re-derives the identical verdict; its rule version folds into
|
||||
// the snapshot only when enabled (sanitizerSnapshot).
|
||||
if isFinal && r.Pipeline.Gates.Sanitizer.Enabled && isRuTarget(r.Book.TargetLang) {
|
||||
if san := sanitizeOutput(output); san.total() > 0 {
|
||||
// Cosmetic-only leak (leading markdown header and/or CJK-leak) → strip it and export
|
||||
// the remainder flagged, not drop the whole chunk (D35.4a). The guard is load-bearing:
|
||||
// we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that is now
|
||||
// clean, so runStage/resume can trust that the derived export text is usable; anything
|
||||
// else falls through to the substantive skip. Both are pure over `output`, so a resumed
|
||||
// checkpoint re-derives the identical verdict.
|
||||
// Cosmetic-only leak (leading markdown header and/or contentless CJK-leak) → strip it and
|
||||
// export the remainder flagged, not drop the whole chunk (D35.4a). The guard is
|
||||
// load-bearing: we tag sanitizer_stripped ONLY when the strip yields a NON-EMPTY output that
|
||||
// is now clean, and we RETURN that stripped text so runStage/resume ship the exact bytes the
|
||||
// guard validated. Anything else falls through to the substantive skip. Pure over `output`,
|
||||
// so a resumed checkpoint re-derives the identical verdict.
|
||||
if san.cosmeticOnly() {
|
||||
if stripped := stripCosmetic(output); strings.TrimSpace(stripped) != "" && sanitizeOutput(stripped).total() == 0 {
|
||||
return classification{FlagSanitizerStripped, san.summary()}
|
||||
return classification{FlagSanitizerStripped, san.summary()}, stripped
|
||||
}
|
||||
}
|
||||
return classification{FlagSanitizerDefect, san.summary()}
|
||||
return classification{FlagSanitizerDefect, san.summary()}, ""
|
||||
}
|
||||
}
|
||||
if !r.Pipeline.Gates.Coverage.Enabled {
|
||||
return cls
|
||||
return cls, ""
|
||||
}
|
||||
// The coverage gate compares the output against the ORIGINAL source, which is a
|
||||
// source→target translation only for the TRANSLATOR role. A monolingual editor
|
||||
|
|
@ -64,12 +71,12 @@ func (r *Runner) classifyOutput(role, source, output, finish string, isFinal boo
|
|||
// (self-review finding). Editor/other-role fidelity is a Phase-2 concern (a
|
||||
// bilingual judge over the draft, §04 mode 2), not this excision gate.
|
||||
if role != roleTranslator {
|
||||
return cls
|
||||
return cls, ""
|
||||
}
|
||||
if cov := coverageCheck(r.Pipeline.Gates.Coverage, source, output, r.Book.SourceLang, r.Book.TargetLang); !cov.cls.ok() {
|
||||
return cov.cls
|
||||
return cov.cls, ""
|
||||
}
|
||||
return cls
|
||||
return cls, ""
|
||||
}
|
||||
|
||||
// translateChunk drives ONE chunk through the stage list. Stages run in order,
|
||||
|
|
@ -94,17 +101,26 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
|||
// regression guard's draft→final comparison (== final in a single-stage pipeline).
|
||||
draftText := ""
|
||||
|
||||
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and
|
||||
// serialize the translator injection block. Recomputed on every run (resumed chunks
|
||||
// included) so the sticky window and the injected bytes are resume-stable.
|
||||
// Hot path: select the glossary records for this chunk ONCE (deterministic, $0), and serialize the
|
||||
// per-role injection blocks via the role→renderer registry (D39 слой 7). Recomputed on every run
|
||||
// (resumed chunks included) so the sticky window and the injected bytes are resume-stable.
|
||||
var memSel memorySelection
|
||||
var translatorInjection, editorInjection string
|
||||
injectionByRole := map[string]string{}
|
||||
activeIDs := map[string]injectionDisposition{}
|
||||
if r.memory != nil {
|
||||
memSel = r.memory.Select(ch.Text, ch.Chapter, stickyPrev, r.Pipeline.Context.GlossaryTokenBudget)
|
||||
translatorInjection = renderGlossaryBlock(memSel.injected)
|
||||
editorInjection = renderEditorConstraintBlock(memSel.injected)
|
||||
for role, render := range roleInjectionRenderers {
|
||||
injectionByRole[role] = render(memSel.injected) // pure → per-role result is map-order-independent
|
||||
}
|
||||
activeIDs = memSel.activeIDs
|
||||
// The disposition-gated suppressor refused to let a lower-trust longer key eat a higher-trust
|
||||
// nested one (D39 слой 4): the approved term survives, but surface the seed-hygiene collision
|
||||
// loudly so an operator can reconcile the draft-nesting-over-approved seed (research/13 §7).
|
||||
if len(memSel.trustGated) > 0 {
|
||||
r.Log.WarnContext(ctx, "memory: lower-trust longer key refused from suppressing a higher-trust nested key (approved term preserved; reconcile the seed)",
|
||||
"chapter", ch.Chapter, "chunk", ch.ChunkIdx, "trust_gated", len(memSel.trustGated),
|
||||
"first", memSel.trustGated[0].Suppressor+"⊃"+memSel.trustGated[0].Protected)
|
||||
}
|
||||
}
|
||||
|
||||
for stageIdx, st := range r.Pipeline.Stages {
|
||||
|
|
@ -125,19 +141,12 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
|||
continue
|
||||
}
|
||||
|
||||
// Inject the glossary per role. The TRANSLATOR gets the src→dst block (its keys are
|
||||
// matched against the source chunk); the monolingual EDITOR gets the CONFIRMED dst
|
||||
// forms as target-consistency constraints (no source — decisions-log D1); every other
|
||||
// stage gets none. Empty injection → the plain 2-message layout. The injection is a
|
||||
// message, so it enters this stage's request_hash automatically (a resumed chunk
|
||||
// reproduces it deterministically from the frozen bank).
|
||||
injection := ""
|
||||
switch st.Role {
|
||||
case roleTranslator:
|
||||
injection = translatorInjection
|
||||
case roleEditor:
|
||||
injection = editorInjection
|
||||
}
|
||||
// Resolve the per-role memory injection from the registry (D39 слой 7): the TRANSLATOR gets the
|
||||
// src→dst glossary block (keys matched against the source chunk); the BILINGUAL editor (D30.1)
|
||||
// gets the CONFIRMED dst forms as target-consistency constraints; every other role gets none
|
||||
// (empty string → the plain 2-message layout). The injection is a message, so it enters this
|
||||
// stage's request_hash automatically (a resumed chunk reproduces it deterministically).
|
||||
injection := injectionByRole[st.Role]
|
||||
sr, err := r.runStage(ctx, st, stageIdx, snapID, ch, prev, injection)
|
||||
if err != nil {
|
||||
return out, activeIDs, err
|
||||
|
|
@ -182,9 +191,12 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
|||
// Cheap deterministic style/number flaggers on the FINAL text (cheapgates.go): observability
|
||||
// only, never a disposition. Runs on the same fresh-or-resumed output as the post-check, so it
|
||||
// is resume-reproducible; skipped when a stage flagged (no usable output). Source is ch.Text
|
||||
// (the 万/億 magnitude gate compares source↔output).
|
||||
// (the 万/億 magnitude gate compares source↔output). These are READABILITY flaggers whose rules
|
||||
// bake in Russian conventions (em-dash dialogue, ёфикатор, ru magnitude words), so they are gated
|
||||
// behind a Russian target (D39 слой 7, L8-readability-gates-target-blind) — a no-op for any other
|
||||
// pair, which would false-flag. Prod zh→ru is unaffected (target=ru).
|
||||
var cheap cheapGateResult
|
||||
if !flagged && prev != "" {
|
||||
if !flagged && prev != "" && isRuTarget(r.Book.TargetLang) {
|
||||
cheap = runCheapGates(ch.Text, draftText, prev, r.cheapGateConfig())
|
||||
if cheap.total() > 0 {
|
||||
r.Log.InfoContext(ctx, "cheap style gates flagged the chunk (observability, not a gate)",
|
||||
|
|
@ -204,19 +216,35 @@ func (r *Runner) translateChunk(ctx context.Context, snapID string, ch Chunk, st
|
|||
}
|
||||
}
|
||||
|
||||
// The export contract (D39 слой 6): NORMALISE every final text before it ships — clean OR
|
||||
// flagged-stripped — so a cosmetic Unicode artifact (fullwidth glyph, U+3000 indent, stray CJK
|
||||
// punctuation, a leaked combining stress mark) is fixed uniformly, not only on a flagged chunk
|
||||
// (L5-normalization-fused-to-flag-path). A cosmetic sanitizer strip exports its cleaned remainder
|
||||
// (recovered != ""); every other flag exports "" (the contaminated output never reaches TM/export,
|
||||
// D2 / D35.4a), and exportNormalize("") == "". Pure ephemeral projection — no wire/checkpoint touch.
|
||||
if flagged {
|
||||
out.Disposition = DispFlagged
|
||||
out.FlagReason = flagReason
|
||||
// A cosmetic sanitizer strip exports its cleaned remainder (recovered != ""); every other
|
||||
// flag (garbage/refusal/glossary-miss/substantive-sanitizer) exports "" — the contaminated
|
||||
// output never reaches TM/export (D2 / D35.4a).
|
||||
out.FinalText = recovered
|
||||
out.FinalText = exportNormalize(recovered)
|
||||
} else {
|
||||
out.FinalText = prev
|
||||
out.FinalText = exportNormalize(prev)
|
||||
}
|
||||
return out, activeIDs, nil
|
||||
}
|
||||
|
||||
// injectionRenderer serializes a chunk's selected memory records into a role's injection message.
|
||||
type injectionRenderer func(injected []pickedEntry) string
|
||||
|
||||
// roleInjectionRenderers maps a stage ROLE to the memory-injection renderer it consumes — a registry
|
||||
// instead of a hand-edited switch (D39 слой 7, L8-role-injection-hardcoded-switch), so a new Ф2
|
||||
// consumer role (annotator, voice-layer — README) is DATA + one renderer, not surgery on a switch.
|
||||
// A role absent from the map gets no injection. Renderers are pure, so the per-role result is
|
||||
// deterministic regardless of map iteration order (no map-order in OUTPUT — invariant #6).
|
||||
var roleInjectionRenderers = map[string]injectionRenderer{
|
||||
roleTranslator: renderGlossaryBlock, // src→dst glossary block
|
||||
roleEditor: renderEditorConstraintBlock, // CONFIRMED dst forms (target-consistency constraints)
|
||||
}
|
||||
|
||||
// persistRetrievalState writes the per-chunk observability record from the deterministic
|
||||
// selection + the post-check result. n_exact_hits/n_sticky/n_ambiguous count the INJECTED
|
||||
// records (what the model saw); spoiler/eviction are the dropped-and-logged totals;
|
||||
|
|
@ -243,6 +271,15 @@ func (r *Runner) persistRetrievalState(snapID string, ch Chunk, sel memorySelect
|
|||
}
|
||||
rs.NSpoilerBlocked = len(sel.rejected)
|
||||
rs.NEvicted = len(sel.evicted)
|
||||
// Loud record of the disposition-gated suppressor (D39 слой 4): a longer lower-trust key refused
|
||||
// from eating a nested higher-trust one — the term-drift code root, now visible instead of a
|
||||
// silent drop (research/13 §7). Detail carries the refused suppressor→protected pairs for a human.
|
||||
rs.NTrustGatedSuppress = len(sel.trustGated)
|
||||
if len(sel.trustGated) > 0 {
|
||||
if b, err := json.Marshal(sel.trustGated); err == nil {
|
||||
rs.TrustGateDetail = string(b)
|
||||
}
|
||||
}
|
||||
if outputChecked {
|
||||
// n_postcheck_miss is the CONFIRMED-miss count (the actionable consistency-failure
|
||||
// signal, external-review #1); the detail carries ALL misses (confirmed AND the
|
||||
|
|
|
|||
|
|
@ -299,6 +299,15 @@ func isCJKTarget(lang string) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// isRuTarget reports whether the translation TARGET is Russian — the guard for the READABILITY gate
|
||||
// set (output-sanitizer + cheap style flaggers), whose rules bake in Cyrillic/Russian conventions
|
||||
// (D39 слой 7, L8-readability-gates-target-blind). They are a no-op for any other target, which would
|
||||
// otherwise false-flag (a →en final reads as a "Latin insertion"). Mirrors the pair-aware
|
||||
// coverage/classify seam. Prod zh→ru is unaffected (target=ru).
|
||||
func isRuTarget(lang string) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(lang), "ru")
|
||||
}
|
||||
|
||||
// --- degeneration detector (n-gram loop) — runs BEFORE the length retry (D2.3) ---
|
||||
|
||||
// degenerateLoop reports whether the text is a repetition loop: the word-level
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ func TestFewShotSplitAndAssembly(t *testing.T) {
|
|||
}
|
||||
|
||||
// TestFewShotAbsentIsNoop: a template with no ---FEWSHOT--- block leaves FewShot empty and the
|
||||
// toggle a no-op, so every existing prompt (translator, analyst, golden stub) is untouched.
|
||||
// toggle a no-op, so every existing prompt (translator, golden stub) is untouched.
|
||||
func TestFewShotAbsentIsNoop(t *testing.T) {
|
||||
tpl := writeTemplate(t, "Только система.\n---USER---\n{{text}}")
|
||||
if tpl.FewShot != "" {
|
||||
|
|
|
|||
|
|
@ -245,10 +245,10 @@ func captureGolden(t *testing.T, label string, r *Runner, res *BookResult, wireB
|
|||
return rss[i].ChunkIdx < rss[j].ChunkIdx
|
||||
})
|
||||
for _, rs := range rss {
|
||||
w("ch%d/%d snap_match=%t exact=%d sticky=%d ambiguous=%d spoiler=%d evicted=%d postcheck_miss=%d style_flags=%d",
|
||||
w("ch%d/%d snap_match=%t exact=%d sticky=%d ambiguous=%d spoiler=%d evicted=%d postcheck_miss=%d style_flags=%d trust_gated=%d",
|
||||
rs.Chapter, rs.ChunkIdx, rs.SnapshotID == snapID, rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged,
|
||||
rs.NSpoilerBlocked, rs.NEvicted, rs.NPostcheckMiss, rs.NStyleFlags)
|
||||
w(" injected_ids=%s postcheck_detail=%s style_detail=%s", rs.InjectedIDs, rs.PostcheckDetail, rs.StyleDetail)
|
||||
rs.NSpoilerBlocked, rs.NEvicted, rs.NPostcheckMiss, rs.NStyleFlags, rs.NTrustGatedSuppress)
|
||||
w(" injected_ids=%s postcheck_detail=%s style_detail=%s trust_gate_detail=%s", rs.InjectedIDs, rs.PostcheckDetail, rs.StyleDetail, rs.TrustGateDetail)
|
||||
}
|
||||
|
||||
w("-- wire bodies (call order) --")
|
||||
|
|
|
|||
|
|
@ -38,7 +38,13 @@ import (
|
|||
// recomputing it status-based (a collision-prone AMBIGUOUS carry no longer silently
|
||||
// upgrades to CONFIRMED), and a spaced-phonetic (Latin/Cyrillic) source key is validated
|
||||
// against a word boundary (approved "rose" no longer fires inside "roseanne").
|
||||
const memoryMatchVersion = "memmatch-v3-perlang-minkey+collision-ambiguous+longest+spoiler+sticky-inherit-disp+phonetic-srcboundary+budget+postcheck-declaware"
|
||||
// v4 (D39 слой 4): the longest-match suppressor is now DISPOSITION-GATED (suppressContained) —
|
||||
// a longer but LOWER-trust key (draft/ambiguous) no longer deletes a nested HIGHER-trust key
|
||||
// (approved/confirmed), closing the term-drift code root the D38.5 seed-promote only patched by
|
||||
// data (L3-suppressor-disposition-blind). A refused suppression is recorded loudly (trustGated).
|
||||
// The injection SELECTION changes for a draft-longer-over-approved-shorter chunk (the approved term
|
||||
// now survives), so a loud --resnapshot + golden re-capture (invariant №8).
|
||||
const memoryMatchVersion = "memmatch-v4-perlang-minkey+collision-ambiguous+longest-trustgated+spoiler+sticky-inherit-disp+phonetic-srcboundary+budget+postcheck-declaware"
|
||||
|
||||
// minKeyLenHan / minKeyLenPhonetic are the per-language single-key floors (A3, registry
|
||||
// "min_key_len per-язык"). An ideographic (Han) key is semantically distinct at 2 chars
|
||||
|
|
@ -133,11 +139,29 @@ type pickedEntry struct {
|
|||
disp injectionDisposition
|
||||
}
|
||||
|
||||
// trustGateEvent records a longest-match suppression the DISPOSITION-gate REFUSED (D39 слой 4):
|
||||
// a longer but LOWER-trust key (draft/ambiguous, e.g. draft 四代族长) that would have deleted a nested
|
||||
// HIGHER-trust key (approved/confirmed, e.g. approved 族长). The draft is editor-excluded (the block
|
||||
// is CONFIRMED-only), so suppressing the approved term would have left the reader with NOTHING — the
|
||||
// term-drift code root (L3). The nested higher-trust match survives and injects; this record turns
|
||||
// the near-silent drop into a loud, operator-visible signal (research/13 §7). A non-empty list means
|
||||
// the seed has a draft term nesting over an approved one — reconcile the seed so it stops recurring.
|
||||
type trustGateEvent struct {
|
||||
Suppressor string `json:"suppressor"` // the longer, lower-trust key blocked from suppressing
|
||||
SuppressorDisp string `json:"suppressor_disp"` // its best injection disposition (ambiguous)
|
||||
Protected string `json:"protected"` // the nested, higher-trust key it would have eaten
|
||||
ProtectedDisp string `json:"protected_disp"` // its best injection disposition (confirmed)
|
||||
}
|
||||
|
||||
// memorySelection is the hot path's output for one chunk.
|
||||
type memorySelection struct {
|
||||
injected []pickedEntry // in budget priority order (what the model sees)
|
||||
rejected []pickedEntry // spoiler-window rejects (C1, logged)
|
||||
evicted []pickedEntry // dropped by the token budget (F2, logged)
|
||||
// trustGated are the longest-match suppressions the disposition-gate refused (a lower-trust
|
||||
// longer key would have eaten a higher-trust nested one) — the term-drift code root, surfaced
|
||||
// loudly instead of a silent drop (research/13 §7). Deterministic (ms order, deduped by pair).
|
||||
trustGated []trustGateEvent
|
||||
// activeIDs are the exact-matched ids (NOT sticky) → the next chunk's sticky_prev,
|
||||
// each mapped to the DISPOSITION it fired with. Sticky carries this disposition
|
||||
// forward instead of recomputing it (D16.2): a sticky carry has no firing key, so it
|
||||
|
|
@ -305,10 +329,12 @@ func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]inj
|
|||
// Closes the source-vs-target boundary asymmetry (the target side already has
|
||||
// containsWholeWord). Han/kana are NOT boundary-checked (no word segmentation) — deliberate.
|
||||
occ = b.suppressUnboundedPhonetic(occ, ntext)
|
||||
// Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when
|
||||
// the longer key belongs to a spoiler-VALID entry at this chapter (self-review #6). A
|
||||
// spoiler-blocked longer key must not suppress a valid shorter name nested inside it.
|
||||
occ = b.suppressContained(occ, chapter)
|
||||
// Longest-match: drop a key fully inside a strictly-longer key's span — but ONLY when the
|
||||
// longer key belongs to a spoiler-VALID entry at this chapter (self-review #6) AND is at least
|
||||
// as TRUSTWORTHY as the nested match it would delete (D39 слой 4 — a lower-trust longer key must
|
||||
// not eat a higher-trust nested one). Refused suppressions are returned for a loud record.
|
||||
var trustGated []trustGateEvent
|
||||
occ, trustGated = b.suppressContained(occ, chapter)
|
||||
|
||||
// Map surviving key occurrences → entries, recording the LONGEST firing key per entry.
|
||||
matchedVia := map[int]string{}
|
||||
|
|
@ -321,7 +347,7 @@ func (b *MemoryBank) Select(chunk string, chapter int, stickyPrev map[string]inj
|
|||
}
|
||||
}
|
||||
|
||||
sel := memorySelection{activeIDs: map[string]injectionDisposition{}}
|
||||
sel := memorySelection{activeIDs: map[string]injectionDisposition{}, trustGated: trustGated}
|
||||
hits := map[string]pickedEntry{} // id → picked (exact)
|
||||
|
||||
// Iterate entries in their frozen (ORDER BY-stable) order — never map order.
|
||||
|
|
@ -466,22 +492,26 @@ func renderGlossaryBlock(injected []pickedEntry) string {
|
|||
return glossaryBlockHeader + "\n" + strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// editorConstraintHeader introduces the monolingual editor's dst-constraint block (D1).
|
||||
// The editor never sees the source, so it gets ONLY the approved target renderings as
|
||||
// consistency constraints — it must normalise any divergent rendering in the draft to
|
||||
// these canonical forms (inflecting for context) and touch nothing else.
|
||||
// editorConstraintHeader introduces the editor's dst-constraint block. It gives the editor the
|
||||
// approved TARGET renderings as consistency constraints — it must normalise any divergent rendering
|
||||
// in the draft to these canonical forms (inflecting for context) and touch nothing else.
|
||||
const editorConstraintHeader = "КАНОНИЧЕСКИЕ ПЕРЕВОДЫ имён и терминов (в черновике эти сущности должны быть переданы ИМЕННО этими формами — приводи к ним любые расхождения, склоняя по контексту; не вводи иных вариантов и не меняй ничего другого):"
|
||||
|
||||
// renderEditorConstraintBlock serializes the selected records' CONFIRMED dst renderings into
|
||||
// the monolingual editor's injection (D1). Unlike renderGlossaryBlock it emits the TARGET
|
||||
// forms only — no "src → dst" (the editor never sees the source: decisions-log D1). It is
|
||||
// CONFIRMED-only on purpose: an AMBIGUOUS record is a candidate the translator MAY have
|
||||
// legitimately rejected, so forcing the editor to rewrite the draft toward it would corrupt
|
||||
// a correct translation (mirrors the gate's CONFIRMED-only discipline, external-review #1).
|
||||
// Deduped by dst (an alias and its main entry share a rendering), in the budget priority
|
||||
// order fixed by Select. Returns "" when nothing CONFIRMED renders → the editor gets NO
|
||||
// injection (the plain draft-only layout). The block is a subset of the already
|
||||
// budget-limited memSel.injected, so it needs no separate token budget.
|
||||
// renderEditorConstraintBlock serializes the selected records' CONFIRMED dst renderings into the
|
||||
// editor's injection. Unlike renderGlossaryBlock it emits the TARGET forms only — bare canonical
|
||||
// Russian, not "src → dst". It is CONFIRMED-only on purpose: an AMBIGUOUS record is a candidate the
|
||||
// translator MAY have legitimately rejected, so forcing the editor to rewrite the draft toward it
|
||||
// would corrupt a correct translation (mirrors the gate's CONFIRMED-only discipline, external-review
|
||||
// #1). NOTE (D30.1 supersede of D1): the editor is now BILINGUAL — editor.md feeds it the source, so
|
||||
// the earlier "monolingual editor never sees the source" rationale for target-forms-only is stale
|
||||
// (the CONFIRMED-only discipline above is the LIVE reason and is unchanged here). Whether a bilingual
|
||||
// editor should receive the src→dst mapping (like the translator) so it can bind each canonical form
|
||||
// to its source term, rather than bare Russian, is an OPEN design question the D30.1 flip left
|
||||
// unclosed (flagged to the orchestrator; behaviour deliberately unchanged in this pack). Deduped by
|
||||
// dst (an alias and its main entry share a rendering), in the budget priority order fixed by Select.
|
||||
// Returns "" when nothing CONFIRMED renders → the editor gets NO injection (the plain draft-only
|
||||
// layout). The block is a subset of the already budget-limited memSel.injected, so it needs no
|
||||
// separate token budget.
|
||||
func renderEditorConstraintBlock(injected []pickedEntry) string {
|
||||
var lines []string
|
||||
seen := map[string]bool{}
|
||||
|
|
@ -604,40 +634,115 @@ func (ac *ahoCorasick) matches(text []rune) []acMatch {
|
|||
return out
|
||||
}
|
||||
|
||||
// suppressContained drops an occurrence fully contained inside a STRICTLY longer one
|
||||
// (longest-match, whole-entity replacement — A3), where the longer one belongs to a
|
||||
// spoiler-VALID entry at this chapter. Gating the SUPPRESSOR on spoiler validity closes
|
||||
// self-review #6: a spoiler-blocked longer key (e.g. 林动的父亲, until_ch=3, read at ch10)
|
||||
// must not silently drop a valid shorter name nested inside it (林动) — the shorter name
|
||||
// survives and injects, the longer one is separately spoiler-rejected+logged. Equal-length
|
||||
// overlaps are both kept (genuine surface ambiguity, surfaced by the injectivity check).
|
||||
// O(n²) over the few matches in a chunk.
|
||||
func (b *MemoryBank) suppressContained(ms []acMatch, chapter int) []acMatch {
|
||||
validSuppressor := func(m acMatch) bool {
|
||||
for _, ei := range b.keyOwners[b.ac.keys[m.keyIdx]] {
|
||||
if !spoilerBlocked(&b.entries[ei], chapter) {
|
||||
return true
|
||||
// trustRank orders injection dispositions by how much the reader may TRUST the record: CONFIRMED
|
||||
// (approved, authoritative) outranks AMBIGUOUS (draft/auto/collision-prone), which outranks REJECT
|
||||
// (spoiler-blocked / unset). It decides the longest-match containment contest below.
|
||||
func trustRank(d injectionDisposition) int {
|
||||
switch d {
|
||||
case memConfirmed:
|
||||
return 2
|
||||
case memAmbiguous:
|
||||
return 1
|
||||
default: // memReject / unset
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
// matchTrust returns the BEST (highest-trust) disposition an occurrence of the given key could
|
||||
// inject with at this chapter — the max over its spoiler-VALID owner entries — and whether ANY
|
||||
// valid owner exists (this second value subsumes the old validSuppressor: no valid owner ⇒ the
|
||||
// occurrence injects nothing). CONFIRMED beats AMBIGUOUS. It is the trust the disposition-gate
|
||||
// decides a containment contest on (D39 слой 4).
|
||||
func (b *MemoryBank) matchTrust(m acMatch, chapter int) (disp injectionDisposition, hasValid bool) {
|
||||
key := b.ac.keys[m.keyIdx]
|
||||
for _, ei := range b.keyOwners[key] {
|
||||
e := &b.entries[ei]
|
||||
if spoilerBlocked(e, chapter) {
|
||||
continue
|
||||
}
|
||||
if d := dispositionFor(e, key); !hasValid || trustRank(d) > trustRank(disp) {
|
||||
disp, hasValid = d, true
|
||||
}
|
||||
}
|
||||
return disp, hasValid
|
||||
}
|
||||
|
||||
// suppressContained applies longest-match / whole-entity replacement (A3), DISPOSITION-GATED (D39
|
||||
// слой 4). It drops an occurrence fully contained inside a STRICTLY longer one, but ONLY when the
|
||||
// longer one is BOTH (a) owned by a spoiler-VALID entry at this chapter (self-review #6: a spoiler-
|
||||
// blocked longer key like 林动的父亲, until_ch=3, read at ch10, must not eat a valid nested 林动) AND
|
||||
// (b) at least as TRUSTWORTHY as the nested match it would delete. A longer but LOWER-trust key
|
||||
// (a draft/ambiguous 四代族长) must NOT suppress a nested HIGHER-trust key (an approved 族长): the draft
|
||||
// is editor-excluded (renderEditorConstraintBlock is CONFIRMED-only), so eating the approved term
|
||||
// leaves the reader with nothing AND the drop is invisible to post-check/retrieval-state — the exact
|
||||
// term-drift code root the D38.5 seed-promote only patched by data (L3-recurrence). The whole-entity
|
||||
// longest-match is preserved when the longer key is ≥ trust (approved 四代族长 still beats approved
|
||||
// 族长; draft 四代族长 still eats a nested draft 族长). Each REFUSED suppression is recorded as a
|
||||
// trustGateEvent so the near-silent drop is surfaced loudly (research/13 §7). Equal-length overlaps
|
||||
// are both kept (genuine surface ambiguity). Deterministic (ms is sorted; events deduped by pair in
|
||||
// ms order). O(n²) over the few matches in a chunk.
|
||||
func (b *MemoryBank) suppressContained(ms []acMatch, chapter int) ([]acMatch, []trustGateEvent) {
|
||||
var kept []acMatch
|
||||
var gated []trustGateEvent
|
||||
for i, m := range ms {
|
||||
conDisp, conValid := b.matchTrust(m, chapter)
|
||||
contained := false
|
||||
var refused *trustGateEvent // a lower-trust valid longer key that WANTED to suppress m
|
||||
for j, o := range ms {
|
||||
if i == j {
|
||||
continue
|
||||
}
|
||||
if o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start) && validSuppressor(o) {
|
||||
contained = true
|
||||
// o must STRICTLY contain m's span to be a longest-match suppressor.
|
||||
if !(o.start <= m.start && o.end >= m.end && (o.end-o.start) > (m.end-m.start)) {
|
||||
continue
|
||||
}
|
||||
supDisp, supValid := b.matchTrust(o, chapter)
|
||||
if !supValid {
|
||||
continue // spoiler-blocked longer key — never a valid suppressor (self-review #6)
|
||||
}
|
||||
if !conValid || trustRank(supDisp) >= trustRank(conDisp) {
|
||||
contained = true // a valid, ≥-trust container wins the whole-entity longest-match
|
||||
break
|
||||
}
|
||||
// supValid && conValid && supDisp < conDisp: a lower-trust longer key wants to eat a
|
||||
// higher-trust nested one — REFUSE (the L3 fix). Remember the FIRST such refusal for the
|
||||
// loud record; keep scanning in case a higher-trust container legitimately suppresses m.
|
||||
if refused == nil {
|
||||
refused = &trustGateEvent{
|
||||
Suppressor: b.ac.keys[o.keyIdx], SuppressorDisp: string(supDisp),
|
||||
Protected: b.ac.keys[m.keyIdx], ProtectedDisp: string(conDisp),
|
||||
}
|
||||
}
|
||||
}
|
||||
if contained {
|
||||
continue // m suppressed by the whole-entity longest-match; any refusal above was moot
|
||||
}
|
||||
if !contained {
|
||||
kept = append(kept, m)
|
||||
if refused != nil {
|
||||
gated = append(gated, *refused) // m survived AND a lower-trust key tried to eat it — loud
|
||||
}
|
||||
}
|
||||
return kept
|
||||
return kept, dedupeTrustGate(gated)
|
||||
}
|
||||
|
||||
// dedupeTrustGate collapses identical (suppressor→protected) events to a distinct set, preserving
|
||||
// first-occurrence (ms) order so the count and detail are deterministic and count distinct term
|
||||
// collisions rather than raw occurrences.
|
||||
func dedupeTrustGate(evs []trustGateEvent) []trustGateEvent {
|
||||
if len(evs) < 2 {
|
||||
return evs
|
||||
}
|
||||
seen := map[[2]string]bool{}
|
||||
var out []trustGateEvent
|
||||
for _, e := range evs {
|
||||
k := [2]string{e.Suppressor, e.Protected}
|
||||
if seen[k] {
|
||||
continue
|
||||
}
|
||||
seen[k] = true
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// suppressUnboundedPhonetic drops an occurrence of a SPACED-SCRIPT phonetic key (Latin or
|
||||
|
|
|
|||
126
backend/internal/pipeline/memory_trustgate_test.go
Normal file
126
backend/internal/pipeline/memory_trustgate_test.go
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"textmachine/backend/internal/store"
|
||||
)
|
||||
|
||||
// memory_trustgate_test.go pins the DISPOSITION-GATED longest-match suppressor (D39 слой 4,
|
||||
// L3-suppressor-disposition-blind): a longer but LOWER-trust key (draft/ambiguous 四代族长) must NOT
|
||||
// suppress a nested HIGHER-trust key (approved 族长) — the term-drift code root. The whole-entity
|
||||
// longest-match is preserved when the longer key is ≥ trust. Each refused suppression is recorded
|
||||
// loudly (trustGated). Han keys are used so normalizeSourceKey is (near) identity — the stored
|
||||
// event keys equal the raw src.
|
||||
|
||||
// trustGateKeys collects the (suppressor→protected) pairs of a selection's trust-gate events.
|
||||
func trustGateKeys(sel memorySelection) map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, e := range sel.trustGated {
|
||||
m[e.Suppressor] = e.Protected
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// TestSuppressorTrustGateFires is the FIX: a longer draft key nested over an approved shorter key no
|
||||
// longer eats it. BEFORE the gate the approved 族长 was dropped from Select (never injected, invisible
|
||||
// to post-check) and the draft 四代族长 injected only AMBIGUOUS (editor-excluded) → the reader got
|
||||
// NOTHING. AFTER: the approved 族长 survives as CONFIRMED (editor gets it) and the collision is logged.
|
||||
func TestSuppressorTrustGateFires(t *testing.T) {
|
||||
zuzhang := gl("族长", "глава клана", "title", "approved")
|
||||
zuzhang.Decl = declJSON(false, "глава клана", "главы клана", "главе клана")
|
||||
sidai := gl("四代族长", "четвёртый глава клана", "title", "draft") // longer, LOWER trust
|
||||
|
||||
b := bankFrom([]store.GlossaryEntry{zuzhang, sidai})
|
||||
sel := b.Select("四代族长 вошёл в зал.", 1, nil, 0)
|
||||
m := injMap(sel)
|
||||
|
||||
if m["族长"] != memConfirmed {
|
||||
t.Fatalf("approved 族长 must survive as CONFIRMED (the fix), got %q; injected=%v", m["族长"], m)
|
||||
}
|
||||
if m["四代族长"] != memAmbiguous {
|
||||
t.Errorf("draft 四代族长 must still inject AMBIGUOUS alongside, got %q", m["四代族长"])
|
||||
}
|
||||
// Editor constraint block: the approved term is now present (was empty before the fix).
|
||||
if blk := renderEditorConstraintBlock(sel.injected); !strings.Contains(blk, "глава клана") {
|
||||
t.Errorf("editor constraint block must carry the approved «глава клана», got %q", blk)
|
||||
}
|
||||
// Loud record of the refused suppression.
|
||||
if got := trustGateKeys(sel); got["四代族长"] != "族长" {
|
||||
t.Errorf("expected a trust-gate event 四代族长⊃族长, got %v", got)
|
||||
}
|
||||
if len(sel.trustGated) != 1 {
|
||||
t.Errorf("expected exactly 1 trust-gate event, got %d", len(sel.trustGated))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuppressorApprovedOverApprovedStillSuppresses is the regression-safety guard: an APPROVED
|
||||
// longer key legitimately eats a nested APPROVED shorter one (whole-entity longest-match, unchanged).
|
||||
// This also spot-verifies the D38.5 promote: once 四代族长 is promoted draft→approved the whole entity
|
||||
// wins — but the FIX above means the promote is no longer LOAD-BEARING for correctness (a draft
|
||||
// 四代族长 no longer silently drops 族长).
|
||||
func TestSuppressorApprovedOverApprovedStillSuppresses(t *testing.T) {
|
||||
zuzhang := gl("族长", "глава клана", "title", "approved")
|
||||
zuzhang.Decl = declJSON(false, "глава клана")
|
||||
sidai := gl("四代族长", "четвёртый глава клана", "title", "approved") // longer, EQUAL trust
|
||||
|
||||
b := bankFrom([]store.GlossaryEntry{zuzhang, sidai})
|
||||
sel := b.Select("四代族长 вошёл в зал.", 1, nil, 0)
|
||||
m := injMap(sel)
|
||||
|
||||
if _, present := m["族长"]; present {
|
||||
t.Errorf("approved 族长 nested in an EQUAL-trust approved 四代族长 must be suppressed (whole-entity), got injected=%v", m)
|
||||
}
|
||||
if m["四代族长"] != memConfirmed {
|
||||
t.Errorf("the whole entity 四代族长 must inject CONFIRMED, got %q", m["四代族长"])
|
||||
}
|
||||
if len(sel.trustGated) != 0 {
|
||||
t.Errorf("an EQUAL-trust suppression is legitimate — no trust-gate event, got %d", len(sel.trustGated))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuppressorDraftOverDraftStillSuppresses pins the edge the task calls out: two DRAFT keys
|
||||
// (equal trust) behave as before — the longer suppresses the nested, no trust-gate event.
|
||||
func TestSuppressorDraftOverDraftStillSuppresses(t *testing.T) {
|
||||
zuzhang := gl("族长", "глава клана", "title", "draft")
|
||||
sidai := gl("四代族长", "четвёртый глава клана", "title", "draft")
|
||||
|
||||
b := bankFrom([]store.GlossaryEntry{zuzhang, sidai})
|
||||
sel := b.Select("四代族长 вошёл в зал.", 1, nil, 0)
|
||||
m := injMap(sel)
|
||||
|
||||
if _, present := m["族长"]; present {
|
||||
t.Errorf("draft 族长 nested in an equal-trust draft 四代族长 must be suppressed, got %v", m)
|
||||
}
|
||||
if m["四代族长"] != memAmbiguous {
|
||||
t.Errorf("the longer draft must inject AMBIGUOUS, got %q", m["四代族长"])
|
||||
}
|
||||
if len(sel.trustGated) != 0 {
|
||||
t.Errorf("equal-trust suppression records no event, got %d", len(sel.trustGated))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuppressorSpoilerBlockedLongDoesNotSuppress preserves self-review #6: a spoiler-BLOCKED longer
|
||||
// key is not a valid suppressor at all, so the nested valid approved key survives — and this is NOT a
|
||||
// trust-gate event (the refusal path records only a spoiler-VALID lower-trust suppressor).
|
||||
func TestSuppressorSpoilerBlockedLongDoesNotSuppress(t *testing.T) {
|
||||
zuzhang := gl("族长", "глава клана", "title", "approved")
|
||||
zuzhang.Decl = declJSON(false, "глава клана")
|
||||
sidai := gl("四代族长", "четвёртый глава клана", "title", "approved")
|
||||
sidai.SinceCh = 200 // spoiler-blocked before ch200
|
||||
|
||||
b := bankFrom([]store.GlossaryEntry{zuzhang, sidai})
|
||||
sel := b.Select("四代族长 вошёл в зал.", 1, nil, 0) // ch1 → 四代族长 blocked
|
||||
m := injMap(sel)
|
||||
|
||||
if m["族长"] != memConfirmed {
|
||||
t.Errorf("approved 族长 must survive a spoiler-blocked longer key, got %q", m["族长"])
|
||||
}
|
||||
if len(sel.rejected) != 1 || sel.rejected[0].entry.src != "四代族长" {
|
||||
t.Errorf("the spoiler-blocked 四代族长 must be rejected+logged, got %v", sel.rejected)
|
||||
}
|
||||
if len(sel.trustGated) != 0 {
|
||||
t.Errorf("a spoiler-block is not a trust-gate event, got %d", len(sel.trustGated))
|
||||
}
|
||||
}
|
||||
197
backend/internal/pipeline/quality.go
Normal file
197
backend/internal/pipeline/quality.go
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// quality.go: the DETERMINISTIC per-run quality-report (D39 слой 5, H5-no-in-loop-quality-signal) —
|
||||
// the online/offline quality telemetry the owner asked for "from day one" (п.25/п.31), which
|
||||
// research/18 §C1 #10 flagged as a NOW lever but was silently deferred to Ф2. It AGGREGATES signals
|
||||
// that are ALREADY computed and stored (retrieval_state: glossary post-check misses, cheap style
|
||||
// flaggers, trust-gated suppressions; chunk_status: echo/CJK/sanitizer flags) plus ONE cheap
|
||||
// deterministic structural KPI (sentences per narrative paragraph — the "рубленые абзацы" claim-1
|
||||
// signal) recomputed from the exported final text. It is a PURE READ-ONLY projection like Status: $0,
|
||||
// no LLM, no snapshot touch, no checkpoint replay — only OBSERVABILITY, never a gate. The semantic
|
||||
// span-judge (inversion/omission backstop for claim-2) is NOT here — it is research-dependent (пак-2).
|
||||
|
||||
// QualityReport is the whole-book per-run quality projection.
|
||||
type QualityReport struct {
|
||||
BookID string `json:"book_id"`
|
||||
TotalChunks int `json:"total_chunks"`
|
||||
// TextChunks is the number of chunks whose exported final text was available for the structural
|
||||
// KPI (done or cosmetically-stripped); a flagged-empty chunk contributes no prose.
|
||||
TextChunks int `json:"text_chunks"`
|
||||
// ProcessedChunks is the number of chunks that REACHED the final stage (a final-stage row exists:
|
||||
// ok, cosmetic-strip, or skipped-because-upstream-flagged) — the rate denominator, so echo/CJK
|
||||
// rates are a bounded [0,1] fraction of processed chunks (an echo chunk is a skipped final row,
|
||||
// disjoint from the exported-text set, which is why dividing by TextChunks alone could exceed 1).
|
||||
ProcessedChunks int `json:"processed_chunks"`
|
||||
|
||||
// Claim-1 structural KPI (рубленые абзацы). MeanSentPerNarrPara ≈ 1 is choppy (one sentence per
|
||||
// paragraph — the owner's exact complaint); higher is merged discourse prose. Aggregated as
|
||||
// total sentences / total narrative paragraphs across the book, so it is a true book-wide mean.
|
||||
NarrativeSentences int `json:"narrative_sentences"`
|
||||
NarrativeParagraphs int `json:"narrative_paragraphs"`
|
||||
MeanSentPerNarrPara float64 `json:"mean_sentences_per_narrative_paragraph"`
|
||||
|
||||
// Deterministic signal aggregates (all observability, never a disposition).
|
||||
DialogueDashFlags int `json:"dialogue_dash_flags"` // Rosenthal dialogue-dash inconsistencies
|
||||
GlossaryMisses int `json:"glossary_misses"` // CONFIRMED post-check misses (D10 consistency)
|
||||
NumberDriftFlags int `json:"number_drift_flags"` // reflow number drift + 万/億 magnitude drift
|
||||
TrustGated int `json:"trust_gated"` // lower-trust suppressions refused (seed hygiene)
|
||||
CJKLeakChunks int `json:"cjk_leak_chunks"` // chunks the sanitizer stripped a CJK leak from
|
||||
EchoChunks int `json:"echo_chunks"` // chunks flagged cjk_artifact (untranslated echo)
|
||||
|
||||
// Rates over ProcessedChunks (0..1) for the two "instant unreadability" families, so the numbers
|
||||
// read as a bounded fraction of processed chunks rather than a bare count.
|
||||
CJKLeakRate float64 `json:"cjk_leak_rate"`
|
||||
EchoRate float64 `json:"echo_rate"`
|
||||
|
||||
Chunks []ChunkQuality `json:"chunks,omitempty"`
|
||||
}
|
||||
|
||||
// ChunkQuality is one chunk's per-chunk quality row (the "where did quality slip" signal).
|
||||
type ChunkQuality struct {
|
||||
Chapter int `json:"chapter"`
|
||||
ChunkIdx int `json:"chunk_idx"`
|
||||
NarrativeSentences int `json:"narrative_sentences"`
|
||||
NarrativeParagraphs int `json:"narrative_paragraphs"`
|
||||
DialogueDashFlags int `json:"dialogue_dash_flags"`
|
||||
GlossaryMisses int `json:"glossary_misses"`
|
||||
NumberDriftFlags int `json:"number_drift_flags"`
|
||||
TrustGated int `json:"trust_gated"`
|
||||
}
|
||||
|
||||
// narrativeStructure computes the claim-1 structural KPI over a FINAL Russian text: the count of
|
||||
// NARRATIVE paragraphs (non-empty lines that are NOT a dialogue turn opened with a dash «—»/«–»/«-»)
|
||||
// and the total sentences within them (the oracle-parity splitSentences). Dialogue turns are excluded
|
||||
// (they are legitimately one short line). Deterministic and pure — the same signal the reflow lever
|
||||
// is supposed to move (exp14 mean-sentences-per-paragraph), reused here as in-loop observability.
|
||||
func narrativeStructure(final string) (sentences, paragraphs int) {
|
||||
for _, line := range strings.Split(final, "\n") {
|
||||
t := strings.TrimSpace(line)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
if rs := []rune(t); rs[0] == '—' || rs[0] == '–' || rs[0] == '-' {
|
||||
continue // a dialogue turn — not a narrative paragraph
|
||||
}
|
||||
paragraphs++
|
||||
sentences += len(splitSentences(t))
|
||||
}
|
||||
return sentences, paragraphs
|
||||
}
|
||||
|
||||
// QualityReport builds the read-only per-run quality projection. It opens no jobs, reserves nothing,
|
||||
// makes no LLM call — it reads the persisted chunk_status / retrieval_state and, for each chunk with
|
||||
// an exported final text, the $0 final checkpoint to recompute the structural KPI. Safe to run
|
||||
// whenever `report`/`status` are (the same exclusive-lock rule). Deterministic over the store.
|
||||
// CAVEAT (same class as Status's config-drift): the exported-text signals key on the CURRENT config's
|
||||
// final-stage NAME; if a config edit renamed the final stage since the run, the stored rows use the
|
||||
// old name and the structural KPI / echo / CJK rates read 0 (the underlying spend/verdict rows are
|
||||
// untouched — surface `status` shows the drift). A run under the same config reads correctly.
|
||||
func (r *Runner) QualityReport() (*QualityReport, error) {
|
||||
statuses, err := r.Store.ChunkStatusesForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
states, err := r.Store.RetrievalStatesForBook(r.Book.BookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rep := &QualityReport{BookID: r.Book.BookID}
|
||||
byChunk := map[chunkKey]*ChunkQuality{}
|
||||
order := []chunkKey{}
|
||||
chunkOf := func(k chunkKey) *ChunkQuality {
|
||||
if q := byChunk[k]; q != nil {
|
||||
return q
|
||||
}
|
||||
q := &ChunkQuality{Chapter: k.chapter, ChunkIdx: k.chunkIdx}
|
||||
byChunk[k] = q
|
||||
order = append(order, k)
|
||||
return q
|
||||
}
|
||||
|
||||
// Aggregate the stored retrieval-state signals (glossary consistency, style breakdown, trust-gated).
|
||||
for _, rs := range states {
|
||||
q := chunkOf(chunkKey{rs.Chapter, rs.ChunkIdx})
|
||||
q.GlossaryMisses += rs.NPostcheckMiss
|
||||
q.TrustGated += rs.NTrustGatedSuppress
|
||||
rep.GlossaryMisses += rs.NPostcheckMiss
|
||||
rep.TrustGated += rs.NTrustGatedSuppress
|
||||
if rs.NStyleFlags > 0 && rs.StyleDetail != "" {
|
||||
var cg cheapGateResult
|
||||
if json.Unmarshal([]byte(rs.StyleDetail), &cg) == nil {
|
||||
dash := cg.DialogueDash
|
||||
drift := cg.NumberDrift + cg.NumberMagnitude
|
||||
q.DialogueDashFlags += dash
|
||||
q.NumberDriftFlags += drift
|
||||
rep.DialogueDashFlags += dash
|
||||
rep.NumberDriftFlags += drift
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The exported-text chunks (the FINAL stage's row): echo/CJK flag rates + the structural KPI.
|
||||
lastStage := ""
|
||||
if n := len(r.Pipeline.Stages); n > 0 {
|
||||
lastStage = r.Pipeline.Stages[n-1].Name
|
||||
}
|
||||
seen := map[chunkKey]bool{}
|
||||
for _, cs := range statuses {
|
||||
k := chunkKey{cs.Chapter, cs.ChunkIdx}
|
||||
if !seen[k] {
|
||||
seen[k] = true
|
||||
rep.TotalChunks++
|
||||
}
|
||||
if cs.Stage != lastStage {
|
||||
continue // the exported text and the final verdict live on the final stage's row
|
||||
}
|
||||
// Every chunk that REACHED the final stage has exactly one lastStage row (ok, cosmetic-strip,
|
||||
// or skipped-because-an-upstream-stage-flagged). This is the common rate denominator so the
|
||||
// echo/CJK rates are a bounded fraction of processed chunks (an echo chunk is a skipped final
|
||||
// row, disjoint from the exported-text set — dividing by TextChunks alone could exceed 100%).
|
||||
rep.ProcessedChunks++
|
||||
if cs.FlagReason == string(FlagCJKArtifact) {
|
||||
rep.EchoChunks++
|
||||
chunkOf(k) // ensure the chunk row exists even if it has no retrieval-state signals
|
||||
}
|
||||
if cs.FlagReason == string(FlagSanitizerStripped) {
|
||||
rep.CJKLeakChunks++ // a stripped chunk carried a markdown/CJK cosmetic leak (detail says which)
|
||||
}
|
||||
// Structural KPI: recompute over the exported final text (ok, or the cosmetic-stripped export).
|
||||
if cs.FinalHash == "" {
|
||||
continue
|
||||
}
|
||||
if cs.Disposition != string(DispOK) && cs.FlagReason != string(FlagSanitizerStripped) {
|
||||
continue // a dropped chunk exported nothing
|
||||
}
|
||||
cp, cperr := r.Store.GetCheckpoint(cs.FinalHash)
|
||||
if cperr != nil {
|
||||
return nil, cperr
|
||||
}
|
||||
if cp == nil || strings.TrimSpace(cp.ResponseText) == "" {
|
||||
continue
|
||||
}
|
||||
sent, para := narrativeStructure(exportNormalize(cp.ResponseText))
|
||||
q := chunkOf(k)
|
||||
q.NarrativeSentences, q.NarrativeParagraphs = sent, para
|
||||
rep.NarrativeSentences += sent
|
||||
rep.NarrativeParagraphs += para
|
||||
rep.TextChunks++
|
||||
}
|
||||
|
||||
if rep.NarrativeParagraphs > 0 {
|
||||
rep.MeanSentPerNarrPara = float64(rep.NarrativeSentences) / float64(rep.NarrativeParagraphs)
|
||||
}
|
||||
if rep.ProcessedChunks > 0 {
|
||||
rep.CJKLeakRate = float64(rep.CJKLeakChunks) / float64(rep.ProcessedChunks)
|
||||
rep.EchoRate = float64(rep.EchoChunks) / float64(rep.ProcessedChunks)
|
||||
}
|
||||
for _, k := range order {
|
||||
rep.Chunks = append(rep.Chunks, *byChunk[k])
|
||||
}
|
||||
return rep, nil
|
||||
}
|
||||
85
backend/internal/pipeline/quality_test.go
Normal file
85
backend/internal/pipeline/quality_test.go
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// quality_test.go pins the per-run quality-report (D39 слой 5, H5): the claim-1 structural KPI and
|
||||
// the aggregation of the already-stored deterministic signals. Pure read-only projection.
|
||||
|
||||
// TestNarrativeStructure pins the claim-1 KPI: narrative paragraphs (non-dialogue lines) and their
|
||||
// sentence count. A choppy text (one sentence per line) reads ≈1.0; merged prose reads higher;
|
||||
// dialogue turns are excluded.
|
||||
func TestNarrativeStructure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
text string
|
||||
wantSent int
|
||||
wantPara int
|
||||
}{
|
||||
// Choppy: 3 one-sentence narrative lines → 3 sentences / 3 paragraphs = 1.0 (претензия-1).
|
||||
{"choppy", "Он вышел.\nНебо серело.\nОн вздохнул.", 3, 3},
|
||||
// Merged: one paragraph, 3 sentences → 3 / 1 = 3.0 (the reflow fix).
|
||||
{"merged", "Он вышел за дверь. Небо серело. Он вздохнул, глядя вдаль.", 3, 1},
|
||||
// Dialogue turns excluded; only the narrative line counts.
|
||||
{"dialogue-excluded", "Дядя нахмурился.\n— Время летит. Садись, — сказал он.\n— Спасибо.", 1, 1},
|
||||
// Blank lines skipped; a two-sentence narrative paragraph plus a one-sentence one.
|
||||
{"mixed", "Первый абзац. Ещё предложение.\n\nВторой абзац.", 3, 2},
|
||||
}
|
||||
for _, c := range cases {
|
||||
sent, para := narrativeStructure(c.text)
|
||||
if sent != c.wantSent || para != c.wantPara {
|
||||
t.Errorf("%s: narrativeStructure = (%d sent, %d para), want (%d, %d)", c.name, sent, para, c.wantSent, c.wantPara)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestQualityReportAggregates runs a real 1-chunk book and asserts the read-only quality projection
|
||||
// aggregates the structural KPI from the exported final text plus the stored signals.
|
||||
func TestQualityReportAggregates(t *testing.T) {
|
||||
rec := &reqRec{}
|
||||
// The editor (final stage) emits a KNOWN 3-line final: 2 narrative paragraphs (3 sentences total)
|
||||
// + 1 dialogue turn (excluded) → mean 1.5 sentences/narrative-paragraph.
|
||||
editorFinal := "Судзуки шёл по коридору. Он думал о вчерашней лекции.\nБиблиотека была тихой.\n— Доброе утро, — сказал он."
|
||||
srv := newJSONProvider(rec, func(body string) (string, string) {
|
||||
if isEditBody(body) {
|
||||
return editorFinal, "stop"
|
||||
}
|
||||
return "Судзуки шёл по коридору.", "stop"
|
||||
})
|
||||
defer srv.Close()
|
||||
bookPath := setupProjectOpts(t, srv.URL, projectOpts{})
|
||||
ctx := context.Background()
|
||||
|
||||
r := newRunner(t, bookPath)
|
||||
defer r.Close()
|
||||
if _, err := r.TranslateBook(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
q, err := r.QualityReport()
|
||||
if err != nil {
|
||||
t.Fatalf("QualityReport: %v", err)
|
||||
}
|
||||
if q.TotalChunks != 1 || q.TextChunks != 1 || q.ProcessedChunks != 1 {
|
||||
t.Fatalf("chunk counts: total=%d processed=%d text=%d, want 1/1/1", q.TotalChunks, q.ProcessedChunks, q.TextChunks)
|
||||
}
|
||||
// Rates are bounded [0,1] over ProcessedChunks (the disjoint-set bug fix).
|
||||
if q.EchoRate < 0 || q.EchoRate > 1 || q.CJKLeakRate < 0 || q.CJKLeakRate > 1 {
|
||||
t.Errorf("rates must be within [0,1], got echo=%.3f cjk=%.3f", q.EchoRate, q.CJKLeakRate)
|
||||
}
|
||||
if q.NarrativeSentences != 3 || q.NarrativeParagraphs != 2 {
|
||||
t.Errorf("structural KPI = %d sent / %d para, want 3/2", q.NarrativeSentences, q.NarrativeParagraphs)
|
||||
}
|
||||
if q.MeanSentPerNarrPara != 1.5 {
|
||||
t.Errorf("mean sentences/narrative-paragraph = %.3f, want 1.5", q.MeanSentPerNarrPara)
|
||||
}
|
||||
// A clean run: no glossary misses, no echo, no CJK strip, no trust-gated, no dialogue-dash issue.
|
||||
if q.GlossaryMisses != 0 || q.EchoChunks != 0 || q.CJKLeakChunks != 0 || q.TrustGated != 0 || q.DialogueDashFlags != 0 {
|
||||
t.Errorf("clean run must have zero defect signals, got %+v", q)
|
||||
}
|
||||
if len(q.Chunks) != 1 || q.Chunks[0].NarrativeParagraphs != 2 {
|
||||
t.Errorf("per-chunk row missing/wrong: %+v", q.Chunks)
|
||||
}
|
||||
}
|
||||
|
|
@ -22,8 +22,9 @@ import (
|
|||
// translation — the only role the excision coverage-gate is meaningful on.
|
||||
const roleTranslator = "translator"
|
||||
|
||||
// roleEditor is the monolingual editor stage (D1): it receives the draft plus the approved
|
||||
// glossary's dst forms as target-consistency constraints, never the source.
|
||||
// roleEditor is the BILINGUAL editor stage (D30.1 supersede of D1): editor.md feeds it BOTH the
|
||||
// source and the draft, plus the approved glossary's CONFIRMED dst forms as target-consistency
|
||||
// constraints. (The stale "monolingual, never sees the source" framing is D1, fully superseded.)
|
||||
const roleEditor = "editor"
|
||||
|
||||
// Runner executes a book's pipeline.
|
||||
|
|
@ -145,8 +146,16 @@ func openRunner(bookPath string, logger *slog.Logger, forWrite bool) (*Runner, e
|
|||
func (r *Runner) Close() error { return r.Store.Close() }
|
||||
|
||||
func (r *Runner) loadTemplates() error {
|
||||
pair := r.Book.LangPair()
|
||||
for _, st := range r.Pipeline.Stages {
|
||||
tpl, err := LoadPromptTemplate(st.Prompt)
|
||||
// Resolve the stage's prompt from the pair-keyed pack by the book's language pair (D39 слой 2):
|
||||
// a legacy single `prompt` is pair-agnostic; a `prompts` pack fails LOUD on a missing pair here
|
||||
// (a ja book against a zh-ru-only pack stops instead of silently running Chinese conventions).
|
||||
promptPath, err := st.PromptPathFor(pair)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tpl, err := LoadPromptTemplate(promptPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
91
backend/internal/pipeline/runner_pairpack_test.go
Normal file
91
backend/internal/pipeline/runner_pairpack_test.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"textmachine/backend/internal/obs"
|
||||
)
|
||||
|
||||
// runner_pairpack_test.go is the end-to-end proof of the pair-keyed prompt seam (D39 слой 2): a book
|
||||
// whose LangPair() is NOT in a stage's pair pack FAILS LOUD at loadTemplates (closing the latent bug
|
||||
// where a ja book rode the zh-parataxis prompt), while a book of the populated pair resolves cleanly.
|
||||
|
||||
// setupPairPackProject writes a minimal project whose stages use a pair-keyed `prompts` pack carrying
|
||||
// ONLY zh-ru, with the book's source_lang set to srcLang. Returns the book.yaml path.
|
||||
func setupPairPackProject(t *testing.T, srcLang string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
writeFile(t, filepath.Join(dir, "prompts", "translator.md"), "Переводи.\n---USER---\n{{text}}")
|
||||
writeFile(t, filepath.Join(dir, "prompts", "editor.md"), "Редактируй.\n---USER---\n{{text}}\n{{draft}}")
|
||||
writeFile(t, filepath.Join(dir, "models.yaml"), fmt.Sprintf(`
|
||||
prices_checked: %q
|
||||
default_model: fake-model
|
||||
providers:
|
||||
fake: { kind: openai, base_url: http://127.0.0.1:0 }
|
||||
models:
|
||||
fake-model: { provider: fake, price: { input_per_m: 1.0, output_per_m: 2.0 } }
|
||||
`, time.Now().UTC().Format("2006-01-02")))
|
||||
// Pair-keyed pack — ONLY zh-ru is populated.
|
||||
writeFile(t, filepath.Join(dir, "pipeline.yaml"), `
|
||||
core: C1
|
||||
version: 1
|
||||
defaults: { max_output_ratio: 2.0, min_max_tokens: 512 }
|
||||
stages:
|
||||
- name: draft
|
||||
role: translator
|
||||
model: fake-model
|
||||
prompts: { zh-ru: prompts/translator.md }
|
||||
prompt_version: v-test
|
||||
reasoning: "off"
|
||||
- name: edit
|
||||
role: editor
|
||||
model: fake-model
|
||||
prompts: { zh-ru: prompts/editor.md }
|
||||
prompt_version: v-test
|
||||
reasoning: "off"
|
||||
`)
|
||||
writeFile(t, filepath.Join(dir, "source.txt"), "四代族长 вошёл в зал.")
|
||||
writeFile(t, filepath.Join(dir, "book.yaml"), fmt.Sprintf(`
|
||||
book_id: pairpack
|
||||
title: Тест
|
||||
source_lang: %s
|
||||
target_lang: ru
|
||||
genre: ранобэ
|
||||
audience: тест
|
||||
venuti: 0.5
|
||||
honorifics: keep
|
||||
transcription: polivanov
|
||||
pipeline: pipeline.yaml
|
||||
models: models.yaml
|
||||
source_file: source.txt
|
||||
ceilings: { book_usd: 1.0, day_usd: 2.0 }
|
||||
`, srcLang))
|
||||
return filepath.Join(dir, "book.yaml")
|
||||
}
|
||||
|
||||
func TestPairPackJaBookFailsLoud(t *testing.T) {
|
||||
// A ja→ru book against a zh-ru-only pack must STOP at loadTemplates, not silently run zh prompts.
|
||||
_, err := NewRunner(setupPairPackProject(t, "ja"), obs.NewLogger())
|
||||
if err == nil {
|
||||
t.Fatal("a ja book against a zh-ru-only prompt pack must fail loud (D39 слой 2)")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "ja-ru") {
|
||||
t.Errorf("fail-loud error must name the missing pair ja-ru, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPairPackZhBookResolves(t *testing.T) {
|
||||
// A zh→ru book of the populated pair resolves the pack cleanly (prod path, behaviour-neutral).
|
||||
r, err := NewRunner(setupPairPackProject(t, "zh"), obs.NewLogger())
|
||||
if err != nil {
|
||||
t.Fatalf("a zh book must resolve the zh-ru pack: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
if len(r.templates) != 2 {
|
||||
t.Errorf("expected both stage templates loaded, got %d", len(r.templates))
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,9 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
"golang.org/x/text/width"
|
||||
)
|
||||
|
||||
// sanitizer.go: the output-sanitizer (D30.3) — a deterministic verdict-axis gate on the
|
||||
|
|
@ -66,7 +69,16 @@ import (
|
|||
// («123»→«123», not deleted) and ideographic comma/period, space-replaces dropped ideographs (no
|
||||
// word-merge), removes an emptied bracket pair, and no longer reformats a legit «2 : 1» — all shift
|
||||
// a stripped chunk's EXPORT text → a loud --resnapshot + golden re-capture (invariant №8).
|
||||
const sanitizerVersion = "sanitizer-v4"
|
||||
// v5 (D39 слой 6, export-contract): the detector is now a PURE detector over the export-normalised
|
||||
// text — the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «123»→«123», ideographic space
|
||||
// U+3000, «、»→«,» «。»→«.») are folded by the shared recoverableFold (reusing the x/text width.Fold
|
||||
// primitive instead of the hand-rolled «r-0xFEE0» arithmetic, closing L5-stripcosmetic-duplicates-nfkc;
|
||||
// deliberately NOT full NFKC, which would clobber «…»/«№») and therefore NO LONGER FIRE the CJK-leak
|
||||
// class — only genuinely contentless CJK (Han/kana/CJK brackets & symbols) does. stripCosmetic
|
||||
// delegates that fold too. This shifts which chunks flag and a stripped chunk's export → a loud
|
||||
// --resnapshot + golden re-capture (invariant №8). [exportNormalize itself is an EPHEMERAL export
|
||||
// projection, not a persisted input; only stripCosmetic's output rides a checkpoint, versioned here.]
|
||||
const sanitizerVersion = "sanitizer-v5"
|
||||
|
||||
// sanitizer tuning constants (versioned by sanitizerVersion).
|
||||
const (
|
||||
|
|
@ -355,37 +367,44 @@ func detectBrokenWords(text string) (int, []string) {
|
|||
|
||||
// --- 6. CJK-leak in the final --------------------------------------------------
|
||||
|
||||
// isCJKLeakRune reports whether r is a stray CJK glyph that has NO place in a Russian final: a
|
||||
// Han ideograph, kana, a fullwidth/halfwidth form (U+FF00–FFEF — «,()!» etc.), or a CJK
|
||||
// symbol/punctuation (U+3001–303F — «、。「」»). The ideographic SPACE U+3000 is deliberately
|
||||
// EXCLUDED (it is an indent artifact, not a content leak — research/18 §C#4 "draft-21 includes a
|
||||
// U+3000 indent → 9 real leaks"): stripCosmetic normalises it to a plain space, but it does not
|
||||
// alone FIRE the class, so a legitimately-indented chunk is not flag-stormed. In clean Russian
|
||||
// prose every one of these is count-0, so the class is high precision. The intrinsic classify()
|
||||
// already flags a ≥15%-CJK output as an ECHO (cjk_artifact, substantive) BEFORE the sanitizer
|
||||
// runs, so this class only ever sees a SPARSE leak in otherwise-Russian text.
|
||||
// isCJKLeakRune reports whether r is a genuinely CONTENTLESS CJK glyph that has NO place in a Russian
|
||||
// final AND cannot be recovered by the export normaliser: a Han ideograph, kana, or a CJK
|
||||
// symbol/bracket/punctuation (U+3001–303F — «「」『』【】〜※»). It is ALWAYS consulted on
|
||||
// recoverableFold-ed text (detectCJKLeak/stripCosmetic fold first), so the RECOVERABLE wrong-glyph
|
||||
// forms are already gone before this runs: fullwidth ASCII «!1» folded to «!1», the ideographic
|
||||
// space U+3000 folded to a plain space, and «、»/«。» mapped to «,»/«.». Those must NOT fire — the
|
||||
// export contract silently fixes them (D39 слой 6); only the contentless class remains here. ACCEPTED
|
||||
// NARROWING (precision over recall): the old code fired on the WHOLE U+FF00–FFEF block; width.Fold now
|
||||
// recovers the common fullwidth-ASCII/symbol forms to real glyphs (so they no longer flag), but a few
|
||||
// RARE legacy forms in that block (halfwidth hangul jamo, halfwidth arrows/symbols, the katakana
|
||||
// middle dot) fold to non-CJK glyphs this predicate does not catch and ship un-flagged — an
|
||||
// extremely improbable artifact in zh/ja→ru output, traded for not over-flagging the common case. In
|
||||
// clean Russian prose every one of these is count-0, so the class is high precision. The intrinsic
|
||||
// classify() already flags a ≥15%-CJK output as an ECHO (cjk_artifact, substantive) BEFORE the
|
||||
// sanitizer runs, so this class only ever sees a SPARSE leak in otherwise-Russian text.
|
||||
func isCJKLeakRune(r rune) bool {
|
||||
switch {
|
||||
case unicode.Is(unicode.Han, r):
|
||||
return true
|
||||
case unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r):
|
||||
return true
|
||||
case r >= 0xFF00 && r <= 0xFFEF: // fullwidth & halfwidth forms
|
||||
return true
|
||||
case r >= 0x3001 && r <= 0x303F: // CJK symbols & punctuation, EXCLUDING U+3000 (ideographic space)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail
|
||||
// per distinct run. A single run («却有») is enough to fire — the class is COSMETIC, so the whole
|
||||
// chunk is not lost: stripCosmetic removes the run and the remainder exports (flagged for review).
|
||||
// detectCJKLeak counts maximal runs of leak runes in the final text and returns a deduped detail per
|
||||
// distinct run. It runs over the RECOVERABLE-FOLD of the text (D39 слой 6): a fullwidth «123» or a
|
||||
// «、» is a wrong-glyph rendering the export contract silently fixes, so it is NOT a leak — only a
|
||||
// genuinely contentless Han/kana/CJK-bracket run is. A single run («却有») is enough to fire — the
|
||||
// class is COSMETIC, so the whole chunk is not lost: stripCosmetic removes the run and the remainder
|
||||
// exports (flagged for review).
|
||||
func detectCJKLeak(text string) (int, []string) {
|
||||
var det []string
|
||||
seen := map[string]bool{}
|
||||
n := 0
|
||||
rs := []rune(text)
|
||||
rs := []rune(recoverableFold(text))
|
||||
for i := 0; i < len(rs); {
|
||||
if !isCJKLeakRune(rs[i]) {
|
||||
i++
|
||||
|
|
@ -415,41 +434,98 @@ func detectCJKLeak(text string) (int, []string) {
|
|||
// after the hashes) does NOT match, so it is left intact (and it never fired the class either).
|
||||
var leadingMarkdownStripRE = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+([\p{L}\p{Nd}])`)
|
||||
|
||||
// --- export-contract normalization (D39 слой 6, D29.1a) --------------------------
|
||||
|
||||
// recoverableFold folds the RECOVERABLE wrong-glyph Unicode renderings a Russian final can carry into
|
||||
// their intended characters — the shared primitive of the export contract. It is a WIDTH fold
|
||||
// (width.Fold: fullwidth ASCII «!12» → «!12», the ideographic space U+3000 → a plain space,
|
||||
// halfwidth kana → fullwidth kana) plus the two CJK punctuation glyphs the width fold leaves alone
|
||||
// («、» → «,», «。» → «.»). It reuses the x/text width.Fold primitive instead of the old hand-rolled
|
||||
// «r-0xFEE0» arithmetic (L5-stripcosmetic-duplicates-nfkc) — but deliberately NOT full norm.NFKC,
|
||||
// which would over-fold legitimate Russian typography (NFKC decomposes «…»→«...», «№²½»→…), a
|
||||
// regression the golden strip contract forbids. Used by exportNormalize (every final), stripCosmetic
|
||||
// (before dropping the contentless leak) and detectCJKLeak (which flags only what SURVIVES the fold —
|
||||
// a genuinely contentless Han/kana/CJK-bracket, never a foldable glyph). Pure.
|
||||
func recoverableFold(text string) string {
|
||||
return ideographicPunctFold.Replace(width.Fold.String(text))
|
||||
}
|
||||
|
||||
// ideographicPunctFold maps the two CJK punctuation glyphs the width fold does NOT touch to their
|
||||
// ASCII equivalents (a «、»/«。» in a Russian final is an unambiguous wrong-glyph rendering of «,»/«.»).
|
||||
var ideographicPunctFold = strings.NewReplacer("、", ",", "。", ".")
|
||||
|
||||
// exportNormalize is the export-contract normalization (D29.1a / target arch слой 6) applied to
|
||||
// EVERY final text before it ships (chunkrun.ChunkOutcome.FinalText), clean OR flagged-stripped — the
|
||||
// fix for a cosmetic defect being normalised on a flagged chunk but shipped RAW on a clean one
|
||||
// (L5-normalization-fused-to-flag-path). It folds the recoverable wrong glyphs (recoverableFold),
|
||||
// strips a stray combining stress mark on a Cyrillic base («источа́ло»→«источало»,
|
||||
// L5-diacritics-class-lost), and trims trailing horizontal whitespace per line (never a newline, so
|
||||
// paragraph structure is kept). It is a PURE, IDEMPOTENT projection over the final text: it never
|
||||
// touches the wire, the stored checkpoint or the disposition — only the exported bytes — so it is
|
||||
// NOT a snapshot/verdict input (recomputed each run; a change re-projects existing checkpoints for
|
||||
// free, no re-bill). Deliberately conservative on whitespace: it does NOT collapse internal doubled
|
||||
// spaces (a clean chunk's spacing is left as the model wrote it).
|
||||
func exportNormalize(text string) string {
|
||||
out := recoverableFold(text)
|
||||
// NFC composes a canonically-decomposed base+mark (a «й» stored as и+U+0306, a «ё» as е+U+0308)
|
||||
// BEFORE the combining-strip below, so the strip only removes the NON-composable stress accents
|
||||
// (U+0301 on a vowel), never corrupting «й»/«ё» into «и»/«е». NFC is canonical-only, so it leaves
|
||||
// «…» and other compatibility characters intact (unlike NFKC).
|
||||
out = norm.NFC.String(out)
|
||||
out = stripCombiningOnCyrillic(out)
|
||||
return trailingHSpaceRE.ReplaceAllString(out, "")
|
||||
}
|
||||
|
||||
// stripCombiningOnCyrillic drops a combining mark (unicode.Mn) that sits directly on a CYRILLIC base
|
||||
// — the leaked stress-accent defect the exp12 flagman named («источа́ло» → «источало»,
|
||||
// L5-diacritics-class-lost). Narrow and zero-FP for Russian prose (which never carries combining
|
||||
// stress marks); a combining mark on a Latin base (a foreign name «é») is preserved. Runs AFTER NFKC
|
||||
// (recoverableFold), which has already composed any canonically-composable base+mark.
|
||||
func stripCombiningOnCyrillic(s string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
prevCyrillic := false
|
||||
for _, r := range s {
|
||||
if unicode.Is(unicode.Mn, r) {
|
||||
if prevCyrillic {
|
||||
continue // drop the stress accent on a Cyrillic vowel; leave prevCyrillic set
|
||||
}
|
||||
b.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
b.WriteRune(r)
|
||||
prevCyrillic = unicode.Is(unicode.Cyrillic, r)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// stripCosmetic deterministically removes the two STRIPPABLE cosmetic leak classes — leading
|
||||
// markdown headers and CJK-leak runs — and tidies the whitespace the removal leaves, so the
|
||||
// remainder is readable. It is a PURE function (resume re-derives the identical export), invoked
|
||||
// ONLY on output the sanitizer classified as cosmeticOnly (classifyOutput guarantees the result
|
||||
// is non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
|
||||
// markdown headers and CONTENTLESS CJK-leak runs — and tidies the whitespace the removal leaves, so
|
||||
// the remainder is readable. It is a PURE function (resume re-derives the identical export), invoked
|
||||
// ONLY on output the sanitizer classified as cosmeticOnly (classifyOutput guarantees the result is
|
||||
// non-empty and clean — sanitizeOutput(stripCosmetic(x)).total()==0 — before tagging it
|
||||
// sanitizer_stripped). It never crosses newlines when tidying, so paragraph structure is kept.
|
||||
func stripCosmetic(text string) string {
|
||||
// 1) markdown header prefixes → keep the heading text, drop the «#### » artifact.
|
||||
out := leadingMarkdownStripRE.ReplaceAllString(text, "$1")
|
||||
// 2) neutralise CJK-leak runes. CONTENT-BEARING glyphs are FOLDED, not dropped (adversarial
|
||||
// review MAJOR): a fullwidth ASCII form is a wrong-glyph rendering of a plain char, so «123»
|
||||
// folds to «123» (the number is kept, not deleted) and «,」→«,» — the same NFKC fold the
|
||||
// memory normaliser uses. Only genuinely contentless runes (Han/kana/CJK brackets/symbols) are
|
||||
// removed, and a removed run leaves a SINGLE SPACE so adjacent Russian words do not merge
|
||||
// («нашёл特产древний» → «нашёл древний», not «нашёлдревний»). U+3000 → a plain space (indent).
|
||||
// 2) fold the RECOVERABLE wrong-glyph renderings (fullwidth ASCII «123»→«123», U+3000→space,
|
||||
// «、»→«,» «。»→«.») via the shared export-contract fold — no longer a hand-rolled NFKC
|
||||
// (L5-stripcosmetic-duplicates-nfkc). After this only genuinely contentless CJK remains to drop.
|
||||
out = recoverableFold(out)
|
||||
// 3) drop the contentless CJK-leak runes (Han/kana/CJK brackets & symbols) the fold cannot
|
||||
// recover, each leaving a SINGLE SPACE so adjacent Russian words do not merge («нашёл特产древний»
|
||||
// → «нашёл древний», not «нашёлдревний»).
|
||||
var b strings.Builder
|
||||
b.Grow(len(out))
|
||||
for _, r := range out {
|
||||
switch {
|
||||
case r == ' ': // U+3000 ideographic space → plain space
|
||||
if isCJKLeakRune(r) {
|
||||
b.WriteByte(' ')
|
||||
case r >= 0xFF01 && r <= 0xFF5E: // fullwidth ASCII form → fold to plain ASCII (1→1, A→A, ,→,)
|
||||
b.WriteRune(r - 0xFEE0)
|
||||
case r == '、': // ideographic comma → plain comma (keep the clause boundary)
|
||||
b.WriteByte(',')
|
||||
case r == '。': // ideographic full stop → plain period
|
||||
b.WriteByte('.')
|
||||
case isCJKLeakRune(r): // Han / kana / CJK brackets / other symbol → drop, leaving a space
|
||||
b.WriteByte(' ')
|
||||
default:
|
||||
} else {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
out = b.String()
|
||||
// 3) tidy the seams the removal created, per line (horizontal whitespace only — never a
|
||||
// 4) tidy the seams the removal created, per line (horizontal whitespace only — never a
|
||||
// newline): collapse doubled spaces, drop a bracket/quote pair the strip emptied («(羅漢拳)» →
|
||||
// «()» → removed), drop a space now sitting before closing punctuation or after an opening
|
||||
// bracket/quote (never-correct Russian spacing), and trim trailing blanks. ':' is deliberately
|
||||
|
|
|
|||
|
|
@ -219,8 +219,6 @@ func TestSanitizerCJKLeak(t *testing.T) {
|
|||
"Судзуки нашёл 特产 древнего клана.", // single Han run
|
||||
"Разряд资质乙等 остался неясен.", // Han glued to Cyrillic
|
||||
"Он сказал: すごい, и замолчал.", // kana leak (ja book)
|
||||
"Цена была 123 золотых.", // fullwidth digits
|
||||
"Он вошёл в дом、и дверь закрылась.", // ideographic comma (U+3001)
|
||||
}
|
||||
for _, s := range fire {
|
||||
if got := sanitizeOutput(s); got.CJKLeak == 0 {
|
||||
|
|
@ -233,8 +231,14 @@ func TestSanitizerCJKLeak(t *testing.T) {
|
|||
"На гербе была латинская надпись, но он её не разобрал в темноте.",
|
||||
"Цена была 123 золотых, и он расплатился не торгуясь.", // ASCII digits — not fullwidth
|
||||
// U+3000 ideographic space alone is an INDENT artifact, NOT a content leak → must NOT fire
|
||||
// (research/18 §C#4); stripCosmetic still normalises it, but it does not flag the chunk.
|
||||
// (research/18 §C#4); the export contract normalises it, but it does not flag the chunk.
|
||||
" Судзуки медленно шёл по длинному коридору к башне.",
|
||||
// D39 слой 6: RECOVERABLE wrong-glyph renderings — fullwidth ASCII «123» and the ideographic
|
||||
// comma «、» — no longer FIRE the leak class. The detector is a pure detector over the folded
|
||||
// text (fullwidth «123»→«123», «、»→«,»), and the export contract fixes them silently. Only
|
||||
// genuinely contentless CJK (Han/kana above) is flagged.
|
||||
"Цена была 123 золотых.", // fullwidth digits → silently folded to «123»
|
||||
"Он вошёл в дом、и дверь закрылась.", // ideographic comma (U+3001) → silently folded to «,»
|
||||
}
|
||||
for _, s := range clean {
|
||||
if got := sanitizeOutput(s); got.CJKLeak != 0 {
|
||||
|
|
@ -305,6 +309,45 @@ func TestStripCosmetic(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestExportNormalize pins the export-contract (D39 слой 6, D29.1a): every final text is normalised
|
||||
// before it ships — a CLEAN chunk (which the sanitizer never flags) now gets its cosmetic Unicode
|
||||
// artifacts fixed, not shipped raw (L5-normalization-fused-to-flag-path). Width fold + CJK-punct map
|
||||
// + combining-stress strip; «…»/«№»/«й»/«ё» preserved; idempotent.
|
||||
func TestExportNormalize(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
// The finding's exact case: a clean chunk with only a U+3000 indent shipped raw before.
|
||||
{" Судзуки медленно шёл к башне.", " Судзуки медленно шёл к башне."},
|
||||
// fullwidth ASCII folded (a clean chunk, never flagged) — the number/letters survive.
|
||||
{"Цена была 123 золотых, индекс ABC.", "Цена была 123 золотых, индекс ABC."},
|
||||
// ideographic comma/period → ASCII on a CLEAN chunk (no Han → sanitizer never fired).
|
||||
{"Он вошёл в дом、дверь закрылась。", "Он вошёл в дом,дверь закрылась."},
|
||||
// combining stress accent on a Cyrillic base dropped (L5-diacritics-class-lost): «источа́ло».
|
||||
{"Он источа́ло тепло за́мка.", "Он источало тепло замка."},
|
||||
// legit typography PRESERVED (the NFKC-would-clobber regression the width fold avoids).
|
||||
{"Он ждал… но никто не пришёл. № 5, ½ часа.", "Он ждал… но никто не пришёл. № 5, ½ часа."},
|
||||
// trailing horizontal whitespace trimmed per line; the newline is kept.
|
||||
{"Первая строка. \nВторая строка.", "Первая строка.\nВторая строка."},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := exportNormalize(c.in)
|
||||
if got != c.want {
|
||||
t.Errorf("exportNormalize(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
if again := exportNormalize(got); again != got {
|
||||
t.Errorf("exportNormalize not idempotent for %q: %q → %q", c.in, got, again)
|
||||
}
|
||||
}
|
||||
// A DECOMPOSED «й» (и U+0438 + combining breve U+0306) / «ё» (е U+0435 + diaeresis U+0308) must be
|
||||
// COMPOSED by NFC, NOT corrupted to «и»/«е» by the combining-stress strip — the load-bearing reason
|
||||
// NFC runs BEFORE stripCombiningOnCyrillic in exportNormalize.
|
||||
if got := exportNormalize("\u0438\u0306"); got != "\u0439" { // decomposed й → composed «й»
|
||||
t.Errorf("decomposed й corrupted by the combining strip: %q (want «й» U+0439)", got)
|
||||
}
|
||||
if got := exportNormalize("\u0435\u0308"); got != "\u0451" { // decomposed ё → composed «ё»
|
||||
t.Errorf("decomposed ё corrupted by the combining strip: %q (want «ё» U+0451)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSanitizerDeterministic asserts the verdict is a pure function of the text (resume-safe).
|
||||
func TestSanitizerDeterministic(t *testing.T) {
|
||||
s := "### Заголовок\nВот отредактированный перевод: текст. Стольь странно.\nПримечание: правка."
|
||||
|
|
|
|||
|
|
@ -244,49 +244,29 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
|||
on := fewShotEnabled(st)
|
||||
ss.FewShot = &on
|
||||
}
|
||||
if prov, ok := r.Models.Providers[r.Models.Models[st.Model].Provider]; ok {
|
||||
ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = prov.Temperature, prov.MaxTokens, prov.Model
|
||||
// Fold the PRIMARY model's wire-affecting inputs (prov-triple, extra_body, resolved capability)
|
||||
// via the shared foldModelWire (D39 слой 7, L8-snapshot-fold-copypaste-tripwire): the same
|
||||
// helper folds the escalate_to fallback below, so a third model axis (channel B / annotator)
|
||||
// reuses it and cannot silently drift the snapshot.
|
||||
primary, perr := r.foldModelWire(st.Model)
|
||||
if perr != nil {
|
||||
return "", "", perr
|
||||
}
|
||||
if extra := r.Models.Models[st.Model].ExtraBody; len(extra) > 0 {
|
||||
raw, merr := json.Marshal(extra)
|
||||
if merr != nil {
|
||||
return "", "", merr
|
||||
}
|
||||
ss.ModelExtra = raw
|
||||
}
|
||||
// Резолвнутая каппа — тем же ResolveCapability, что и у клиента, поэтому
|
||||
// хэшируемое всегда совпадает с отправляемым. json.Marshal сортирует
|
||||
// ключи карт (off_extra_body) → детерминированный рендер.
|
||||
capRaw, cerr := json.Marshal(r.Models.ResolveCapability(st.Model))
|
||||
if cerr != nil {
|
||||
return "", "", cerr
|
||||
}
|
||||
ss.Capability = capRaw
|
||||
// Fold the single-hop fallback model + its resolved capability (Веха 2.5):
|
||||
// both are wire-affecting inputs of an escalated call, so changing them is a
|
||||
// loud --resnapshot, not a silent false-hit on an escalated checkpoint.
|
||||
// TRIPWIRE (пакет №4): этот блок — зеркало primary-фолда выше (2×, ниже порога
|
||||
// извлечения ≥3). Когда канал B/annotator добавит ТРЕТЬЮ модельную ось стадии —
|
||||
// сначала извлечь общий foldModelWire(model) (prov-тройка, extra, capability),
|
||||
// потом добавлять ось: копипаста здесь тише всего расходится и портит
|
||||
// идентичность снапшота (false-hit/false-miss на эскалированных чекпоинтах).
|
||||
ss.ProviderTemp, ss.ProviderMaxTok, ss.ProviderModel = primary.provTemp, primary.provMaxTok, primary.provModel
|
||||
ss.ModelExtra = primary.extra
|
||||
ss.Capability = primary.capability
|
||||
// Fold the single-hop fallback model + its wire inputs (Веха 2.5): both are wire-affecting
|
||||
// inputs of an escalated call, so changing them is a loud --resnapshot, not a silent false-hit
|
||||
// on an escalated checkpoint. Same fold as the primary — no copy-paste to drift.
|
||||
if st.EscalateTo != "" {
|
||||
ss.EscalateTo = st.EscalateTo
|
||||
escCap, eerr := json.Marshal(r.Models.ResolveCapability(st.EscalateTo))
|
||||
esc, eerr := r.foldModelWire(st.EscalateTo)
|
||||
if eerr != nil {
|
||||
return "", "", eerr
|
||||
}
|
||||
ss.EscalateCapability = escCap
|
||||
if prov, ok := r.Models.Providers[r.Models.Models[st.EscalateTo].Provider]; ok {
|
||||
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok, ss.EscalateProviderModel = prov.Temperature, prov.MaxTokens, prov.Model
|
||||
}
|
||||
if extra := r.Models.Models[st.EscalateTo].ExtraBody; len(extra) > 0 {
|
||||
raw, merr := json.Marshal(extra)
|
||||
if merr != nil {
|
||||
return "", "", merr
|
||||
}
|
||||
ss.EscalateExtra = raw
|
||||
}
|
||||
ss.EscalateProviderTemp, ss.EscalateProviderMaxTok, ss.EscalateProviderModel = esc.provTemp, esc.provMaxTok, esc.provModel
|
||||
ss.EscalateExtra = esc.extra
|
||||
ss.EscalateCapability = esc.capability
|
||||
}
|
||||
snap.Stages = append(snap.Stages, ss)
|
||||
}
|
||||
|
|
@ -297,3 +277,39 @@ func (r *Runner) snapshotID() (id, payload string, err error) {
|
|||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:]), string(data), nil
|
||||
}
|
||||
|
||||
// modelWireFold is a model's snapshot-folded wire-affecting inputs: its provider-level
|
||||
// temperature/max_tokens/model overrides (local kind, applied AFTER the request-hash), its top-level
|
||||
// extra_body, and its RESOLVED capability (budget-key/temperature-mode/reasoning-control). Extracted
|
||||
// (D39 слой 7, L8-snapshot-fold-copypaste-tripwire) so the primary model, the escalate_to fallback,
|
||||
// and any future third model axis fold the SAME way — the copy-paste the old tripwire warned about.
|
||||
type modelWireFold struct {
|
||||
provTemp float64
|
||||
provMaxTok int
|
||||
provModel string
|
||||
extra json.RawMessage // nil when the model has no extra_body → omitted (omitempty)
|
||||
capability json.RawMessage // always present (ResolveCapability, the same the client sends)
|
||||
}
|
||||
|
||||
// foldModelWire renders one model's wire-fold. ResolveCapability matches exactly what the client
|
||||
// sends, and json.Marshal sorts map keys (off_extra_body) → deterministic render. Returns an error
|
||||
// only on a marshal failure (a struct of scalars/known types cannot fail in practice).
|
||||
func (r *Runner) foldModelWire(model string) (modelWireFold, error) {
|
||||
var f modelWireFold
|
||||
if prov, ok := r.Models.Providers[r.Models.Models[model].Provider]; ok {
|
||||
f.provTemp, f.provMaxTok, f.provModel = prov.Temperature, prov.MaxTokens, prov.Model
|
||||
}
|
||||
if extra := r.Models.Models[model].ExtraBody; len(extra) > 0 {
|
||||
raw, err := json.Marshal(extra)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.extra = raw
|
||||
}
|
||||
capRaw, err := json.Marshal(r.Models.ResolveCapability(model))
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.capability = capRaw
|
||||
return f, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -168,13 +168,15 @@ func (r *Runner) runStage(ctx context.Context, st config.Stage, stageIdx int, sn
|
|||
if disposition == DispOK {
|
||||
finalHash = last.reqHash
|
||||
} else if last.cls.Reason == FlagSanitizerStripped {
|
||||
// Cosmetic leak (D35.4a): strip it and COMMIT the cleaned remainder as a $0 derived export
|
||||
// checkpoint, then point final_hash at it — so the standard final_hash→checkpoint.response_text
|
||||
// export (records.json / exp12_extract) yields the cleaned text instead of empty, while the
|
||||
// chunk stays flagged for a human. classifyOutput guaranteed the strip is non-empty and clean.
|
||||
// The derived checkpoint is durably written BEFORE chunk_status references it (the ok path's
|
||||
// checkpoint-before-pointer discipline), so a resume never dangles final_hash at a missing row.
|
||||
recovered = stripCosmetic(last.text)
|
||||
// Cosmetic leak (D35.4a): COMMIT the cleaned remainder as a $0 derived export checkpoint, then
|
||||
// point final_hash at it — so the standard final_hash→checkpoint.response_text export
|
||||
// (records.json / exp12_extract) yields the cleaned text instead of empty, while the chunk stays
|
||||
// flagged for a human. The strip was computed ONCE by classifyOutput and rides on last.stripped
|
||||
// (T3.4 / L8-stripcosmetic-recomputed-twice): runStage no longer re-derives it, so the exact
|
||||
// bytes the "non-empty & clean" guard validated are what commit. The derived checkpoint is
|
||||
// durably written BEFORE chunk_status references it (the ok path's checkpoint-before-pointer
|
||||
// discipline), so a resume never dangles final_hash at a missing row.
|
||||
recovered = last.stripped
|
||||
dh, err := r.commitSanitizedExport(st, ch, job, last, attemptsMade-1, recovered)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -242,6 +244,11 @@ func (r *Runner) commitSanitizedExport(st config.Stage, ch Chunk, job *store.Job
|
|||
type stageAttempt struct {
|
||||
reqHash string
|
||||
cls classification
|
||||
// stripped is the resolved cosmetic-strip EXPORT text classifyOutput computed ONCE (T3.4):
|
||||
// non-empty only when cls.Reason == FlagSanitizerStripped. runStage ships it verbatim instead of
|
||||
// re-deriving stripCosmetic, so the "non-empty & clean" invariant classifyOutput asserts is the
|
||||
// exact text committed to the derived export checkpoint.
|
||||
stripped string
|
||||
text string
|
||||
usage llm.Usage
|
||||
finish string
|
||||
|
|
@ -279,7 +286,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
|
|||
att.finish = cp.FinishReason
|
||||
att.modelActual = cp.ModelActual
|
||||
att.cumCost = cp.CostUSD
|
||||
att.cls = r.classifyOutput(st.Role, ch.Text, cp.ResponseText, cp.FinishReason, isFinal)
|
||||
att.cls, att.stripped = r.classifyOutput(st.Role, ch.Text, cp.ResponseText, cp.FinishReason, isFinal)
|
||||
rl := r.baseRequestLog(st, ch, model, reqHash)
|
||||
rl.ModelActual = cp.ModelActual
|
||||
rl.TMHit, rl.OK, rl.FinishReason, rl.Degraded = true, att.cls.ok(), cp.FinishReason, degradedTag(att.cls)
|
||||
|
|
@ -445,7 +452,7 @@ func (r *Runner) runAttempt(ctx context.Context, st config.Stage, model, snapID
|
|||
// non-empty truncated length draft is now classified (flagged/retried), never
|
||||
// silently passed downstream as OK; an empty completion is flagged too, not a
|
||||
// run-crash.
|
||||
att.cls = r.classifyOutput(st.Role, ch.Text, resp.Text, resp.FinishReason, isFinal)
|
||||
att.cls, att.stripped = r.classifyOutput(st.Role, ch.Text, resp.Text, resp.FinishReason, isFinal)
|
||||
|
||||
rl := r.baseRequestLog(st, ch, model, reqHash)
|
||||
rl.ModelActual = modelActual
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -74,6 +74,11 @@ type RetrievalState struct {
|
|||
InjectedIDs string // JSON
|
||||
NStyleFlags int // cheap style/number gate hits (dialogue-dash, ё, translit-interj, 万/億)
|
||||
StyleDetail string // JSON breakdown of the style-flag hits
|
||||
// NTrustGatedSuppress counts refused lower-trust longest-match suppressions (D39 слой 4): a
|
||||
// longer draft/ambiguous key blocked from eating a nested approved/confirmed one. A nonzero
|
||||
// count is a loud seed-hygiene signal (a draft term nesting over an approved term to reconcile).
|
||||
NTrustGatedSuppress int
|
||||
TrustGateDetail string // JSON of the refused suppressor→protected pairs
|
||||
}
|
||||
|
||||
// ReplaceGlossary replaces a book's ENTIRE glossary (rows + aliases) with the given
|
||||
|
|
@ -239,8 +244,8 @@ func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
|
|||
book_id, chapter, chunk_idx, snapshot_id,
|
||||
n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked, n_evicted,
|
||||
embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
||||
n_style_flags, style_detail, updated_at
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
|
||||
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail, updated_at
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,datetime('now'))
|
||||
ON CONFLICT (book_id, chapter, chunk_idx) DO UPDATE SET
|
||||
snapshot_id = excluded.snapshot_id,
|
||||
n_exact_hits = excluded.n_exact_hits,
|
||||
|
|
@ -254,11 +259,13 @@ func (s *Store) UpsertRetrievalState(rs RetrievalState) error {
|
|||
injected_ids = excluded.injected_ids,
|
||||
n_style_flags = excluded.n_style_flags,
|
||||
style_detail = excluded.style_detail,
|
||||
n_trust_gated_suppress = excluded.n_trust_gated_suppress,
|
||||
trust_gate_detail = excluded.trust_gate_detail,
|
||||
updated_at = excluded.updated_at`,
|
||||
rs.BookID, rs.Chapter, rs.ChunkIdx, rs.SnapshotID,
|
||||
rs.NExactHits, rs.NSticky, rs.NAmbiguousFlagged, rs.NSpoilerBlocked, rs.NEvicted,
|
||||
rs.EmbeddingTierUsed, rs.NPostcheckMiss, rs.PostcheckDetail, rs.InjectedIDs,
|
||||
rs.NStyleFlags, rs.StyleDetail)
|
||||
rs.NStyleFlags, rs.StyleDetail, rs.NTrustGatedSuppress, rs.TrustGateDetail)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -270,12 +277,12 @@ func (s *Store) GetRetrievalState(bookID string, chapter, chunkIdx int) (*Retrie
|
|||
err := s.r.QueryRowContext(ctx, `
|
||||
SELECT snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged, n_spoiler_blocked,
|
||||
n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
||||
n_style_flags, style_detail
|
||||
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail
|
||||
FROM retrieval_state WHERE book_id = ? AND chapter = ? AND chunk_idx = ?`,
|
||||
bookID, chapter, chunkIdx).Scan(
|
||||
&rs.SnapshotID, &rs.NExactHits, &rs.NSticky, &rs.NAmbiguousFlagged, &rs.NSpoilerBlocked,
|
||||
&rs.NEvicted, &rs.EmbeddingTierUsed, &rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
|
||||
&rs.NStyleFlags, &rs.StyleDetail)
|
||||
&rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
@ -291,7 +298,7 @@ func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error)
|
|||
return queryAll(s.r, `
|
||||
SELECT chapter, chunk_idx, snapshot_id, n_exact_hits, n_sticky, n_ambiguous_flagged,
|
||||
n_spoiler_blocked, n_evicted, embedding_tier_used, n_postcheck_miss, postcheck_detail, injected_ids,
|
||||
n_style_flags, style_detail
|
||||
n_style_flags, style_detail, n_trust_gated_suppress, trust_gate_detail
|
||||
FROM retrieval_state WHERE book_id = ?
|
||||
ORDER BY chapter, chunk_idx`,
|
||||
func(rows *sql.Rows) (RetrievalState, error) {
|
||||
|
|
@ -299,7 +306,7 @@ func (s *Store) RetrievalStatesForBook(bookID string) ([]RetrievalState, error)
|
|||
err := rows.Scan(&rs.Chapter, &rs.ChunkIdx, &rs.SnapshotID, &rs.NExactHits, &rs.NSticky,
|
||||
&rs.NAmbiguousFlagged, &rs.NSpoilerBlocked, &rs.NEvicted, &rs.EmbeddingTierUsed,
|
||||
&rs.NPostcheckMiss, &rs.PostcheckDetail, &rs.InjectedIDs,
|
||||
&rs.NStyleFlags, &rs.StyleDetail)
|
||||
&rs.NStyleFlags, &rs.StyleDetail, &rs.NTrustGatedSuppress, &rs.TrustGateDetail)
|
||||
return rs, err
|
||||
}, bookID)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -272,6 +272,16 @@ var migrations = []string{
|
|||
ALTER TABLE retrieval_state ADD COLUMN n_style_flags INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE retrieval_state ADD COLUMN style_detail TEXT NOT NULL DEFAULT '';
|
||||
`,
|
||||
// v8 (D39 слой 4): the disposition-gated suppressor's LOUD record. n_trust_gated_suppress counts
|
||||
// the times a longer but LOWER-trust glossary key (draft/ambiguous) was REFUSED from suppressing a
|
||||
// nested HIGHER-trust key (approved/confirmed) — the term-drift code root, converted from a silent
|
||||
// drop into a visible signal (research/13 §7). A nonzero count means the seed has a draft term
|
||||
// nesting over an approved one to reconcile. trust_gate_detail is the JSON of the refused pairs.
|
||||
// Recomputed deterministically each run (self-heals on resume), NOT wire-load-bearing.
|
||||
`
|
||||
ALTER TABLE retrieval_state ADD COLUMN n_trust_gated_suppress INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE retrieval_state ADD COLUMN trust_gate_detail TEXT NOT NULL DEFAULT '';
|
||||
`,
|
||||
}
|
||||
|
||||
// DESIGN NOTE (D21.10 — reserved memory-bank v2 record types; Ф2, NOT a migration, NOT code).
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue