package config import ( "fmt" "os" "path/filepath" "sort" "strings" "gopkg.in/yaml.v3" ) // pipeline.go loads the pipeline-core config (C1/C2… — Р2). Граница «конфиг vs // код» зафиксирована: конфиг задаёт состав/порядок стадий, роль→модель, версии // промптов, пороги гейтов, режим инъекции глоссария, эскалационные цепочки, // fan-out N, токен-бюджеты сборки контекста и cache TTL. Циклы по // главам/чанкам, ветвление по гейтам, механика эскалации/ретраев и правило // «эскалация → re-gate → флаг» зашиты в раннер. // Pipeline is the parsed pipeline-*.yaml. type Pipeline struct { Core string `yaml:"core"` // C0|C1|C2|C3 Version int `yaml:"version"` Defaults PipelineDefaults `yaml:"defaults"` Context ContextAssembly `yaml:"context"` Retries Retries `yaml:"retries"` Stages []Stage `yaml:"stages"` Gates Gates `yaml:"gates"` Escal Escalation `yaml:"escalation"` Fanout Fanout `yaml:"fanout"` } // PipelineDefaults are cross-stage knobs. type PipelineDefaults struct { // MaxOutputRatio sizes max_tokens ≈ ratio × input tokens (токен-калибровка // полигона: русский выход zh→ru ≈1.9× входа; дефолт с запасом). MaxOutputRatio float64 `yaml:"max_output_ratio"` MinMaxTokens int `yaml:"min_max_tokens"` } // ContextAssembly holds the prompt-layout budgets (Р5-раскладка: стабильный // префикс / волатильный хвост — сборка Фазы 1; поля фиксируются сейчас, чтобы // схема конфига не менялась под ногами «Редакции»). type ContextAssembly struct { GlossaryInjection string `yaml:"glossary_injection"` // selective | full_prefix (Р5: обе схемы) GlossaryTokenBudget int `yaml:"glossary_token_budget"` STMDepth int `yaml:"stm_depth"` OverlapTokens int `yaml:"overlap_tokens"` CacheTTL string `yaml:"cache_ttl"` // per-stage TTL override — Фаза 1 } // Retries distinguishes the CONTENT retry (регенерация после провала гейта до // эскалации) from transport retries (профиль в models.yaml) — спека их // различает явно (валидация, линза 2 п.5). type Retries struct { RegenerateBeforeEscalate int `yaml:"regenerate_before_escalate"` } // Stage is one pipeline pass. type Stage struct { Name string `yaml:"name"` Role string `yaml:"role"` Model string `yaml:"model"` // Prompt is the LEGACY single, pair-AGNOSTIC template path (kept for tests / simple configs). // Prompts is the pair-KEYED convention pack (D39 слой 2): src→tgt → template path, resolved by the // book's LangPair() at load with a FAIL-LOUD on a missing pair (never a silent fall-through to // another pair's conventions — the latent bug where a ja book rode the zh-parataxis prompt, // L4-prompts-not-per-pair-zh-baked). The pack is where a per-pair prompt (and its LANGUAGE — a // property of the pack, not a hardcode) lives; today only zh-ru is populated. Exactly ONE of // Prompt/Prompts per stage (validated in LoadPipeline). The pair→prompt binding is snapshot-folded // transitively: the resolved template's content rides PromptSHA256 and the book's pair rides // BriefHash (source_lang/target_lang), so a pair or content change is a loud --resnapshot. Prompt string `yaml:"prompt"` Prompts map[string]string `yaml:"prompts"` PromptVersion string `yaml:"prompt_version"` Temperature float64 `yaml:"temperature"` Reasoning string `yaml:"reasoning"` // "", off, low, medium, high // ReasoningMaxTokens is the explicit reasoning-token BUFFER reserved for this stage // when it thinks (reasoning low|medium|high) on an ADDITIVE-billing provider (xAI — // reasoning bills on top of completion, D13.6). Required (>0) in exactly that case // (fail-fast in LoadPipeline); ignored for subset providers / reasoning-off. It only // sizes the reservation (EstimateUSD), so it is NOT part of the snapshot/wire. ReasoningMaxTokens int `yaml:"reasoning_max_tokens"` // EscalateTo is the SINGLE-HOP fallback model tried ONCE when this stage's // output is a deterministic content-failure another model might fix (echo / // excision / refusal — D12). Empty = no escalation (e.g. the editor is pinned // per book — its style must not drift to a foreign model, D12/2605.13368). The // fallback is its OWN model, which is the request_hash axis (a distinct // checkpoint, never the failed primary's). Gated by escalation.budget_usd. EscalateTo string `yaml:"escalate_to"` // Channel selects the safety channel: "" | sfw (default) | adult (18+). An // adult stage is isolated by TYPE (D4.1): its model AND its escalate_to must be // on permissive providers, enforced at load — never a fall-through to a refusing // SFW provider. Channel string `yaml:"channel"` // FewShot switches the prompt's optional ---FEWSHOT--- example section on/off (D38.4). // nil (absent) = ON: the examples are appended to the system prompt (the P1a discourse // default). false = zero-shot: the ---FEWSHOT--- block is dropped, leaving only the core // instructions — for a reasoning model whose own CoT is disrupted by hand-written examples // (deepseek-thinking swap-arm, exp14 §2а). A no-op for a prompt with no ---FEWSHOT--- section. // Wire-affecting (it changes the system message), so it is folded into the snapshot: a flip // is a loud --resnapshot, never a silent false-hit. FewShot *bool `yaml:"few_shot"` } // Gates is the QA-gate config skeleton (пороги полигона; исполнение — Фаза 1). type Gates struct { Coverage CoverageGate `yaml:"coverage"` Glossary GlossaryGate `yaml:"glossary"` Sanitizer SanitizerGate `yaml:"sanitizer"` RegressionGuard RegressionGuardGate `yaml:"regression_guard"` } // RegressionGuardGate controls the post-reflow regression guard (D38 infra-pack, // research/18 §C#5): two deterministic OBSERVABILITY flaggers over the draft→final transform — // a length collapse and a numeric drift — that surface a reflow that dropped content or drifted a // number (四成四=44%→«четыре десятых»). Opt-in (default false), and by design NEVER a disposition // change: the reflow editor legitimately restructures/merges, so a hard skip would false-flag a // good edit — a hit is recorded in the cheap-gate observability channel (retrieval-state // n_style_flags) and surfaced in the report, never dropping the chunk. Its thresholds are code // consts (versioned with the cheap gates), so the gate carries only an on/off switch. type RegressionGuardGate struct { Enabled bool `yaml:"enabled"` } // SanitizerGate controls the output-sanitizer (D30.3): a deterministic verdict-axis // gate on the FINAL chunk text that flags "instant unreadability" defects no other gate // catches — leaked service preambles, trailing note/edit blocks, markdown ### headers, // Latin-script insertions in the Russian output, and broken/split word forms (exp12 / // flagman §5). Opt-in (default false), following the coverage gate's discipline (D12 Q4): // when enabled a defect flips the chunk to flagged (D2 flag+skip — the garbage output // never commits to TM/export). Its rules are tuned PRECISION over recall (each class fires // only on a high-confidence signal), so a premature always-on default would false-flag // legitimate prose. The rule VERSION is folded into the snapshot only when enabled // (sanitizerVersion, mirroring coverage), so a rule edit is a loud --resnapshot — and it // moves to verdictSnapshotID once content-addressed resume lands (D15.2). type SanitizerGate struct { Enabled bool `yaml:"enabled"` } // GlossaryGate controls the memory-bank post-check (E1). The post-check ALWAYS runs // (observability — it records misses into the retrieval-state), converting silent // glossary drift into a loud, visible signal. PostcheckGate promotes a miss from a // mere record to a DISPOSITION flag (the chunk is flagged, downstream stages skip): // opt-in (default false = flagger), flipped on only AFTER the owner validates the // false-flag rate on real chapters (E1) — exactly the coverage gate's opt-in discipline // (D12 Q4). A naive/under-filled decl false-flags (research/14 §2), so a premature hard // gate would flag-storm and train editors to ignore it. type GlossaryGate struct { PostcheckGate bool `yaml:"postcheck_gate"` } // CoverageGate v1 (Р7): метрика — символы без пробелов; нижняя граница — // подозрение на вырезание, верхняя — аномалия; пороги из эксперимента 02. type CoverageGate struct { Enabled bool `yaml:"enabled"` LenRatio map[string][]float64 `yaml:"len_ratio_bounds"` // "zh-ru": [low, high] SentCovMin float64 `yaml:"sent_cov_min"` MinChunkChars int `yaml:"min_chunk_chars"` } // Escalation holds named model chains; channel-aware filtering (Anthropic вне // 18+ и т.д.) применяет раннер, не конфиг. type Escalation struct { Chains map[string][]string `yaml:"chains"` BudgetUSD float64 `yaml:"budget_usd"` // адресный премиум-бюджет книги; формула — [НУЖНО РЕШЕНИЕ] п.1 } // Fanout is the C2 skeleton: N候補 candidates per chunk (fan-out — механика // раннера, N — конфиг). type Fanout struct { Candidates int `yaml:"candidates"` } // PromptPathFor resolves the effective prompt template path for this stage given the book's language // pair (D39 слой 2). A pair-keyed `prompts` pack resolves the entry for `pair`, failing LOUD if the // pair is absent (never a silent fall-through to another pair's conventions — the latent bug where a // ja book rode the zh-parataxis prompt). A legacy single `prompt` is pair-agnostic (returned as-is). // Callers pass Book.LangPair(). Paths are already resolved to absolute in LoadPipeline. func (st Stage) PromptPathFor(pair string) (string, error) { if len(st.Prompts) == 0 { return st.Prompt, nil } if p, ok := st.Prompts[pair]; ok { return p, nil } have := make([]string, 0, len(st.Prompts)) for k := range st.Prompts { have = append(have, k) } sort.Strings(have) return "", fmt.Errorf("stage %q: no prompt for language pair %q — the pair-keyed pack has [%s]; add a %q prompt (conventions authored in the right language) or fix the book's source_lang/target_lang (D39 слой 2: никогда молча не подставляем чужую пару)", st.Name, pair, strings.Join(have, ", "), pair) } // CheckRunnable rejects a config whose mechanics the Phase-0 runner does NOT // implement — раздельно от schema-валидации (LoadPipeline), чтобы скелеты C2/C3 // парсились и валидировались как схема, но НЕ исполнялись молча. Без этого // гейта запуск pipeline-c2.yaml прогнал бы стадии линейно: fanout.candidates // игнорируется, вердикт судьи утёк бы в редактуру, а деньги Opus сгорели бы на // мусор (находка ревью; Р2 требует закрывать дыру «конфиг vs код» fail-loud). func (p *Pipeline) CheckRunnable() error { switch p.Core { case "C0", "C1": default: return fmt.Errorf("pipeline core %q не исполним в Фазе 0 (реализованы только C0/C1 — линейный проход стадий; C2/C3 selection/fusion — механика раннера Фазы 2)", p.Core) } if p.Fanout.Candidates > 1 { return fmt.Errorf("pipeline fanout.candidates=%d не исполним в Фазе 0 (fan-out N кандидатов — механика раннера Фазы 2; C1 = один черновик)", p.Fanout.Candidates) } // Coverage QA-гейт исполним с Вехи 2.5 (excision-детект, coverage.go): раннер // считает его по результату стадии и эмитит excision_suspect. Бывший fail-loud // на gates.enabled снят; корректность порогов включённого гейта проверяет // LoadPipeline (иначе тихий no-op — против Р7). for _, st := range p.Stages { if st.Role == "judge" { return fmt.Errorf("pipeline stage %q role=judge не исполним в Фазе 0 (селектор получает N кандидатов — механика Фазы 2, а раннер гонит стадии линейно)", st.Name) } } return nil } // CheckAdultChannel enforces the D20.2-Q3 load-time consistency lint: a book marked // adult:true MUST route through at least one channel:adult stage, else the 18+ text would // silently run on the SFW pipeline (all-SFW providers that refuse/excise explicit content). // Adult stays wire/verdict-NEUTRAL (no snapshot layer — D20.2-Q3: the first runtime consumer // assigns meaning; here it is only a routing sanity check, never a hash input). The REVERSE // (a channel:adult stage in an adult:false book) is deliberately NOT gated: the stage's // channel already forces a permissive provider (D4.1), and Adult carries no assigned meaning // yet, so a permissive stage may legitimately exist before the book flag is set. func (p *Pipeline) CheckAdultChannel(bookAdult bool) error { if !bookAdult { return nil } for _, st := range p.Stages { if st.Channel == "adult" { return nil } } return fmt.Errorf("book adult:true, но ни одна стадия не имеет channel:adult — 18+ текст пошёл бы по SFW-каналу (SFW-провайдеры отказывают/вырезают explicit); заведите channel:adult-стадию на permissive-провайдере (D4.1/D20.2-Q3) либо поставьте adult:false") } // LoadPipeline reads and validates a pipeline config against the models known // to models.yaml. func LoadPipeline(path string, models *Models) (*Pipeline, error) { raw, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("config: read %s: %w", path, err) } var p Pipeline if err := yaml.Unmarshal(raw, &p); err != nil { return nil, fmt.Errorf("config: parse %s: %w", path, err) } // Prompt paths resolve relative to the pipeline file's directory, so a // project works from any CWD. dir := filepath.Dir(path) resolvePrompt := func(pr string) string { if pr != "" && !filepath.IsAbs(pr) { return filepath.Join(dir, pr) } return pr } for i := range p.Stages { p.Stages[i].Prompt = resolvePrompt(p.Stages[i].Prompt) for pair, pr := range p.Stages[i].Prompts { p.Stages[i].Prompts[pair] = resolvePrompt(pr) } } var problems []string bad := func(format string, a ...any) { problems = append(problems, fmt.Sprintf(format, a...)) } if p.Core == "" { bad("core is required (C0|C1|C2|C3)") } if len(p.Stages) == 0 { bad("at least one stage is required") } if p.Defaults.MaxOutputRatio <= 0 { p.Defaults.MaxOutputRatio = 2.0 } if p.Defaults.MinMaxTokens <= 0 { p.Defaults.MinMaxTokens = 2048 } if p.Fanout.Candidates <= 0 { p.Fanout.Candidates = 1 } switch p.Context.GlossaryInjection { case "", "selective", "full_prefix": default: bad("context.glossary_injection must be selective|full_prefix, got %q", p.Context.GlossaryInjection) } seen := map[string]bool{} for i, st := range p.Stages { if st.Name == "" { bad("stage %d: name is required", i) } if seen[st.Name] { bad("stage %q: duplicate name", st.Name) } seen[st.Name] = true if st.Role == "" { bad("stage %q: role is required", st.Name) } if st.PromptVersion == "" { bad("stage %q: prompt_version is required (версионирование промптов — Р2)", st.Name) } if _, ok := models.Models[st.Model]; !ok { bad("stage %q: model %q is not defined in models.yaml", st.Name, st.Model) } // Prompt (legacy single, pair-agnostic) XOR Prompts (pair-keyed pack, D39 слой 2) — exactly one. switch { case st.Prompt != "" && len(st.Prompts) > 0: bad("stage %q: set either `prompt` (single) OR `prompts` (pair-keyed pack), not both", st.Name) case st.Prompt == "" && len(st.Prompts) == 0: bad("stage %q: a prompt is required — `prompt` (single) or a pair-keyed `prompts` pack (D39 слой 2)", st.Name) case st.Prompt != "": if _, err := os.Stat(st.Prompt); err != nil { bad("stage %q: prompt template %s is not readable: %v", st.Name, st.Prompt, err) } default: // pair-keyed pack: every declared pair's template must be readable (deterministic order) pairs := make([]string, 0, len(st.Prompts)) for pair := range st.Prompts { pairs = append(pairs, pair) } sort.Strings(pairs) for _, pair := range pairs { pr := st.Prompts[pair] if pr == "" { bad("stage %q: prompt pack pair %q has an empty path", st.Name, pair) } else if _, err := os.Stat(pr); err != nil { bad("stage %q: prompt pack %q → %s is not readable: %v", st.Name, pair, pr, err) } } } switch st.Reasoning { case "", "off", "low", "medium", "high": default: bad("stage %q: reasoning must be off|low|medium|high, got %q", st.Name, st.Reasoning) } // D13.6: on an ADDITIVE-billing provider (xAI — reasoning bills ON TOP of completion) a // stage that MAY think needs a reserved reasoning_max_tokens buffer, else the reservation // under-budgets and the spend ceiling is blind to the overshoot pricing.go only // acknowledges. "May think" = any effort EXCEPT an explicit reasoning:"off" (which sends // the provider's disable — off_effort:"none" for grok): low|medium|high think, and an // OMITTED/"" effort falls to the provider DEFAULT, which for xAI-grok is "low" (thinking // ON, additive) — so "" must NOT escape the gate (self-review finding #4: the earlier // thinks-only check let it through). Subset providers never need a buffer. This is the // structural gate D6.2 requires before think-ON is allowed on grok (the contrast the task // draws with the echo mine). Checks the primary AND the escalate_to model (same effort). if st.Reasoning != "off" && st.ReasoningMaxTokens <= 0 { if models.providerReasoning(st.Model) == "additive" { bad("stage %q: reasoning=%q on additive-billing provider (model %q) requires reasoning_max_tokens>0 — xAI bills reasoning on top of completion and thinks by default unless reasoning is explicitly \"off\"; without a reserved buffer the spend ceiling is blind to the overshoot (D6.2/D13.6)", st.Name, st.Reasoning, st.Model) } if st.EscalateTo != "" && models.providerReasoning(st.EscalateTo) == "additive" { bad("stage %q: reasoning=%q with escalate_to on an additive-billing provider (model %q) requires reasoning_max_tokens>0 (D6.2/D13.6)", st.Name, st.Reasoning, st.EscalateTo) } } switch st.Channel { case "", "sfw", "adult": default: bad("stage %q: channel must be sfw|adult, got %q", st.Name, st.Channel) } if st.EscalateTo != "" { if _, ok := models.Models[st.EscalateTo]; !ok { bad("stage %q: escalate_to model %q is not defined in models.yaml", st.Name, st.EscalateTo) } else if st.EscalateTo == st.Model { bad("stage %q: escalate_to must differ from the primary model %q (a same-model hop is a guaranteed repeat)", st.Name, st.Model) } // D12 editor-pinned, enforced structurally: only the translator role may // fall back to another model. An editor/other stage that escalated would // drift its style/terms to a foreign model (2605.13368) — forbid it at load. if st.Role != "translator" { bad("stage %q: escalate_to is only allowed on a translator role (D12 editor-pinned — a %q stage must not fall back to a foreign model)", st.Name, st.Role) } } // D4.1: channel-B (18+) isolation is enforced by TYPE — a permissive stage // may only run on, and escalate to, a permissive provider. if st.Channel == "adult" { if !models.providerPermissive(st.Model) { bad("stage %q: channel=adult requires model %q on a permissive provider (D4.1)", st.Name, st.Model) } if st.EscalateTo != "" && !models.providerPermissive(st.EscalateTo) { bad("stage %q: channel=adult escalate_to %q must be on a permissive provider (D4.1 — no fall-through to a refusing SFW channel)", st.Name, st.EscalateTo) } } } for chain, ms := range p.Escal.Chains { for _, m := range ms { if _, ok := models.Models[m]; !ok { bad("escalation chain %q: model %q is not defined in models.yaml", chain, m) } } } // Coverage QA-gate thresholds (шаг 6): an ENABLED gate must be able to flag // something, otherwise it is a silent no-op that passes every chunk (against Р7). if cov := p.Gates.Coverage; cov.Enabled { // sent_cov_min must be a LIVE threshold in (0,1]: it is the pair-INDEPENDENT // half of the gate, so requiring it guarantees the gate can never silently pass // every chunk for a language pair that lacks a len_ratio corridor (self-review: // the old "both empty" check missed the per-pair gap — a non-empty len_ratio map // that just lacks the book's pair, with sent_cov_min=0, passed everything). if cov.SentCovMin <= 0 || cov.SentCovMin > 1 { bad("gates.coverage.sent_cov_min must be within (0, 1] when the gate is enabled, got %v (else the gate silently passes any language pair without a len_ratio corridor)", cov.SentCovMin) } for pair, b := range cov.LenRatio { if len(b) != 2 || b[0] <= 0 || b[1] < b[0] { bad("gates.coverage.len_ratio_bounds[%q] must be [low, high] with 0 < low <= high, got %v", pair, b) } } } if len(problems) > 0 { return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - ")) } return &p, nil }