363 lines
19 KiB
Go
363 lines
19 KiB
Go
package config
|
||
|
||
import (
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
|
||
"gopkg.in/yaml.v3"
|
||
)
|
||
|
||
// pipeline.go loads the pipeline-core config (C1/C2… — Р2). Граница «конфиг vs
|
||
// код» зафиксирована: конфиг задаёт состав/порядок стадий, роль→модель, версии
|
||
// промптов, пороги гейтов, режим инъекции глоссария, эскалационные цепочки,
|
||
// fan-out N, токен-бюджеты сборки контекста и cache TTL. Циклы по
|
||
// главам/чанкам, ветвление по гейтам, механика эскалации/ретраев и правило
|
||
// «эскалация → re-gate → флаг» зашиты в раннер.
|
||
|
||
// Pipeline is the parsed pipeline-*.yaml.
|
||
type Pipeline struct {
|
||
Core string `yaml:"core"` // C0|C1|C2|C3
|
||
Version int `yaml:"version"`
|
||
Defaults PipelineDefaults `yaml:"defaults"`
|
||
Context ContextAssembly `yaml:"context"`
|
||
Retries Retries `yaml:"retries"`
|
||
Stages []Stage `yaml:"stages"`
|
||
Gates Gates `yaml:"gates"`
|
||
Escal Escalation `yaml:"escalation"`
|
||
Fanout Fanout `yaml:"fanout"`
|
||
}
|
||
|
||
// PipelineDefaults are cross-stage knobs.
|
||
type PipelineDefaults struct {
|
||
// MaxOutputRatio sizes max_tokens ≈ ratio × input tokens (токен-калибровка
|
||
// полигона: русский выход zh→ru ≈1.9× входа; дефолт с запасом).
|
||
MaxOutputRatio float64 `yaml:"max_output_ratio"`
|
||
MinMaxTokens int `yaml:"min_max_tokens"`
|
||
}
|
||
|
||
// ContextAssembly holds the prompt-layout budgets (Р5-раскладка: стабильный
|
||
// префикс / волатильный хвост — сборка Фазы 1; поля фиксируются сейчас, чтобы
|
||
// схема конфига не менялась под ногами «Редакции»).
|
||
type ContextAssembly struct {
|
||
GlossaryInjection string `yaml:"glossary_injection"` // selective | full_prefix (Р5: обе схемы)
|
||
GlossaryTokenBudget int `yaml:"glossary_token_budget"`
|
||
STMDepth int `yaml:"stm_depth"`
|
||
OverlapTokens int `yaml:"overlap_tokens"`
|
||
CacheTTL string `yaml:"cache_ttl"` // per-stage TTL override — Фаза 1
|
||
}
|
||
|
||
// Retries distinguishes the CONTENT retry (регенерация после провала гейта до
|
||
// эскалации) from transport retries (профиль в models.yaml) — спека их
|
||
// различает явно (валидация, линза 2 п.5).
|
||
type Retries struct {
|
||
RegenerateBeforeEscalate int `yaml:"regenerate_before_escalate"`
|
||
}
|
||
|
||
// Stage is one pipeline pass.
|
||
type Stage struct {
|
||
Name string `yaml:"name"`
|
||
Role string `yaml:"role"`
|
||
Model string `yaml:"model"`
|
||
Prompt string `yaml:"prompt"` // path to the versioned template file
|
||
PromptVersion string `yaml:"prompt_version"`
|
||
Temperature float64 `yaml:"temperature"`
|
||
Reasoning string `yaml:"reasoning"` // "", off, low, medium, high
|
||
// ReasoningMaxTokens is the explicit reasoning-token BUFFER reserved for this stage
|
||
// when it thinks (reasoning low|medium|high) on an ADDITIVE-billing provider (xAI —
|
||
// reasoning bills on top of completion, D13.6). Required (>0) in exactly that case
|
||
// (fail-fast in LoadPipeline); ignored for subset providers / reasoning-off. It only
|
||
// sizes the reservation (EstimateUSD), so it is NOT part of the snapshot/wire.
|
||
ReasoningMaxTokens int `yaml:"reasoning_max_tokens"`
|
||
// EscalateTo is the SINGLE-HOP fallback model tried ONCE when this stage's
|
||
// output is a deterministic content-failure another model might fix (echo /
|
||
// excision / refusal — D12). Empty = no escalation (e.g. the editor is pinned
|
||
// per book — its style must not drift to a foreign model, D12/2605.13368). The
|
||
// fallback is its OWN model, which is the request_hash axis (a distinct
|
||
// checkpoint, never the failed primary's). Gated by escalation.budget_usd.
|
||
EscalateTo string `yaml:"escalate_to"`
|
||
// Channel selects the safety channel: "" | sfw (default) | adult (18+). An
|
||
// adult stage is isolated by TYPE (D4.1): its model AND its escalate_to must be
|
||
// on permissive providers, enforced at load — never a fall-through to a refusing
|
||
// SFW provider.
|
||
Channel string `yaml:"channel"`
|
||
// FewShot switches the prompt's optional ---FEWSHOT--- example section on/off (D38.4).
|
||
// nil (absent) = ON: the examples are appended to the system prompt (the P1a discourse
|
||
// default). false = zero-shot: the ---FEWSHOT--- block is dropped, leaving only the core
|
||
// instructions — for a reasoning model whose own CoT is disrupted by hand-written examples
|
||
// (deepseek-thinking swap-arm, exp14 §2а). A no-op for a prompt with no ---FEWSHOT--- section.
|
||
// Wire-affecting (it changes the system message), so it is folded into the snapshot: a flip
|
||
// is a loud --resnapshot, never a silent false-hit.
|
||
FewShot *bool `yaml:"few_shot"`
|
||
}
|
||
|
||
// Gates is the QA-gate config skeleton (пороги полигона; исполнение — Фаза 1).
|
||
type Gates struct {
|
||
Coverage CoverageGate `yaml:"coverage"`
|
||
Glossary GlossaryGate `yaml:"glossary"`
|
||
Sanitizer SanitizerGate `yaml:"sanitizer"`
|
||
RegressionGuard RegressionGuardGate `yaml:"regression_guard"`
|
||
}
|
||
|
||
// RegressionGuardGate controls the post-reflow regression guard (D38 infra-pack,
|
||
// research/18 §C#5): two deterministic OBSERVABILITY flaggers over the draft→final transform —
|
||
// a length collapse and a numeric drift — that surface a reflow that dropped content or drifted a
|
||
// number (四成四=44%→«четыре десятых»). Opt-in (default false), and by design NEVER a disposition
|
||
// change: the reflow editor legitimately restructures/merges, so a hard skip would false-flag a
|
||
// good edit — a hit is recorded in the cheap-gate observability channel (retrieval-state
|
||
// n_style_flags) and surfaced in the report, never dropping the chunk. Its thresholds are code
|
||
// consts (versioned with the cheap gates), so the gate carries only an on/off switch.
|
||
type RegressionGuardGate struct {
|
||
Enabled bool `yaml:"enabled"`
|
||
}
|
||
|
||
// SanitizerGate controls the output-sanitizer (D30.3): a deterministic verdict-axis
|
||
// gate on the FINAL chunk text that flags "instant unreadability" defects no other gate
|
||
// catches — leaked service preambles, trailing note/edit blocks, markdown ### headers,
|
||
// Latin-script insertions in the Russian output, and broken/split word forms (exp12 /
|
||
// flagman §5). Opt-in (default false), following the coverage gate's discipline (D12 Q4):
|
||
// when enabled a defect flips the chunk to flagged (D2 flag+skip — the garbage output
|
||
// never commits to TM/export). Its rules are tuned PRECISION over recall (each class fires
|
||
// only on a high-confidence signal), so a premature always-on default would false-flag
|
||
// legitimate prose. The rule VERSION is folded into the snapshot only when enabled
|
||
// (sanitizerVersion, mirroring coverage), so a rule edit is a loud --resnapshot — and it
|
||
// moves to verdictSnapshotID once content-addressed resume lands (D15.2).
|
||
type SanitizerGate struct {
|
||
Enabled bool `yaml:"enabled"`
|
||
}
|
||
|
||
// GlossaryGate controls the memory-bank post-check (E1). The post-check ALWAYS runs
|
||
// (observability — it records misses into the retrieval-state), converting silent
|
||
// glossary drift into a loud, visible signal. PostcheckGate promotes a miss from a
|
||
// mere record to a DISPOSITION flag (the chunk is flagged, downstream stages skip):
|
||
// opt-in (default false = flagger), flipped on only AFTER the owner validates the
|
||
// false-flag rate on real chapters (E1) — exactly the coverage gate's opt-in discipline
|
||
// (D12 Q4). A naive/under-filled decl false-flags (research/14 §2), so a premature hard
|
||
// gate would flag-storm and train editors to ignore it.
|
||
type GlossaryGate struct {
|
||
PostcheckGate bool `yaml:"postcheck_gate"`
|
||
}
|
||
|
||
// CoverageGate v1 (Р7): метрика — символы без пробелов; нижняя граница —
|
||
// подозрение на вырезание, верхняя — аномалия; пороги из эксперимента 02.
|
||
type CoverageGate struct {
|
||
Enabled bool `yaml:"enabled"`
|
||
LenRatio map[string][]float64 `yaml:"len_ratio_bounds"` // "zh-ru": [low, high]
|
||
SentCovMin float64 `yaml:"sent_cov_min"`
|
||
MinChunkChars int `yaml:"min_chunk_chars"`
|
||
}
|
||
|
||
// Escalation holds named model chains; channel-aware filtering (Anthropic вне
|
||
// 18+ и т.д.) применяет раннер, не конфиг.
|
||
type Escalation struct {
|
||
Chains map[string][]string `yaml:"chains"`
|
||
BudgetUSD float64 `yaml:"budget_usd"` // адресный премиум-бюджет книги; формула — [НУЖНО РЕШЕНИЕ] п.1
|
||
}
|
||
|
||
// Fanout is the C2 skeleton: N候補 candidates per chunk (fan-out — механика
|
||
// раннера, N — конфиг).
|
||
type Fanout struct {
|
||
Candidates int `yaml:"candidates"`
|
||
}
|
||
|
||
// gatesEnabled reports whether any QA-gate is on. Эскалация запускается только
|
||
// по провалу гейта, поэтому это сигнал «эскалация вообще достижима» (Фаза 1+).
|
||
func (p *Pipeline) gatesEnabled() bool {
|
||
return p.Gates.Coverage.Enabled
|
||
}
|
||
|
||
// CheckRunnable rejects a config whose mechanics the Phase-0 runner does NOT
|
||
// implement — раздельно от schema-валидации (LoadPipeline), чтобы скелеты C2/C3
|
||
// парсились и валидировались как схема, но НЕ исполнялись молча. Без этого
|
||
// гейта запуск pipeline-c2.yaml прогнал бы стадии линейно: fanout.candidates
|
||
// игнорируется, вердикт судьи утёк бы в редактуру, а деньги Opus сгорели бы на
|
||
// мусор (находка ревью; Р2 требует закрывать дыру «конфиг vs код» fail-loud).
|
||
func (p *Pipeline) CheckRunnable() error {
|
||
switch p.Core {
|
||
case "C0", "C1":
|
||
default:
|
||
return fmt.Errorf("pipeline core %q не исполним в Фазе 0 (реализованы только C0/C1 — линейный проход стадий; C2/C3 selection/fusion — механика раннера Фазы 2)", p.Core)
|
||
}
|
||
if p.Fanout.Candidates > 1 {
|
||
return fmt.Errorf("pipeline fanout.candidates=%d не исполним в Фазе 0 (fan-out N кандидатов — механика раннера Фазы 2; C1 = один черновик)", p.Fanout.Candidates)
|
||
}
|
||
// Coverage QA-гейт исполним с Вехи 2.5 (excision-детект, coverage.go): раннер
|
||
// считает его по результату стадии и эмитит excision_suspect. Бывший fail-loud
|
||
// на gates.enabled снят; корректность порогов включённого гейта проверяет
|
||
// LoadPipeline (иначе тихий no-op — против Р7).
|
||
for _, st := range p.Stages {
|
||
if st.Role == "judge" {
|
||
return fmt.Errorf("pipeline stage %q role=judge не исполним в Фазе 0 (селектор получает N кандидатов — механика Фазы 2, а раннер гонит стадии линейно)", st.Name)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// CheckAdultChannel enforces the D20.2-Q3 load-time consistency lint: a book marked
|
||
// adult:true MUST route through at least one channel:adult stage, else the 18+ text would
|
||
// silently run on the SFW pipeline (all-SFW providers that refuse/excise explicit content).
|
||
// Adult stays wire/verdict-NEUTRAL (no snapshot layer — D20.2-Q3: the first runtime consumer
|
||
// assigns meaning; here it is only a routing sanity check, never a hash input). The REVERSE
|
||
// (a channel:adult stage in an adult:false book) is deliberately NOT gated: the stage's
|
||
// channel already forces a permissive provider (D4.1), and Adult carries no assigned meaning
|
||
// yet, so a permissive stage may legitimately exist before the book flag is set.
|
||
func (p *Pipeline) CheckAdultChannel(bookAdult bool) error {
|
||
if !bookAdult {
|
||
return nil
|
||
}
|
||
for _, st := range p.Stages {
|
||
if st.Channel == "adult" {
|
||
return nil
|
||
}
|
||
}
|
||
return fmt.Errorf("book adult:true, но ни одна стадия не имеет channel:adult — 18+ текст пошёл бы по SFW-каналу (SFW-провайдеры отказывают/вырезают explicit); заведите channel:adult-стадию на permissive-провайдере (D4.1/D20.2-Q3) либо поставьте adult:false")
|
||
}
|
||
|
||
// LoadPipeline reads and validates a pipeline config against the models known
|
||
// to models.yaml.
|
||
func LoadPipeline(path string, models *Models) (*Pipeline, error) {
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("config: read %s: %w", path, err)
|
||
}
|
||
var p Pipeline
|
||
if err := yaml.Unmarshal(raw, &p); err != nil {
|
||
return nil, fmt.Errorf("config: parse %s: %w", path, err)
|
||
}
|
||
// Prompt paths resolve relative to the pipeline file's directory, so a
|
||
// project works from any CWD.
|
||
dir := filepath.Dir(path)
|
||
for i := range p.Stages {
|
||
if pr := p.Stages[i].Prompt; pr != "" && !filepath.IsAbs(pr) {
|
||
p.Stages[i].Prompt = filepath.Join(dir, pr)
|
||
}
|
||
}
|
||
|
||
var problems []string
|
||
bad := func(format string, a ...any) { problems = append(problems, fmt.Sprintf(format, a...)) }
|
||
|
||
if p.Core == "" {
|
||
bad("core is required (C0|C1|C2|C3)")
|
||
}
|
||
if len(p.Stages) == 0 {
|
||
bad("at least one stage is required")
|
||
}
|
||
if p.Defaults.MaxOutputRatio <= 0 {
|
||
p.Defaults.MaxOutputRatio = 2.0
|
||
}
|
||
if p.Defaults.MinMaxTokens <= 0 {
|
||
p.Defaults.MinMaxTokens = 2048
|
||
}
|
||
if p.Fanout.Candidates <= 0 {
|
||
p.Fanout.Candidates = 1
|
||
}
|
||
switch p.Context.GlossaryInjection {
|
||
case "", "selective", "full_prefix":
|
||
default:
|
||
bad("context.glossary_injection must be selective|full_prefix, got %q", p.Context.GlossaryInjection)
|
||
}
|
||
seen := map[string]bool{}
|
||
for i, st := range p.Stages {
|
||
if st.Name == "" {
|
||
bad("stage %d: name is required", i)
|
||
}
|
||
if seen[st.Name] {
|
||
bad("stage %q: duplicate name", st.Name)
|
||
}
|
||
seen[st.Name] = true
|
||
if st.Role == "" {
|
||
bad("stage %q: role is required", st.Name)
|
||
}
|
||
if st.PromptVersion == "" {
|
||
bad("stage %q: prompt_version is required (версионирование промптов — Р2)", st.Name)
|
||
}
|
||
if _, ok := models.Models[st.Model]; !ok {
|
||
bad("stage %q: model %q is not defined in models.yaml", st.Name, st.Model)
|
||
}
|
||
if st.Prompt == "" {
|
||
bad("stage %q: prompt template path is required", st.Name)
|
||
} else if _, err := os.Stat(st.Prompt); err != nil {
|
||
bad("stage %q: prompt template %s is not readable: %v", st.Name, st.Prompt, err)
|
||
}
|
||
switch st.Reasoning {
|
||
case "", "off", "low", "medium", "high":
|
||
default:
|
||
bad("stage %q: reasoning must be off|low|medium|high, got %q", st.Name, st.Reasoning)
|
||
}
|
||
// D13.6: on an ADDITIVE-billing provider (xAI — reasoning bills ON TOP of completion) a
|
||
// stage that MAY think needs a reserved reasoning_max_tokens buffer, else the reservation
|
||
// under-budgets and the spend ceiling is blind to the overshoot pricing.go only
|
||
// acknowledges. "May think" = any effort EXCEPT an explicit reasoning:"off" (which sends
|
||
// the provider's disable — off_effort:"none" for grok): low|medium|high think, and an
|
||
// OMITTED/"" effort falls to the provider DEFAULT, which for xAI-grok is "low" (thinking
|
||
// ON, additive) — so "" must NOT escape the gate (self-review finding #4: the earlier
|
||
// thinks-only check let it through). Subset providers never need a buffer. This is the
|
||
// structural gate D6.2 requires before think-ON is allowed on grok (the contrast the task
|
||
// draws with the echo mine). Checks the primary AND the escalate_to model (same effort).
|
||
if st.Reasoning != "off" && st.ReasoningMaxTokens <= 0 {
|
||
if models.providerReasoning(st.Model) == "additive" {
|
||
bad("stage %q: reasoning=%q on additive-billing provider (model %q) requires reasoning_max_tokens>0 — xAI bills reasoning on top of completion and thinks by default unless reasoning is explicitly \"off\"; without a reserved buffer the spend ceiling is blind to the overshoot (D6.2/D13.6)", st.Name, st.Reasoning, st.Model)
|
||
}
|
||
if st.EscalateTo != "" && models.providerReasoning(st.EscalateTo) == "additive" {
|
||
bad("stage %q: reasoning=%q with escalate_to on an additive-billing provider (model %q) requires reasoning_max_tokens>0 (D6.2/D13.6)", st.Name, st.Reasoning, st.EscalateTo)
|
||
}
|
||
}
|
||
switch st.Channel {
|
||
case "", "sfw", "adult":
|
||
default:
|
||
bad("stage %q: channel must be sfw|adult, got %q", st.Name, st.Channel)
|
||
}
|
||
if st.EscalateTo != "" {
|
||
if _, ok := models.Models[st.EscalateTo]; !ok {
|
||
bad("stage %q: escalate_to model %q is not defined in models.yaml", st.Name, st.EscalateTo)
|
||
} else if st.EscalateTo == st.Model {
|
||
bad("stage %q: escalate_to must differ from the primary model %q (a same-model hop is a guaranteed repeat)", st.Name, st.Model)
|
||
}
|
||
// D12 editor-pinned, enforced structurally: only the translator role may
|
||
// fall back to another model. An editor/other stage that escalated would
|
||
// drift its style/terms to a foreign model (2605.13368) — forbid it at load.
|
||
if st.Role != "translator" {
|
||
bad("stage %q: escalate_to is only allowed on a translator role (D12 editor-pinned — a %q stage must not fall back to a foreign model)", st.Name, st.Role)
|
||
}
|
||
}
|
||
// D4.1: channel-B (18+) isolation is enforced by TYPE — a permissive stage
|
||
// may only run on, and escalate to, a permissive provider.
|
||
if st.Channel == "adult" {
|
||
if !models.providerPermissive(st.Model) {
|
||
bad("stage %q: channel=adult requires model %q on a permissive provider (D4.1)", st.Name, st.Model)
|
||
}
|
||
if st.EscalateTo != "" && !models.providerPermissive(st.EscalateTo) {
|
||
bad("stage %q: channel=adult escalate_to %q must be on a permissive provider (D4.1 — no fall-through to a refusing SFW channel)", st.Name, st.EscalateTo)
|
||
}
|
||
}
|
||
}
|
||
for chain, ms := range p.Escal.Chains {
|
||
for _, m := range ms {
|
||
if _, ok := models.Models[m]; !ok {
|
||
bad("escalation chain %q: model %q is not defined in models.yaml", chain, m)
|
||
}
|
||
}
|
||
}
|
||
// Coverage QA-gate thresholds (шаг 6): an ENABLED gate must be able to flag
|
||
// something, otherwise it is a silent no-op that passes every chunk (against Р7).
|
||
if cov := p.Gates.Coverage; cov.Enabled {
|
||
// sent_cov_min must be a LIVE threshold in (0,1]: it is the pair-INDEPENDENT
|
||
// half of the gate, so requiring it guarantees the gate can never silently pass
|
||
// every chunk for a language pair that lacks a len_ratio corridor (self-review:
|
||
// the old "both empty" check missed the per-pair gap — a non-empty len_ratio map
|
||
// that just lacks the book's pair, with sent_cov_min=0, passed everything).
|
||
if cov.SentCovMin <= 0 || cov.SentCovMin > 1 {
|
||
bad("gates.coverage.sent_cov_min must be within (0, 1] when the gate is enabled, got %v (else the gate silently passes any language pair without a len_ratio corridor)", cov.SentCovMin)
|
||
}
|
||
for pair, b := range cov.LenRatio {
|
||
if len(b) != 2 || b[0] <= 0 || b[1] < b[0] {
|
||
bad("gates.coverage.len_ratio_bounds[%q] must be [low, high] with 0 < low <= high, got %v", pair, b)
|
||
}
|
||
}
|
||
}
|
||
if len(problems) > 0 {
|
||
return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||
}
|
||
return &p, nil
|
||
}
|