306 lines
15 KiB
Go
306 lines
15 KiB
Go
package config
|
||
|
||
import (
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
|
||
"gopkg.in/yaml.v3"
|
||
)
|
||
|
||
// pipeline.go loads the pipeline-core config (C1/C2… — Р2). Граница «конфиг vs
|
||
// код» зафиксирована: конфиг задаёт состав/порядок стадий, роль→модель, версии
|
||
// промптов, пороги гейтов, режим инъекции глоссария, эскалационные цепочки,
|
||
// fan-out N, токен-бюджеты сборки контекста и cache TTL. Циклы по
|
||
// главам/чанкам, ветвление по гейтам, механика эскалации/ретраев и правило
|
||
// «эскалация → re-gate → флаг» зашиты в раннер.
|
||
|
||
// Pipeline is the parsed pipeline-*.yaml.
|
||
type Pipeline struct {
|
||
Core string `yaml:"core"` // C0|C1|C2|C3
|
||
Version int `yaml:"version"`
|
||
Defaults PipelineDefaults `yaml:"defaults"`
|
||
Context ContextAssembly `yaml:"context"`
|
||
Retries Retries `yaml:"retries"`
|
||
Stages []Stage `yaml:"stages"`
|
||
Gates Gates `yaml:"gates"`
|
||
Escal Escalation `yaml:"escalation"`
|
||
Fanout Fanout `yaml:"fanout"`
|
||
}
|
||
|
||
// PipelineDefaults are cross-stage knobs.
|
||
type PipelineDefaults struct {
|
||
// MaxOutputRatio sizes max_tokens ≈ ratio × input tokens (токен-калибровка
|
||
// полигона: русский выход zh→ru ≈1.9× входа; дефолт с запасом).
|
||
MaxOutputRatio float64 `yaml:"max_output_ratio"`
|
||
MinMaxTokens int `yaml:"min_max_tokens"`
|
||
}
|
||
|
||
// ContextAssembly holds the prompt-layout budgets (Р5-раскладка: стабильный
|
||
// префикс / волатильный хвост — сборка Фазы 1; поля фиксируются сейчас, чтобы
|
||
// схема конфига не менялась под ногами «Редакции»).
|
||
type ContextAssembly struct {
|
||
GlossaryInjection string `yaml:"glossary_injection"` // selective | full_prefix (Р5: обе схемы)
|
||
GlossaryTokenBudget int `yaml:"glossary_token_budget"`
|
||
STMDepth int `yaml:"stm_depth"`
|
||
OverlapTokens int `yaml:"overlap_tokens"`
|
||
CacheTTL string `yaml:"cache_ttl"` // per-stage TTL override — Фаза 1
|
||
}
|
||
|
||
// Retries distinguishes the CONTENT retry (регенерация после провала гейта до
|
||
// эскалации) from transport retries (профиль в models.yaml) — спека их
|
||
// различает явно (валидация, линза 2 п.5).
|
||
type Retries struct {
|
||
RegenerateBeforeEscalate int `yaml:"regenerate_before_escalate"`
|
||
}
|
||
|
||
// Stage is one pipeline pass.
|
||
type Stage struct {
|
||
Name string `yaml:"name"`
|
||
Role string `yaml:"role"`
|
||
Model string `yaml:"model"`
|
||
Prompt string `yaml:"prompt"` // path to the versioned template file
|
||
PromptVersion string `yaml:"prompt_version"`
|
||
Temperature float64 `yaml:"temperature"`
|
||
Reasoning string `yaml:"reasoning"` // "", off, low, medium, high
|
||
// ReasoningMaxTokens is the explicit reasoning-token BUFFER reserved for this stage
|
||
// when it thinks (reasoning low|medium|high) on an ADDITIVE-billing provider (xAI —
|
||
// reasoning bills on top of completion, D13.6). Required (>0) in exactly that case
|
||
// (fail-fast in LoadPipeline); ignored for subset providers / reasoning-off. It only
|
||
// sizes the reservation (EstimateUSD), so it is NOT part of the snapshot/wire.
|
||
ReasoningMaxTokens int `yaml:"reasoning_max_tokens"`
|
||
// EscalateTo is the SINGLE-HOP fallback model tried ONCE when this stage's
|
||
// output is a deterministic content-failure another model might fix (echo /
|
||
// excision / refusal — D12). Empty = no escalation (e.g. the editor is pinned
|
||
// per book — its style must not drift to a foreign model, D12/2605.13368). The
|
||
// fallback is its OWN model, which is the request_hash axis (a distinct
|
||
// checkpoint, never the failed primary's). Gated by escalation.budget_usd.
|
||
EscalateTo string `yaml:"escalate_to"`
|
||
// Channel selects the safety channel: "" | sfw (default) | adult (18+). An
|
||
// adult stage is isolated by TYPE (D4.1): its model AND its escalate_to must be
|
||
// on permissive providers, enforced at load — never a fall-through to a refusing
|
||
// SFW provider.
|
||
Channel string `yaml:"channel"`
|
||
}
|
||
|
||
// Gates is the QA-gate config skeleton (пороги полигона; исполнение — Фаза 1).
|
||
type Gates struct {
|
||
Coverage CoverageGate `yaml:"coverage"`
|
||
Glossary GlossaryGate `yaml:"glossary"`
|
||
}
|
||
|
||
// GlossaryGate controls the memory-bank post-check (E1). The post-check ALWAYS runs
|
||
// (observability — it records misses into the retrieval-state), converting silent
|
||
// glossary drift into a loud, visible signal. PostcheckGate promotes a miss from a
|
||
// mere record to a DISPOSITION flag (the chunk is flagged, downstream stages skip):
|
||
// opt-in (default false = flagger), flipped on only AFTER the owner validates the
|
||
// false-flag rate on real chapters (E1) — exactly the coverage gate's opt-in discipline
|
||
// (D12 Q4). A naive/under-filled decl false-flags (research/14 §2), so a premature hard
|
||
// gate would flag-storm and train editors to ignore it.
|
||
type GlossaryGate struct {
|
||
PostcheckGate bool `yaml:"postcheck_gate"`
|
||
}
|
||
|
||
// CoverageGate v1 (Р7): метрика — символы без пробелов; нижняя граница —
|
||
// подозрение на вырезание, верхняя — аномалия; пороги из эксперимента 02.
|
||
type CoverageGate struct {
|
||
Enabled bool `yaml:"enabled"`
|
||
LenRatio map[string][]float64 `yaml:"len_ratio_bounds"` // "zh-ru": [low, high]
|
||
SentCovMin float64 `yaml:"sent_cov_min"`
|
||
MinChunkChars int `yaml:"min_chunk_chars"`
|
||
}
|
||
|
||
// Escalation holds named model chains; channel-aware filtering (Anthropic вне
|
||
// 18+ и т.д.) применяет раннер, не конфиг.
|
||
type Escalation struct {
|
||
Chains map[string][]string `yaml:"chains"`
|
||
BudgetUSD float64 `yaml:"budget_usd"` // адресный премиум-бюджет книги; формула — [НУЖНО РЕШЕНИЕ] п.1
|
||
}
|
||
|
||
// Fanout is the C2 skeleton: N候補 candidates per chunk (fan-out — механика
|
||
// раннера, N — конфиг).
|
||
type Fanout struct {
|
||
Candidates int `yaml:"candidates"`
|
||
}
|
||
|
||
// gatesEnabled reports whether any QA-gate is on. Эскалация запускается только
|
||
// по провалу гейта, поэтому это сигнал «эскалация вообще достижима» (Фаза 1+).
|
||
func (p *Pipeline) gatesEnabled() bool {
|
||
return p.Gates.Coverage.Enabled
|
||
}
|
||
|
||
// CheckRunnable rejects a config whose mechanics the Phase-0 runner does NOT
|
||
// implement — раздельно от schema-валидации (LoadPipeline), чтобы скелеты C2/C3
|
||
// парсились и валидировались как схема, но НЕ исполнялись молча. Без этого
|
||
// гейта запуск pipeline-c2.yaml прогнал бы стадии линейно: fanout.candidates
|
||
// игнорируется, вердикт судьи утёк бы в редактуру, а деньги Opus сгорели бы на
|
||
// мусор (находка ревью; Р2 требует закрывать дыру «конфиг vs код» fail-loud).
|
||
func (p *Pipeline) CheckRunnable() error {
|
||
switch p.Core {
|
||
case "C0", "C1":
|
||
default:
|
||
return fmt.Errorf("pipeline core %q не исполним в Фазе 0 (реализованы только C0/C1 — линейный проход стадий; C2/C3 selection/fusion — механика раннера Фазы 2)", p.Core)
|
||
}
|
||
if p.Fanout.Candidates > 1 {
|
||
return fmt.Errorf("pipeline fanout.candidates=%d не исполним в Фазе 0 (fan-out N кандидатов — механика раннера Фазы 2; C1 = один черновик)", p.Fanout.Candidates)
|
||
}
|
||
// Coverage QA-гейт исполним с Вехи 2.5 (excision-детект, coverage.go): раннер
|
||
// считает его по результату стадии и эмитит excision_suspect. Бывший fail-loud
|
||
// на gates.enabled снят; корректность порогов включённого гейта проверяет
|
||
// LoadPipeline (иначе тихий no-op — против Р7).
|
||
for _, st := range p.Stages {
|
||
if st.Role == "judge" {
|
||
return fmt.Errorf("pipeline stage %q role=judge не исполним в Фазе 0 (селектор получает N кандидатов — механика Фазы 2, а раннер гонит стадии линейно)", st.Name)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// LoadPipeline reads and validates a pipeline config against the models known
|
||
// to models.yaml.
|
||
func LoadPipeline(path string, models *Models) (*Pipeline, error) {
|
||
raw, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("config: read %s: %w", path, err)
|
||
}
|
||
var p Pipeline
|
||
if err := yaml.Unmarshal(raw, &p); err != nil {
|
||
return nil, fmt.Errorf("config: parse %s: %w", path, err)
|
||
}
|
||
// Prompt paths resolve relative to the pipeline file's directory, so a
|
||
// project works from any CWD.
|
||
dir := filepath.Dir(path)
|
||
for i := range p.Stages {
|
||
if pr := p.Stages[i].Prompt; pr != "" && !filepath.IsAbs(pr) {
|
||
p.Stages[i].Prompt = filepath.Join(dir, pr)
|
||
}
|
||
}
|
||
|
||
var problems []string
|
||
bad := func(format string, a ...any) { problems = append(problems, fmt.Sprintf(format, a...)) }
|
||
|
||
if p.Core == "" {
|
||
bad("core is required (C0|C1|C2|C3)")
|
||
}
|
||
if len(p.Stages) == 0 {
|
||
bad("at least one stage is required")
|
||
}
|
||
if p.Defaults.MaxOutputRatio <= 0 {
|
||
p.Defaults.MaxOutputRatio = 2.0
|
||
}
|
||
if p.Defaults.MinMaxTokens <= 0 {
|
||
p.Defaults.MinMaxTokens = 2048
|
||
}
|
||
if p.Fanout.Candidates <= 0 {
|
||
p.Fanout.Candidates = 1
|
||
}
|
||
switch p.Context.GlossaryInjection {
|
||
case "", "selective", "full_prefix":
|
||
default:
|
||
bad("context.glossary_injection must be selective|full_prefix, got %q", p.Context.GlossaryInjection)
|
||
}
|
||
seen := map[string]bool{}
|
||
for i, st := range p.Stages {
|
||
if st.Name == "" {
|
||
bad("stage %d: name is required", i)
|
||
}
|
||
if seen[st.Name] {
|
||
bad("stage %q: duplicate name", st.Name)
|
||
}
|
||
seen[st.Name] = true
|
||
if st.Role == "" {
|
||
bad("stage %q: role is required", st.Name)
|
||
}
|
||
if st.PromptVersion == "" {
|
||
bad("stage %q: prompt_version is required (версионирование промптов — Р2)", st.Name)
|
||
}
|
||
if _, ok := models.Models[st.Model]; !ok {
|
||
bad("stage %q: model %q is not defined in models.yaml", st.Name, st.Model)
|
||
}
|
||
if st.Prompt == "" {
|
||
bad("stage %q: prompt template path is required", st.Name)
|
||
} else if _, err := os.Stat(st.Prompt); err != nil {
|
||
bad("stage %q: prompt template %s is not readable: %v", st.Name, st.Prompt, err)
|
||
}
|
||
switch st.Reasoning {
|
||
case "", "off", "low", "medium", "high":
|
||
default:
|
||
bad("stage %q: reasoning must be off|low|medium|high, got %q", st.Name, st.Reasoning)
|
||
}
|
||
// D13.6: on an ADDITIVE-billing provider (xAI — reasoning bills ON TOP of completion) a
|
||
// stage that MAY think needs a reserved reasoning_max_tokens buffer, else the reservation
|
||
// under-budgets and the spend ceiling is blind to the overshoot pricing.go only
|
||
// acknowledges. "May think" = any effort EXCEPT an explicit reasoning:"off" (which sends
|
||
// the provider's disable — off_effort:"none" for grok): low|medium|high think, and an
|
||
// OMITTED/"" effort falls to the provider DEFAULT, which for xAI-grok is "low" (thinking
|
||
// ON, additive) — so "" must NOT escape the gate (self-review finding #4: the earlier
|
||
// thinks-only check let it through). Subset providers never need a buffer. This is the
|
||
// structural gate D6.2 requires before think-ON is allowed on grok (the contrast the task
|
||
// draws with the echo mine). Checks the primary AND the escalate_to model (same effort).
|
||
if st.Reasoning != "off" && st.ReasoningMaxTokens <= 0 {
|
||
if models.providerReasoning(st.Model) == "additive" {
|
||
bad("stage %q: reasoning=%q on additive-billing provider (model %q) requires reasoning_max_tokens>0 — xAI bills reasoning on top of completion and thinks by default unless reasoning is explicitly \"off\"; without a reserved buffer the spend ceiling is blind to the overshoot (D6.2/D13.6)", st.Name, st.Reasoning, st.Model)
|
||
}
|
||
if st.EscalateTo != "" && models.providerReasoning(st.EscalateTo) == "additive" {
|
||
bad("stage %q: reasoning=%q with escalate_to on an additive-billing provider (model %q) requires reasoning_max_tokens>0 (D6.2/D13.6)", st.Name, st.Reasoning, st.EscalateTo)
|
||
}
|
||
}
|
||
switch st.Channel {
|
||
case "", "sfw", "adult":
|
||
default:
|
||
bad("stage %q: channel must be sfw|adult, got %q", st.Name, st.Channel)
|
||
}
|
||
if st.EscalateTo != "" {
|
||
if _, ok := models.Models[st.EscalateTo]; !ok {
|
||
bad("stage %q: escalate_to model %q is not defined in models.yaml", st.Name, st.EscalateTo)
|
||
} else if st.EscalateTo == st.Model {
|
||
bad("stage %q: escalate_to must differ from the primary model %q (a same-model hop is a guaranteed repeat)", st.Name, st.Model)
|
||
}
|
||
// D12 editor-pinned, enforced structurally: only the translator role may
|
||
// fall back to another model. An editor/other stage that escalated would
|
||
// drift its style/terms to a foreign model (2605.13368) — forbid it at load.
|
||
if st.Role != "translator" {
|
||
bad("stage %q: escalate_to is only allowed on a translator role (D12 editor-pinned — a %q stage must not fall back to a foreign model)", st.Name, st.Role)
|
||
}
|
||
}
|
||
// D4.1: channel-B (18+) isolation is enforced by TYPE — a permissive stage
|
||
// may only run on, and escalate to, a permissive provider.
|
||
if st.Channel == "adult" {
|
||
if !models.providerPermissive(st.Model) {
|
||
bad("stage %q: channel=adult requires model %q on a permissive provider (D4.1)", st.Name, st.Model)
|
||
}
|
||
if st.EscalateTo != "" && !models.providerPermissive(st.EscalateTo) {
|
||
bad("stage %q: channel=adult escalate_to %q must be on a permissive provider (D4.1 — no fall-through to a refusing SFW channel)", st.Name, st.EscalateTo)
|
||
}
|
||
}
|
||
}
|
||
for chain, ms := range p.Escal.Chains {
|
||
for _, m := range ms {
|
||
if _, ok := models.Models[m]; !ok {
|
||
bad("escalation chain %q: model %q is not defined in models.yaml", chain, m)
|
||
}
|
||
}
|
||
}
|
||
// Coverage QA-gate thresholds (шаг 6): an ENABLED gate must be able to flag
|
||
// something, otherwise it is a silent no-op that passes every chunk (against Р7).
|
||
if cov := p.Gates.Coverage; cov.Enabled {
|
||
// sent_cov_min must be a LIVE threshold in (0,1]: it is the pair-INDEPENDENT
|
||
// half of the gate, so requiring it guarantees the gate can never silently pass
|
||
// every chunk for a language pair that lacks a len_ratio corridor (self-review:
|
||
// the old "both empty" check missed the per-pair gap — a non-empty len_ratio map
|
||
// that just lacks the book's pair, with sent_cov_min=0, passed everything).
|
||
if cov.SentCovMin <= 0 || cov.SentCovMin > 1 {
|
||
bad("gates.coverage.sent_cov_min must be within (0, 1] when the gate is enabled, got %v (else the gate silently passes any language pair without a len_ratio corridor)", cov.SentCovMin)
|
||
}
|
||
for pair, b := range cov.LenRatio {
|
||
if len(b) != 2 || b[0] <= 0 || b[1] < b[0] {
|
||
bad("gates.coverage.len_ratio_bounds[%q] must be [low, high] with 0 < low <= high, got %v", pair, b)
|
||
}
|
||
}
|
||
}
|
||
if len(problems) > 0 {
|
||
return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||
}
|
||
return &p, nil
|
||
}
|