// Package config loads and fail-fast-validates the three YAML files of a run: // models.yaml (провайдеры/модели/цены/таймауты — Р4/Р5: цены только в конфиге // с датой проверки), pipeline-*.yaml (ядро C1/C2 — граница «конфиг vs код» по // Р2) and book.yaml (translation brief + brief_hash). // // Дисциплина донора: все проблемы конфига собираются СПИСКОМ и валят старт // одной ошибкой — оператор чинит всё сразу, а не по одной кочке за запуск. package config import ( "fmt" "os" "strings" "time" "gopkg.in/yaml.v3" "textmachine/backend/internal/ledger" "textmachine/backend/internal/llm" ) // Models is the parsed models.yaml. type Models struct { // PricesChecked is the date the prices below were verified against the // providers' official pages. Fail-fast rejects a stale (>120 дней) config: // волатильность цен API — риск Р10 №5. PricesChecked string `yaml:"prices_checked"` DefaultModel string `yaml:"default_model"` Providers map[string]Provider `yaml:"providers"` Models map[string]Model `yaml:"models"` } // Provider is one backend endpoint. type Provider struct { Kind string `yaml:"kind"` // openai | anthropic | local BaseURL string `yaml:"base_url"` APIKeyEnv string `yaml:"api_key_env"` Reasoning string `yaml:"reasoning"` // subset | additive (openai kind only) // EchoesWhenThinkingOff marks a provider empirically shown to ECHO the // untranslated CJK source instead of translating when thinking is OFF // (DeepSeek, traced 2026-07-04 — reproducibly on dense CJK, 3/3 retries; // PROGRESS «Ответ Полигону», BACKEND_SESSION_PROMPT_SILENT_REFUSALS §2). Its // benign state is held by an off-by-omission: reasoning="off" resolves through // ReasoningNone which emits NOTHING thinking-related, so the provider default // (thinking ON) stands. A config that DISABLES thinking at reasoning=off on // such a provider re-arms a silent refusal (echo = HTTP 200, no translation) // in prod. LoadModels (echoMineViolation) fail-fasts on exactly that — the // regression gate §2 mandates. It distinguishes the SAFE GLM editor, whose // thinking:disabled is fine because its input is a Russian draft (no CJK to // echo): GLM's provider carries no such flag. Validation-only metadata — it // never reaches the wire, so it is NOT part of the snapshot (no --resnapshot). EchoesWhenThinkingOff bool `yaml:"echoes_when_thinking_off"` // Permissive marks a provider allowed to serve the 18+ channel B (the most // permissive large APIs: xAI/Grok, DeepSeek official, local abliterated — D3). // It is the TYPE that enforces channel-B isolation (D4.1): a channel=adult stage // may only run on, and escalate to, a permissive provider — a structural gate, // not a comment, so an SFW-only provider can never silently serve/answer a // permissive chunk. Validation-only metadata (never on the wire, not in snapshot). Permissive bool `yaml:"permissive"` CacheTTL string `yaml:"cache_ttl"` // anthropic kind: "", "5m", "1h" Model string `yaml:"model"` // local kind: the backend's own tag MaxTokens int `yaml:"max_tokens"` // Temperature (local kind): override запроса — ollama чтит request- // температуру поверх Modelfile, и температура облачной роли молча // расстроила бы локальную модель. 0 = наследовать запрос. Temperature float64 `yaml:"temperature"` Timeouts Timeouts `yaml:"timeouts"` // Capabilities is the DEFAULT wire shape for this provider's models (D3.1); // a per-Model block overrides it field-by-field. Nil = OpenAI-compat // baseline (max_tokens + send temperature + off-by-omission reasoning). Capabilities *CapabilitiesConfig `yaml:"capabilities"` } // CapabilitiesConfig is the declarative per-model wire shape (D3.1). Empty // fields inherit: a Model block layers over its Provider block, which layers // over the kind baseline. Separate from Provider.Reasoning (subset|additive), // which is a billing semantic — this is the wire body form. type CapabilitiesConfig struct { BudgetField string `yaml:"budget_field"` // max_tokens | max_completion_tokens Temperature *TemperatureCap `yaml:"temperature"` Reasoning *ReasoningCapCfg `yaml:"reasoning"` } // TemperatureCap declares how temperature reaches the wire. type TemperatureCap struct { Mode string `yaml:"mode"` // send | omit | force Value float64 `yaml:"value"` // used when mode == force } // ReasoningCapCfg declares how the neutral reasoning effort maps to the wire. type ReasoningCapCfg struct { Control string `yaml:"control"` // none | effort | extra_body_disable | mandatory OffEffort string `yaml:"off_effort"` // effort: value sent when reasoning=off OffExtraBody map[string]any `yaml:"off_extra_body"` // extra_body_disable: merged when reasoning=off OnExtraBody map[string]any `yaml:"on_extra_body"` // extra_body_disable: merged when reasoning is on (D6.2) } // Timeouts is the retry profile per provider (профиль per-провайдер из // вердикта валидации; per-role overrides — Фаза 1). type Timeouts struct { AttemptS int `yaml:"attempt_s"` MaxAttempts int `yaml:"max_attempts"` BackoffCapS int `yaml:"backoff_cap_s"` } func (t Timeouts) Profile() llm.RetryProfile { return llm.RetryProfile{ AttemptTimeout: time.Duration(t.AttemptS) * time.Second, MaxAttempts: t.MaxAttempts, BackoffCap: time.Duration(t.BackoffCapS) * time.Second, } } // Model is one priced model entry. type Model struct { Provider string `yaml:"provider"` Price Price `yaml:"price"` // ExtraBody is merged into request JSON for this model (провайдер- // специфичные ручки: GLM thinking.type, Qwen enable_thinking …). ExtraBody map[string]any `yaml:"extra_body"` // MinCachePrefixTokens: Anthropic-модели — минимальный кэшируемый префикс // (короче — кэш молча не создаётся); сборщик контекста Фазы 1 сверяется. MinCachePrefixTokens int `yaml:"min_cache_prefix_tokens"` Note string `yaml:"note"` // Capabilities overrides this model's provider-default wire shape (D3.1), // field-by-field (model wins). Nil = inherit the provider/kind baseline. Capabilities *CapabilitiesConfig `yaml:"capabilities"` } // Price mirrors ledger.ModelPrice in YAML form (USD per 1M tokens). type Price struct { InputPerM float64 `yaml:"input_per_m"` CachedPerM float64 `yaml:"cached_per_m"` CacheWritePerM float64 `yaml:"cache_write_per_m"` OutputPerM float64 `yaml:"output_per_m"` } func (p Price) ToLedger() ledger.ModelPrice { return ledger.ModelPrice{ InputPerM: p.InputPerM, CachedPerM: p.CachedPerM, CacheWritePerM: p.CacheWritePerM, OutputPerM: p.OutputPerM, } } // maxPriceAge is how stale prices_checked may be before the config is // rejected (ежеквартальный пересмотр Р4 + буфер). const maxPriceAge = 120 * 24 * time.Hour // LoadModels reads and validates models.yaml. func LoadModels(path string) (*Models, error) { raw, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("config: read %s: %w", path, err) } var m Models if err := yaml.Unmarshal(raw, &m); err != nil { return nil, fmt.Errorf("config: parse %s: %w", path, err) } var problems []string bad := func(format string, a ...any) { problems = append(problems, fmt.Sprintf(format, a...)) } if m.PricesChecked == "" { bad("prices_checked is required (цены без даты проверки запрещены Р4)") } else if t, err := time.Parse("2006-01-02", m.PricesChecked); err != nil { bad("prices_checked %q is not YYYY-MM-DD", m.PricesChecked) } else if time.Since(t) > maxPriceAge { bad("prices_checked %s is older than %d days — re-verify provider prices", m.PricesChecked, int(maxPriceAge.Hours()/24)) } if m.DefaultModel == "" { bad("default_model is required (fallback-якорь цены: неизвестная модель никогда не стоит $0)") } else if _, ok := m.Models[m.DefaultModel]; !ok { bad("default_model %q is not defined in models", m.DefaultModel) } for name, p := range m.Providers { switch p.Kind { case "openai", "anthropic", "local": default: bad("provider %s: unknown kind %q", name, p.Kind) } if p.BaseURL == "" && p.Kind != "anthropic" { bad("provider %s: base_url is required", name) } if p.Kind == "openai" && p.Reasoning != "" && p.Reasoning != "subset" && p.Reasoning != "additive" { bad("provider %s: reasoning must be subset|additive, got %q", name, p.Reasoning) } if p.Kind == "anthropic" && p.CacheTTL != "" && p.CacheTTL != "5m" && p.CacheTTL != "1h" { bad("provider %s: cache_ttl must be 5m|1h, got %q", name, p.CacheTTL) } if p.Kind == "local" && p.Model == "" { bad("provider %s: local kind requires model (its own tag)", name) } validateCapabilities(bad, "provider "+name, p.Capabilities) } for name, mod := range m.Models { prov, ok := m.Providers[mod.Provider] if !ok { bad("model %s: provider %q is not defined", name, mod.Provider) continue } isLocal := prov.Kind == "local" if !isLocal && (mod.Price.InputPerM <= 0 || mod.Price.OutputPerM <= 0) { bad("model %s: non-local models require non-zero input/output prices", name) } validateCapabilities(bad, "model "+name, mod.Capabilities) if why := m.echoMineViolation(name); why != "" { bad("model %s on echo-prone provider %s: %s — this re-arms the DeepSeek echo mine "+ "(the provider echoes the untranslated CJK source when thinking is OFF; PROGRESS «Ответ Полигону», §2); "+ "an echo-prone provider MUST keep thinking at its default (reasoning.control none|mandatory, and no "+ "thinking-disable extra_body) so a silent refusal is never shipped to prod", name, mod.Provider, why) } } if len(problems) > 0 { return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - ")) } return &m, nil } // Prices builds the ledger pricer from the config. Besides the model entries, // each LOCAL provider's own backend tag is registered at $0: адаптер локальной // ноги отдаёт LLMResponse.Model = свой тег (huihui_ai/qwen3-…), и без этой // записи PriceFor свалился бы на платный default-якорь — бесплатные локальные // вызовы книжились бы по облачной цене, фантомно съедая потолки. func (m *Models) Prices() (*ledger.Pricer, error) { prices := make(map[string]ledger.ModelPrice, len(m.Models)+len(m.Providers)) for _, p := range m.Providers { if p.Kind == "local" && p.Model != "" { prices[p.Model] = ledger.ModelPrice{} } } for name, mod := range m.Models { prices[name] = mod.Price.ToLedger() } return ledger.NewPricer(prices, m.Models[m.DefaultModel].Price.ToLedger()) } // providerPermissive reports whether a model's provider is marked permissive // (channel-B-eligible, D4.1). Unknown model → false (fail closed: an undefined // model is never treated as permissive). func (m *Models) providerPermissive(modelName string) bool { mod, ok := m.Models[modelName] if !ok { return false } return m.Providers[mod.Provider].Permissive } // providerReasoning returns a model's provider billing semantic ("subset" | "additive" | ""; // D3/D6.2). Unknown model → "" (no additive assumption). func (m *Models) providerReasoning(modelName string) string { mod, ok := m.Models[modelName] if !ok { return "" } return m.Providers[mod.Provider].Reasoning } // AdditiveReasoningTokens is the reasoning-token buffer to ADD to a reservation for a call to // modelName at effort `reasoning` (D13.6): the stage's declared reasoning_max_tokens when the // provider bills reasoning ADDITIVELY (xAI — on top of completion) AND the effort MAY think, // else 0 (a subset provider folds reasoning into maxTokens). "May think" = any effort except an // explicit "off": low|medium|high think, and an OMITTED/"" effort falls to the provider default // (xAI-grok = thinking ON) — so "" must reserve the buffer too (self-review finding #4). The // runner passes this to ledger.EstimateUSD so the reservation covers the overshoot instead of // blinding the ceiling. LoadPipeline fail-fasts if the buffer is missing for an additive // non-"off" stage, so a >0 declaredBuffer is guaranteed here for that case. func (m *Models) AdditiveReasoningTokens(modelName, reasoning string, declaredBuffer int) int { if declaredBuffer <= 0 || reasoning == "off" { return 0 } if m.providerReasoning(modelName) == "additive" { return declaredBuffer } return 0 } // APIKey resolves a provider's key from the environment ("" if the provider // declares no env var — the local backend). func (p Provider) APIKey() string { if p.APIKeyEnv == "" { return "" } return os.Getenv(p.APIKeyEnv) } // ResolveCapability computes the wire shape for a model (D3.1): the per-kind // baseline, then the provider-default capabilities, then the per-model override // (model wins, field-by-field). The runner uses the SAME method to fold the // capability into the job snapshot, so what is hashed always matches what is // sent — editing a capability invalidates checkpoints instead of false-hitting. func (m *Models) ResolveCapability(modelName string) llm.Capability { mod := m.Models[modelName] prov := m.Providers[mod.Provider] resolved := baselineCapability(prov.Kind) applyCapConfig(&resolved, prov.Capabilities) applyCapConfig(&resolved, mod.Capabilities) return resolved } // thinkingControlExtraKeys are keys whose purpose is to toggle a provider's thinking // on the wire — top-level (GLM/DeepSeek {"thinking":…}, Qwen {"enable_thinking":…}, a // raw {"reasoning_effort":…}) OR nested (DeepSeek-V3.1+ disables via // chat_template_kwargs.thinking:false). On an echo-prone provider their mere PRESENCE, // at any depth, is the mine, whatever the value: thinking must stay at the provider // default, and these keys exist only to move it off that default. var thinkingControlExtraKeys = map[string]bool{ "thinking": true, "enable_thinking": true, "reasoning_effort": true, "reasoning": true, } // findThinkingControlKey RECURSIVELY scans an extra_body value for any thinking-control // key, returning its dotted path (e.g. "chat_template_kwargs.thinking") or "". The // recursion is what catches nested disable forms a flat top-level check would miss // (external-review finding — chat_template_kwargs.thinking is DeepSeek's documented // V3.1+ disable, the most likely real arming form). func findThinkingControlKey(v any, path string) string { m, ok := v.(map[string]any) if !ok { return "" } for k, sub := range m { here := k if path != "" { here = path + "." + k } if thinkingControlExtraKeys[k] { return here } if found := findThinkingControlKey(sub, here); found != "" { return found } } return "" } // echoMineViolation reports, for one model, whether its RESOLVED wire shape would // suppress thinking at reasoning=off on an echo-prone provider — the DeepSeek echo // mine (§2). It returns a human cause string, or "" when safe. It covers both injection // surfaces: the capabilities.reasoning switch (extra_body_disable merges a // thinking-disable at off; effort sends reasoning_effort at off — both push thinking // off the default) AND a thinking-control key anywhere in the model's extra_body, // TOP-LEVEL OR NESTED (merged into the wire body by openAIRequest). A non-echo-prone // provider (GLM: its editor input is a Russian draft, nothing CJK to echo) is never // gated, so its {thinking:{type:disabled}} stays legal. func (m *Models) echoMineViolation(name string) string { mod := m.Models[name] prov, ok := m.Providers[mod.Provider] if !ok || !prov.EchoesWhenThinkingOff { return "" } switch m.ResolveCapability(name).Reasoning.Control { case llm.ReasoningExtraBodyDisable: return "capabilities.reasoning.control=extra_body_disable would merge a thinking-disable at reasoning=off" case llm.ReasoningEffortField: return "capabilities.reasoning.control=effort would send reasoning_effort at reasoning=off, suppressing thinking" } if path := findThinkingControlKey(map[string]any(mod.ExtraBody), ""); path != "" { return fmt.Sprintf("extra_body carries the thinking-control key %q", path) } return "" } // baselineCapability is the per-kind default before any capabilities block: the // OpenAI-compat wire (max_tokens + send temperature), with reasoning off by // omission for cloud and by the ollama "none" switch for local. func baselineCapability(kind string) llm.Capability { c := llm.Capability{Budget: llm.BudgetMaxTokens, Temp: llm.TempSend} if kind == "local" { c.Reasoning = llm.ReasoningCap{Control: llm.ReasoningEffortField, OffEffort: "none"} } else { c.Reasoning = llm.ReasoningCap{Control: llm.ReasoningNone} } return c } // applyCapConfig layers a capabilities block over a resolved capability. Each // top-level field (budget / temperature / reasoning) inherits when the block // leaves it empty, so a model overrides only what it declares. func applyCapConfig(c *llm.Capability, cfg *CapabilitiesConfig) { if cfg == nil { return } if cfg.BudgetField != "" { c.Budget = llm.BudgetField(cfg.BudgetField) } if cfg.Temperature != nil { c.Temp = llm.TempMode(cfg.Temperature.Mode) c.TempValue = cfg.Temperature.Value } if cfg.Reasoning != nil { c.Reasoning = llm.ReasoningCap{ Control: llm.ReasoningControl(cfg.Reasoning.Control), OffEffort: cfg.Reasoning.OffEffort, OffExtraBody: cfg.Reasoning.OffExtraBody, OnExtraBody: cfg.Reasoning.OnExtraBody, } } } // validateCapabilities fail-fasts a capabilities block's enum fields (part of // the models.yaml problem list, so a typo is caught at load, not as a 4xx). func validateCapabilities(bad func(string, ...any), where string, c *CapabilitiesConfig) { if c == nil { return } switch c.BudgetField { case "", "max_tokens", "max_completion_tokens": default: bad("%s: capabilities.budget_field must be max_tokens|max_completion_tokens, got %q", where, c.BudgetField) } if c.Temperature != nil { switch c.Temperature.Mode { case "send", "omit", "force": default: bad("%s: capabilities.temperature.mode must be send|omit|force, got %q", where, c.Temperature.Mode) } } if c.Reasoning != nil { switch c.Reasoning.Control { case "none", "effort", "extra_body_disable", "mandatory": default: bad("%s: capabilities.reasoning.control must be none|effort|extra_body_disable|mandatory, got %q", where, c.Reasoning.Control) } // The control must carry the companion field it consumes, else the // disable silently no-ops at runtime (a typo'd off_extra_body key is // dropped by non-strict yaml → nil → thinking stays ON). Catch it at // load, not as a paid-but-empty completion. if c.Reasoning.Control == "extra_body_disable" && len(c.Reasoning.OffExtraBody) == 0 { bad("%s: capabilities.reasoning.control=extra_body_disable requires a non-empty off_extra_body (the disable switch)", where) } if c.Reasoning.Control == "effort" && c.Reasoning.OffEffort == "" { bad("%s: capabilities.reasoning.control=effort requires off_effort (the value sent when reasoning=off)", where) } } } // CheckKeys verifies that every non-local model the pipeline can ACTUALLY call // has its provider API key resolvable in the environment — preflight fail-fast, // чтобы пустой ключ не всплывал 401-м уже ПОСЛЕ reserve/списания слота. // // Проверяются только достижимые модели, не весь Р4-стек: // - модели СТАДИЙ — всегда (зовутся на каждом чанке); // - модели ЭСКАЛАЦИОННЫХ цепочек — ТОЛЬКО если включён хоть один гейт: // эскалация запускается исключительно по провалу гейта, поэтому при // gates.*.enabled=false (Фаза 0) она недостижима и её ключи не нужны. // Без этого условия перевод одного чанка (DeepSeek+GLM) требовал бы ещё // ключи Anthropic/Kimi из дефолтных цепочек, которые в Фазе 0 не зовутся. // // local-only прогон облачных ключей не требует. func (m *Models) CheckKeys(pipe *Pipeline) error { needed := map[string]struct{}{} for _, st := range pipe.Stages { needed[st.Model] = struct{}{} } // Escalation models — named chains (шаг 7) and single-hop fallbacks (Веха 2.5) — // are reachable ONLY once an escalation budget is set (opt-in), NOT when a gate is // enabled: the coverage gate is a deterministic runner-side check that calls no // model, so enabling it must not demand escalation keys (self-review finding — // otherwise turning on the gate blocks a valid deepseek+glm run on a missing Kimi // key from an unused chain stub). if pipe.Escal.BudgetUSD > 0 { for _, chain := range pipe.Escal.Chains { for _, mdl := range chain { needed[mdl] = struct{}{} } } for _, st := range pipe.Stages { if st.EscalateTo != "" { needed[st.EscalateTo] = struct{}{} } } } var problems []string seen := map[string]bool{} // dedupe per provider for name := range needed { mod, ok := m.Models[name] if !ok { continue // существование уже проверено LoadPipeline } prov := m.Providers[mod.Provider] if prov.Kind == "local" || prov.APIKeyEnv == "" || seen[mod.Provider] { continue } seen[mod.Provider] = true if prov.APIKey() == "" { problems = append(problems, fmt.Sprintf("provider %s: env %s не задан (нужен для модели %s)", mod.Provider, prov.APIKeyEnv, name)) } } if len(problems) > 0 { return fmt.Errorf("отсутствуют API-ключи (заполните backend/.env):\n - %s", strings.Join(problems, "\n - ")) } return nil }