textmachine/backend/internal/config/models.go

// Package config loads and fail-fast-validates the three YAML files of a run:
// models.yaml (провайдеры/модели/цены/таймауты — Р4/Р5: цены только в конфиге
// с датой проверки), pipeline-*.yaml (ядро C1/C2 — граница «конфиг vs код» по
// Р2) and book.yaml (translation brief + brief_hash).
//
// Дисциплина донора: все проблемы конфига собираются СПИСКОМ и валят старт
// одной ошибкой — оператор чинит всё сразу, а не по одной кочке за запуск.
package config

import (
	"fmt"
	"os"
	"strings"
	"time"

	"gopkg.in/yaml.v3"

	"textmachine/backend/internal/ledger"
	"textmachine/backend/internal/llm"
)

// Models is the parsed models.yaml.
type Models struct {
	// PricesChecked is the date the prices below were verified against the
	// providers' official pages. Fail-fast rejects a stale (>120 дней) config:
	// волатильность цен API — риск Р10 №5.
	PricesChecked string              `yaml:"prices_checked"`
	DefaultModel  string              `yaml:"default_model"`
	Providers     map[string]Provider `yaml:"providers"`
	Models        map[string]Model    `yaml:"models"`
}

// Provider is one backend endpoint.
type Provider struct {
	Kind      string `yaml:"kind"` // openai | anthropic | local
	BaseURL   string `yaml:"base_url"`
	APIKeyEnv string `yaml:"api_key_env"`
	Reasoning string `yaml:"reasoning"` // subset | additive | additive_total (openai kind only)
	// EchoesWhenThinkingOff marks a provider empirically shown to ECHO the
	// untranslated CJK source instead of translating when thinking is OFF
	// (DeepSeek, traced 2026-07-04 — reproducibly on dense CJK, 3/3 retries;
	// PROGRESS «Ответ Полигону», BACKEND_SESSION_PROMPT_SILENT_REFUSALS §2). Its
	// benign state is held by an off-by-omission: reasoning="off" resolves through
	// ReasoningNone which emits NOTHING thinking-related, so the provider default
	// (thinking ON) stands. A config that DISABLES thinking at reasoning=off on
	// such a provider re-arms a silent refusal (echo = HTTP 200, no translation)
	// in prod. LoadModels (echoMineViolation) fail-fasts on exactly that — the
	// regression gate §2 mandates. It distinguishes the SAFE GLM editor, whose
	// thinking:disabled is fine because its input is a Russian draft (no CJK to
	// echo): GLM's provider carries no such flag. Validation-only metadata — it
	// never reaches the wire, so it is NOT part of the snapshot (no --resnapshot).
	EchoesWhenThinkingOff bool `yaml:"echoes_when_thinking_off"`
	// Permissive marks a provider allowed to serve the 18+ channel B (the most
	// permissive large APIs: xAI/Grok, DeepSeek official, local abliterated — D3).
	// It is the TYPE that enforces channel-B isolation (D4.1): a channel=adult stage
	// may only run on, and escalate to, a permissive provider — a structural gate,
	// not a comment, so an SFW-only provider can never silently serve/answer a
	// permissive chunk. Validation-only metadata (never on the wire, not in snapshot).
	Permissive bool   `yaml:"permissive"`
	CacheTTL   string `yaml:"cache_ttl"` // anthropic kind: "", "5m", "1h"
	Model      string `yaml:"model"`     // local kind: the backend's own tag
	MaxTokens  int    `yaml:"max_tokens"`
	// Temperature (local kind): override запроса — ollama чтит request-
	// температуру поверх Modelfile, и температура облачной роли молча
	// расстроила бы локальную модель. 0 = наследовать запрос.
	Temperature float64  `yaml:"temperature"`
	Timeouts    Timeouts `yaml:"timeouts"`
	// Capabilities is the DEFAULT wire shape for this provider's models (D3.1);
	// a per-Model block overrides it field-by-field. Nil = OpenAI-compat
	// baseline (max_tokens + send temperature + off-by-omission reasoning).
	Capabilities *CapabilitiesConfig `yaml:"capabilities"`
}

// CapabilitiesConfig is the declarative per-model wire shape (D3.1). Empty
// fields inherit: a Model block layers over its Provider block, which layers
// over the kind baseline. Separate from Provider.Reasoning (subset|additive),
// which is a billing semantic — this is the wire body form.
type CapabilitiesConfig struct {
	BudgetField string           `yaml:"budget_field"` // max_tokens | max_completion_tokens
	Temperature *TemperatureCap  `yaml:"temperature"`
	Reasoning   *ReasoningCapCfg `yaml:"reasoning"`
}

// TemperatureCap declares how temperature reaches the wire.
type TemperatureCap struct {
	Mode  string  `yaml:"mode"`  // send | omit | force
	Value float64 `yaml:"value"` // used when mode == force
}

// ReasoningCapCfg declares how the neutral reasoning effort maps to the wire.
type ReasoningCapCfg struct {
	Control      string         `yaml:"control"`        // none | effort | extra_body_disable | mandatory
	OffEffort    string         `yaml:"off_effort"`     // effort: value sent when reasoning=off
	OffExtraBody map[string]any `yaml:"off_extra_body"` // extra_body_disable: merged when reasoning=off
	OnExtraBody  map[string]any `yaml:"on_extra_body"`  // extra_body_disable: merged when reasoning is on (D6.2)
}

// Timeouts is the retry profile per provider (профиль per-провайдер из
// вердикта валидации; per-role overrides — Фаза 1).
type Timeouts struct {
	AttemptS    int `yaml:"attempt_s"`
	MaxAttempts int `yaml:"max_attempts"`
	BackoffCapS int `yaml:"backoff_cap_s"`
}

func (t Timeouts) Profile() llm.RetryProfile {
	return llm.RetryProfile{
		AttemptTimeout: time.Duration(t.AttemptS) * time.Second,
		MaxAttempts:    t.MaxAttempts,
		BackoffCap:     time.Duration(t.BackoffCapS) * time.Second,
	}
}

// Model is one priced model entry.
type Model struct {
	Provider string `yaml:"provider"`
	Price    Price  `yaml:"price"`
	// ExtraBody is merged into request JSON for this model (провайдер-
	// специфичные ручки: GLM thinking.type, Qwen enable_thinking …).
	ExtraBody map[string]any `yaml:"extra_body"`
	// MinCachePrefixTokens: Anthropic-модели — минимальный кэшируемый префикс
	// (короче — кэш молча не создаётся); сборщик контекста Фазы 1 сверяется.
	MinCachePrefixTokens int    `yaml:"min_cache_prefix_tokens"`
	Note                 string `yaml:"note"`
	// Capabilities overrides this model's provider-default wire shape (D3.1),
	// field-by-field (model wins). Nil = inherit the provider/kind baseline.
	Capabilities *CapabilitiesConfig `yaml:"capabilities"`
}

// Price mirrors ledger.ModelPrice in YAML form (USD per 1M tokens).
type Price struct {
	InputPerM      float64 `yaml:"input_per_m"`
	CachedPerM     float64 `yaml:"cached_per_m"`
	CacheWritePerM float64 `yaml:"cache_write_per_m"`
	OutputPerM     float64 `yaml:"output_per_m"`
}

func (p Price) ToLedger() ledger.ModelPrice {
	return ledger.ModelPrice{
		InputPerM:      p.InputPerM,
		CachedPerM:     p.CachedPerM,
		CacheWritePerM: p.CacheWritePerM,
		OutputPerM:     p.OutputPerM,
	}
}

// maxPriceAge is how stale prices_checked may be before the config is
// rejected (ежеквартальный пересмотр Р4 + буфер).
const maxPriceAge = 120 * 24 * time.Hour

// LoadModels reads and validates models.yaml.
func LoadModels(path string) (*Models, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("config: read %s: %w", path, err)
	}
	var m Models
	if err := yaml.Unmarshal(raw, &m); err != nil {
		return nil, fmt.Errorf("config: parse %s: %w", path, err)
	}

	var problems []string
	bad := func(format string, a ...any) { problems = append(problems, fmt.Sprintf(format, a...)) }

	if m.PricesChecked == "" {
		bad("prices_checked is required (цены без даты проверки запрещены Р4)")
	} else if t, err := time.Parse("2006-01-02", m.PricesChecked); err != nil {
		bad("prices_checked %q is not YYYY-MM-DD", m.PricesChecked)
	} else if time.Since(t) > maxPriceAge {
		bad("prices_checked %s is older than %d days — re-verify provider prices", m.PricesChecked, int(maxPriceAge.Hours()/24))
	}
	if m.DefaultModel == "" {
		bad("default_model is required (fallback-якорь цены: неизвестная модель никогда не стоит $0)")
	} else if _, ok := m.Models[m.DefaultModel]; !ok {
		bad("default_model %q is not defined in models", m.DefaultModel)
	}
	for name, p := range m.Providers {
		switch p.Kind {
		case "openai", "anthropic", "local":
		default:
			bad("provider %s: unknown kind %q", name, p.Kind)
		}
		if p.BaseURL == "" && p.Kind != "anthropic" {
			bad("provider %s: base_url is required", name)
		}
		if p.Kind == "openai" && p.Reasoning != "" && p.Reasoning != "subset" && p.Reasoning != "additive" && p.Reasoning != "additive_total" {
			bad("provider %s: reasoning must be subset|additive|additive_total, got %q", name, p.Reasoning)
		}
		if p.Kind == "anthropic" && p.CacheTTL != "" && p.CacheTTL != "5m" && p.CacheTTL != "1h" {
			bad("provider %s: cache_ttl must be 5m|1h, got %q", name, p.CacheTTL)
		}
		if p.Kind == "local" && p.Model == "" {
			bad("provider %s: local kind requires model (its own tag)", name)
		}
		validateCapabilities(bad, "provider "+name, p.Capabilities)
	}
	for name, mod := range m.Models {
		prov, ok := m.Providers[mod.Provider]
		if !ok {
			bad("model %s: provider %q is not defined", name, mod.Provider)
			continue
		}
		isLocal := prov.Kind == "local"
		if !isLocal && (mod.Price.InputPerM <= 0 || mod.Price.OutputPerM <= 0) {
			bad("model %s: non-local models require non-zero input/output prices", name)
		}
		validateCapabilities(bad, "model "+name, mod.Capabilities)
		if why := m.echoMineViolation(name); why != "" {
			bad("model %s on echo-prone provider %s: %s — this re-arms the DeepSeek echo mine "+
				"(the provider echoes the untranslated CJK source when thinking is OFF; PROGRESS «Ответ Полигону», §2); "+
				"an echo-prone provider MUST keep thinking at its default (reasoning.control none|mandatory, and no "+
				"thinking-disable extra_body) so a silent refusal is never shipped to prod", name, mod.Provider, why)
		}
	}
	if len(problems) > 0 {
		return nil, fmt.Errorf("config %s:\n  - %s", path, strings.Join(problems, "\n  - "))
	}
	return &m, nil
}

// Prices builds the ledger pricer from the config. Besides the model entries,
// each LOCAL provider's own backend tag is registered at $0: адаптер локальной
// ноги отдаёт LLMResponse.Model = свой тег (huihui_ai/qwen3-…), и без этой
// записи PriceFor свалился бы на платный default-якорь — бесплатные локальные
// вызовы книжились бы по облачной цене, фантомно съедая потолки.
func (m *Models) Prices() (*ledger.Pricer, error) {
	prices := make(map[string]ledger.ModelPrice, len(m.Models)+len(m.Providers))
	for _, p := range m.Providers {
		if p.Kind == "local" && p.Model != "" {
			prices[p.Model] = ledger.ModelPrice{}
		}
	}
	for name, mod := range m.Models {
		prices[name] = mod.Price.ToLedger()
	}
	return ledger.NewPricer(prices, m.Models[m.DefaultModel].Price.ToLedger())
}

// providerPermissive reports whether a model's provider is marked permissive
// (channel-B-eligible, D4.1). Unknown model → false (fail closed: an undefined
// model is never treated as permissive).
func (m *Models) providerPermissive(modelName string) bool {
	mod, ok := m.Models[modelName]
	if !ok {
		return false
	}
	return m.Providers[mod.Provider].Permissive
}

// providerReasoning returns a model's provider billing semantic ("subset" | "additive" | "";
// D3/D6.2). Unknown model → "" (no additive assumption).
func (m *Models) providerReasoning(modelName string) string {
	mod, ok := m.Models[modelName]
	if !ok {
		return ""
	}
	return m.Providers[mod.Provider].Reasoning
}

// AdditiveReasoningTokens is the reasoning-token buffer to ADD to a reservation for a call to
// modelName at effort `reasoning` (D13.6): the stage's declared reasoning_max_tokens when the
// provider bills reasoning ADDITIVELY (xAI — on top of completion) AND the effort MAY think,
// else 0 (a subset provider folds reasoning into maxTokens). "May think" = any effort except an
// explicit "off": low|medium|high think, and an OMITTED/"" effort falls to the provider default
// (xAI-grok = thinking ON) — so "" must reserve the buffer too (self-review finding #4). The
// runner passes this to ledger.EstimateUSD so the reservation covers the overshoot instead of
// blinding the ceiling. LoadPipeline fail-fasts if the buffer is missing for an additive
// non-"off" stage, so a >0 declaredBuffer is guaranteed here for that case.
func (m *Models) AdditiveReasoningTokens(modelName, reasoning string, declaredBuffer int) int {
	if declaredBuffer <= 0 || reasoning == "off" {
		return 0
	}
	if m.providerReasoning(modelName) == "additive" {
		return declaredBuffer
	}
	return 0
}

// APIKey resolves a provider's key from the environment ("" if the provider
// declares no env var — the local backend).
func (p Provider) APIKey() string {
	if p.APIKeyEnv == "" {
		return ""
	}
	return os.Getenv(p.APIKeyEnv)
}

// ResolveCapability computes the wire shape for a model (D3.1): the per-kind
// baseline, then the provider-default capabilities, then the per-model override
// (model wins, field-by-field). The runner uses the SAME method to fold the
// capability into the job snapshot, so what is hashed always matches what is
// sent — editing a capability invalidates checkpoints instead of false-hitting.
func (m *Models) ResolveCapability(modelName string) llm.Capability {
	mod := m.Models[modelName]
	prov := m.Providers[mod.Provider]
	resolved := baselineCapability(prov.Kind)
	applyCapConfig(&resolved, prov.Capabilities)
	applyCapConfig(&resolved, mod.Capabilities)
	return resolved
}

// thinkingControlExtraKeys are keys whose purpose is to toggle a provider's thinking
// on the wire — top-level (GLM/DeepSeek {"thinking":…}, Qwen {"enable_thinking":…}, a
// raw {"reasoning_effort":…}) OR nested (DeepSeek-V3.1+ disables via
// chat_template_kwargs.thinking:false). On an echo-prone provider their mere PRESENCE,
// at any depth, is the mine, whatever the value: thinking must stay at the provider
// default, and these keys exist only to move it off that default.
var thinkingControlExtraKeys = map[string]bool{
	"thinking": true, "enable_thinking": true, "reasoning_effort": true, "reasoning": true,
}

// findThinkingControlKey RECURSIVELY scans an extra_body value for any thinking-control
// key, returning its dotted path (e.g. "chat_template_kwargs.thinking") or "". The
// recursion is what catches nested disable forms a flat top-level check would miss
// (external-review finding — chat_template_kwargs.thinking is DeepSeek's documented
// V3.1+ disable, the most likely real arming form).
func findThinkingControlKey(v any, path string) string {
	m, ok := v.(map[string]any)
	if !ok {
		return ""
	}
	for k, sub := range m {
		here := k
		if path != "" {
			here = path + "." + k
		}
		if thinkingControlExtraKeys[k] {
			return here
		}
		if found := findThinkingControlKey(sub, here); found != "" {
			return found
		}
	}
	return ""
}

// echoMineViolation reports, for one model, whether its RESOLVED wire shape would
// suppress thinking at reasoning=off on an echo-prone provider — the DeepSeek echo
// mine (§2). It returns a human cause string, or "" when safe. It covers both injection
// surfaces: the capabilities.reasoning switch (extra_body_disable merges a
// thinking-disable at off; effort sends reasoning_effort at off — both push thinking
// off the default) AND a thinking-control key anywhere in the model's extra_body,
// TOP-LEVEL OR NESTED (merged into the wire body by openAIRequest). A non-echo-prone
// provider (GLM: its editor input is a Russian draft, nothing CJK to echo) is never
// gated, so its {thinking:{type:disabled}} stays legal.
func (m *Models) echoMineViolation(name string) string {
	mod := m.Models[name]
	prov, ok := m.Providers[mod.Provider]
	if !ok || !prov.EchoesWhenThinkingOff {
		return ""
	}
	switch m.ResolveCapability(name).Reasoning.Control {
	case llm.ReasoningExtraBodyDisable:
		return "capabilities.reasoning.control=extra_body_disable would merge a thinking-disable at reasoning=off"
	case llm.ReasoningEffortField:
		return "capabilities.reasoning.control=effort would send reasoning_effort at reasoning=off, suppressing thinking"
	}
	if path := findThinkingControlKey(map[string]any(mod.ExtraBody), ""); path != "" {
		return fmt.Sprintf("extra_body carries the thinking-control key %q", path)
	}
	return ""
}

// baselineCapability is the per-kind default before any capabilities block: the
// OpenAI-compat wire (max_tokens + send temperature), with reasoning off by
// omission for cloud and by the ollama "none" switch for local.
func baselineCapability(kind string) llm.Capability {
	c := llm.Capability{Budget: llm.BudgetMaxTokens, Temp: llm.TempSend}
	if kind == "local" {
		c.Reasoning = llm.ReasoningCap{Control: llm.ReasoningEffortField, OffEffort: "none"}
	} else {
		c.Reasoning = llm.ReasoningCap{Control: llm.ReasoningNone}
	}
	return c
}

// applyCapConfig layers a capabilities block over a resolved capability. Each
// top-level field (budget / temperature / reasoning) inherits when the block
// leaves it empty, so a model overrides only what it declares.
func applyCapConfig(c *llm.Capability, cfg *CapabilitiesConfig) {
	if cfg == nil {
		return
	}
	if cfg.BudgetField != "" {
		c.Budget = llm.BudgetField(cfg.BudgetField)
	}
	if cfg.Temperature != nil {
		c.Temp = llm.TempMode(cfg.Temperature.Mode)
		c.TempValue = cfg.Temperature.Value
	}
	if cfg.Reasoning != nil {
		c.Reasoning = llm.ReasoningCap{
			Control:      llm.ReasoningControl(cfg.Reasoning.Control),
			OffEffort:    cfg.Reasoning.OffEffort,
			OffExtraBody: cfg.Reasoning.OffExtraBody,
			OnExtraBody:  cfg.Reasoning.OnExtraBody,
		}
	}
}

// validateCapabilities fail-fasts a capabilities block's enum fields (part of
// the models.yaml problem list, so a typo is caught at load, not as a 4xx).
func validateCapabilities(bad func(string, ...any), where string, c *CapabilitiesConfig) {
	if c == nil {
		return
	}
	switch c.BudgetField {
	case "", "max_tokens", "max_completion_tokens":
	default:
		bad("%s: capabilities.budget_field must be max_tokens|max_completion_tokens, got %q", where, c.BudgetField)
	}
	if c.Temperature != nil {
		switch c.Temperature.Mode {
		case "send", "omit", "force":
		default:
			bad("%s: capabilities.temperature.mode must be send|omit|force, got %q", where, c.Temperature.Mode)
		}
	}
	if c.Reasoning != nil {
		switch c.Reasoning.Control {
		case "none", "effort", "extra_body_disable", "mandatory":
		default:
			bad("%s: capabilities.reasoning.control must be none|effort|extra_body_disable|mandatory, got %q", where, c.Reasoning.Control)
		}
		// The control must carry the companion field it consumes, else the
		// disable silently no-ops at runtime (a typo'd off_extra_body key is
		// dropped by non-strict yaml → nil → thinking stays ON). Catch it at
		// load, not as a paid-but-empty completion.
		if c.Reasoning.Control == "extra_body_disable" && len(c.Reasoning.OffExtraBody) == 0 {
			bad("%s: capabilities.reasoning.control=extra_body_disable requires a non-empty off_extra_body (the disable switch)", where)
		}
		if c.Reasoning.Control == "effort" && c.Reasoning.OffEffort == "" {
			bad("%s: capabilities.reasoning.control=effort requires off_effort (the value sent when reasoning=off)", where)
		}
	}
}

// CheckKeys verifies that every non-local model the pipeline can ACTUALLY call
// has its provider API key resolvable in the environment — preflight fail-fast,
// чтобы пустой ключ не всплывал 401-м уже ПОСЛЕ reserve/списания слота.
//
// Проверяются только достижимые модели, не весь Р4-стек:
//   - модели СТАДИЙ — всегда (зовутся на каждом чанке);
//   - модели ЭСКАЛАЦИОННЫХ цепочек — ТОЛЬКО если включён хоть один гейт:
//     эскалация запускается исключительно по провалу гейта, поэтому при
//     gates.*.enabled=false (Фаза 0) она недостижима и её ключи не нужны.
//     Без этого условия перевод одного чанка (DeepSeek+GLM) требовал бы ещё
//     ключи Anthropic/Kimi из дефолтных цепочек, которые в Фазе 0 не зовутся.
//
// local-only прогон облачных ключей не требует.
func (m *Models) CheckKeys(pipe *Pipeline) error {
	needed := map[string]struct{}{}
	for _, st := range pipe.Stages {
		needed[st.Model] = struct{}{}
	}
	// Escalation models — named chains (шаг 7) and single-hop fallbacks (Веха 2.5) —
	// are reachable ONLY once an escalation budget is set (opt-in), NOT when a gate is
	// enabled: the coverage gate is a deterministic runner-side check that calls no
	// model, so enabling it must not demand escalation keys (self-review finding —
	// otherwise turning on the gate blocks a valid deepseek+glm run on a missing Kimi
	// key from an unused chain stub).
	if pipe.Escal.BudgetUSD > 0 {
		for _, chain := range pipe.Escal.Chains {
			for _, mdl := range chain {
				needed[mdl] = struct{}{}
			}
		}
		for _, st := range pipe.Stages {
			if st.EscalateTo != "" {
				needed[st.EscalateTo] = struct{}{}
			}
		}
	}
	var problems []string
	seen := map[string]bool{} // dedupe per provider
	for name := range needed {
		mod, ok := m.Models[name]
		if !ok {
			continue // существование уже проверено LoadPipeline
		}
		prov := m.Providers[mod.Provider]
		if prov.Kind == "local" || prov.APIKeyEnv == "" || seen[mod.Provider] {
			continue
		}
		seen[mod.Provider] = true
		if prov.APIKey() == "" {
			problems = append(problems, fmt.Sprintf("provider %s: env %s не задан (нужен для модели %s)", mod.Provider, prov.APIKeyEnv, name))
		}
	}
	if len(problems) > 0 {
		return fmt.Errorf("отсутствуют API-ключи (заполните backend/.env):\n  - %s", strings.Join(problems, "\n  - "))
	}
	return nil
}