From 737d4c6a68af9ee0f6919e6f3ac3b6d7e209d1bf Mon Sep 17 00:00:00 2001 From: "Claude (backend session)" Date: Sat, 4 Jul 2026 22:31:49 +0300 Subject: [PATCH] Fail-fast on any config that disables DeepSeek thinking at reasoning=off, the echo-mine regression gate that keeps thinking on by default --- backend/configs/models.yaml | 8 ++ backend/internal/config/echo_mine_test.go | 157 ++++++++++++++++++++++ backend/internal/config/models.go | 72 ++++++++-- docs/PROGRESS.md | 12 ++ 4 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 backend/internal/config/echo_mine_test.go diff --git a/backend/configs/models.yaml b/backend/configs/models.yaml index 6233429..17c7d17 100644 --- a/backend/configs/models.yaml +++ b/backend/configs/models.yaml @@ -20,6 +20,14 @@ providers: base_url: https://api.deepseek.com/v1 api_key_env: DEEPSEEK_API_KEY reasoning: subset # thinking внутри completion_tokens + # ⚠️ ЭХО-МИНА (трасса 2026-07-04, PROGRESS «Ответ Полигону»): на плотном CJK + # DeepSeek с thinking ВЫКЛ воспроизводимо возвращает ИСХОДНИК вместо перевода + # (тихий отказ, HTTP 200). Держится только тем, что reasoning:"off" через + # ReasoningNone — осознанный no-op (thinking остаётся ON по дефолту). Флаг + # включает fail-fast (config.echoMineViolation): нельзя завести deepseek + # capabilities.reasoning.off_extra_body:{thinking:{type:disabled}} — включит + # эхо в проде. GLM с thinking:disabled безопасен (вход — русский черновик). + echoes_when_thinking_off: true timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60 } zai: # GLM, международный контур (docs.z.ai) diff --git a/backend/internal/config/echo_mine_test.go b/backend/internal/config/echo_mine_test.go new file mode 100644 index 0000000..e4641f0 --- /dev/null +++ b/backend/internal/config/echo_mine_test.go @@ -0,0 +1,157 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gopkg.in/yaml.v3" + + "textmachine/backend/internal/llm" +) + +// echo_mine_test.go is the §2 regression gate for the DeepSeek echo mine: on a +// provider empirically shown to ECHO the untranslated CJK source when thinking is +// OFF (traced 2026-07-04), no model may be configured to disable/suppress thinking +// at reasoning=off — that re-arms a silent refusal (HTTP 200, no translation) in +// prod. LoadModels must fail-fast on both injection surfaces, while leaving the +// SAFE GLM editor (thinking:disabled over a Russian draft) untouched. +// +// Mutation check (how this guards against a future edit re-arming the mine): +// delete the echoMineViolation call in LoadModels (or the field) and the four +// "armed" cases below stop failing — the test goes red exactly when the guard is +// removed. + +// writeAndLoadModels materializes a models.yaml body and runs it through the real +// LoadModels validation (mirrors the config_test.go temp-file pattern). +func writeAndLoadModels(t *testing.T, body string) (*Models, error) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "models.yaml") + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return LoadModels(path) +} + +func TestDeepSeekEchoMineRejected(t *testing.T) { + today := time.Now().UTC().Format("2006-01-02") + // provCaps is injected under provider p; modelCaps under model m. + tmpl := func(echoProne bool, modelExtra string) string { + flag := "" + if echoProne { + flag = "\n echoes_when_thinking_off: true" + } + return fmt.Sprintf(` +prices_checked: %q +default_model: m +providers: + p: + kind: openai + base_url: http://x%s +models: + m: + provider: p + price: { input_per_m: 1, output_per_m: 2 }%s +`, today, flag, modelExtra) + } + + cases := []struct { + name string + echoProne bool + modelExtra string + wantErr bool + }{ + { + // The exact traced mine: the thinking-disable switch via capabilities. + name: "armed via capabilities.reasoning.extra_body_disable", + echoProne: true, + modelExtra: "\n capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }", + wantErr: true, + }, + { + // The second surface: a thinking-disable smuggled through extra_body, + // which openAIRequest merges into the wire body — the control-only guard + // would miss this, so it must be caught too. + name: "armed via extra_body thinking key", + echoProne: true, + modelExtra: "\n extra_body: { thinking: { type: disabled } }", + wantErr: true, + }, + { + // reasoning_effort at off also pushes thinking off the default. + name: "armed via capabilities.reasoning.effort", + echoProne: true, + modelExtra: "\n capabilities: { reasoning: { control: effort, off_effort: minimal } }", + wantErr: true, + }, + { + // The SAFE GLM case: an IDENTICAL thinking-disable is legal on a provider + // that is NOT echo-prone (its editor input is a Russian draft). + name: "safe: same thinking-disable on a non-echo-prone provider", + echoProne: false, + modelExtra: "\n capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }", + wantErr: false, + }, + { + // The current benign DeepSeek state: no reasoning capability → off is a + // no-op → thinking stays at the provider default (ON). + name: "safe: echo-prone provider with the default (no) reasoning control", + echoProne: true, + modelExtra: "", + wantErr: false, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, err := writeAndLoadModels(t, tmpl(c.echoProne, c.modelExtra)) + if c.wantErr { + if err == nil || !strings.Contains(err.Error(), "echo mine") { + t.Fatalf("expected fail-fast mentioning the echo mine, got %v", err) + } + return + } + if err != nil { + t.Fatalf("expected the config to load, got %v", err) + } + }) + } +} + +// TestBoevoyConfigDeepSeekThinkingStaysOn guards the REAL configs/models.yaml +// against a future edit arming the mine. It asserts the three load-bearing facts +// directly (unmarshalling raw to dodge the prices_checked staleness date-bomb that +// LoadModels enforces): (1) the deepseek provider is still marked echo-prone — the +// guard is inert without the flag; (2) deepseek-v4-flash resolves to ReasoningNone +// so reasoning="off" stays an off-by-omission (thinking ON by default); (3) +// echoMineViolation is empty. Arm the real config and (2)/(3) go red here, while +// tmctl itself fails loudly via LoadModels. +func TestBoevoyConfigDeepSeekThinkingStaysOn(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "configs", "models.yaml")) + if err != nil { + t.Fatalf("read boevoy models.yaml: %v", err) + } + var m Models + if err := yaml.Unmarshal(raw, &m); err != nil { + t.Fatalf("parse boevoy models.yaml: %v", err) + } + + const model = "deepseek-v4-flash" + mod, ok := m.Models[model] + if !ok { + t.Fatalf("boevoy config no longer defines %s — update this guard deliberately", model) + } + prov := m.Providers[mod.Provider] + if !prov.EchoesWhenThinkingOff { + t.Fatalf("provider %s must stay marked echoes_when_thinking_off — the echo-mine guard is inert without it", mod.Provider) + } + if got := m.ResolveCapability(model).Reasoning.Control; got != llm.ReasoningNone { + t.Fatalf("%s must resolve to ReasoningNone so reasoning=off stays a no-op (thinking ON), got %q — echo mine armed", model, got) + } + if why := m.echoMineViolation(model); why != "" { + t.Fatalf("%s re-arms the echo mine: %s", model, why) + } +} diff --git a/backend/internal/config/models.go b/backend/internal/config/models.go index 6ee03dc..b5eca47 100644 --- a/backend/internal/config/models.go +++ b/backend/internal/config/models.go @@ -32,13 +32,27 @@ type Models struct { // Provider is one backend endpoint. type Provider struct { - Kind string `yaml:"kind"` // openai | anthropic | local - BaseURL string `yaml:"base_url"` - APIKeyEnv string `yaml:"api_key_env"` - Reasoning string `yaml:"reasoning"` // subset | additive (openai kind only) - CacheTTL string `yaml:"cache_ttl"` // anthropic kind: "", "5m", "1h" - Model string `yaml:"model"` // local kind: the backend's own tag - MaxTokens int `yaml:"max_tokens"` + Kind string `yaml:"kind"` // openai | anthropic | local + BaseURL string `yaml:"base_url"` + APIKeyEnv string `yaml:"api_key_env"` + Reasoning string `yaml:"reasoning"` // subset | additive (openai kind only) + // EchoesWhenThinkingOff marks a provider empirically shown to ECHO the + // untranslated CJK source instead of translating when thinking is OFF + // (DeepSeek, traced 2026-07-04 — reproducibly on dense CJK, 3/3 retries; + // PROGRESS «Ответ Полигону», BACKEND_SESSION_PROMPT_SILENT_REFUSALS §2). Its + // benign state is held by an off-by-omission: reasoning="off" resolves through + // ReasoningNone which emits NOTHING thinking-related, so the provider default + // (thinking ON) stands. A config that DISABLES thinking at reasoning=off on + // such a provider re-arms a silent refusal (echo = HTTP 200, no translation) + // in prod. LoadModels (echoMineViolation) fail-fasts on exactly that — the + // regression gate §2 mandates. It distinguishes the SAFE GLM editor, whose + // thinking:disabled is fine because its input is a Russian draft (no CJK to + // echo): GLM's provider carries no such flag. Validation-only metadata — it + // never reaches the wire, so it is NOT part of the snapshot (no --resnapshot). + EchoesWhenThinkingOff bool `yaml:"echoes_when_thinking_off"` + CacheTTL string `yaml:"cache_ttl"` // anthropic kind: "", "5m", "1h" + Model string `yaml:"model"` // local kind: the backend's own tag + MaxTokens int `yaml:"max_tokens"` // Temperature (local kind): override запроса — ollama чтит request- // температуру поверх Modelfile, и температура облачной роли молча // расстроила бы локальную модель. 0 = наследовать запрос. @@ -68,7 +82,7 @@ type TemperatureCap struct { // ReasoningCapCfg declares how the neutral reasoning effort maps to the wire. type ReasoningCapCfg struct { - Control string `yaml:"control"` // none | effort | extra_body_disable | mandatory + Control string `yaml:"control"` // none | effort | extra_body_disable | mandatory OffEffort string `yaml:"off_effort"` // effort: value sent when reasoning=off OffExtraBody map[string]any `yaml:"off_extra_body"` // extra_body_disable: merged when reasoning=off OnExtraBody map[string]any `yaml:"on_extra_body"` // extra_body_disable: merged when reasoning is on (D6.2) @@ -184,6 +198,12 @@ func LoadModels(path string) (*Models, error) { bad("model %s: non-local models require non-zero input/output prices", name) } validateCapabilities(bad, "model "+name, mod.Capabilities) + if why := m.echoMineViolation(name); why != "" { + bad("model %s on echo-prone provider %s: %s — this re-arms the DeepSeek echo mine "+ + "(the provider echoes the untranslated CJK source when thinking is OFF; PROGRESS «Ответ Полигону», §2); "+ + "an echo-prone provider MUST keep thinking at its default (reasoning.control none|mandatory, and no "+ + "thinking-disable extra_body) so a silent refusal is never shipped to prod", name, mod.Provider, why) + } } if len(problems) > 0 { return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - ")) @@ -232,6 +252,42 @@ func (m *Models) ResolveCapability(modelName string) llm.Capability { return resolved } +// thinkingControlExtraKeys are model-level extra_body keys whose sole purpose is +// to toggle a provider's thinking on the wire (GLM/DeepSeek {"thinking":…}, Qwen +// {"enable_thinking":…}, a raw {"reasoning_effort":…}). On an echo-prone provider +// their mere PRESENCE is the mine, whatever the value: thinking must stay at the +// provider default, and these keys exist only to move it off that default. +var thinkingControlExtraKeys = []string{"thinking", "enable_thinking", "reasoning_effort"} + +// echoMineViolation reports, for one model, whether its RESOLVED wire shape would +// suppress thinking at reasoning=off on an echo-prone provider — the DeepSeek echo +// mine (§2). It returns a human cause string, or "" when safe. Both injection +// surfaces are covered: the capabilities.reasoning switch (extra_body_disable +// merges a thinking-disable at off; effort sends reasoning_effort at off — both +// push thinking off the default) AND a thinking-control key smuggled through the +// model's top-level extra_body (merged into the wire body by openAIRequest). A +// non-echo-prone provider (GLM: its editor input is a Russian draft, nothing CJK +// to echo) is never gated, so its {thinking:{type:disabled}} stays legal. +func (m *Models) echoMineViolation(name string) string { + mod := m.Models[name] + prov, ok := m.Providers[mod.Provider] + if !ok || !prov.EchoesWhenThinkingOff { + return "" + } + switch m.ResolveCapability(name).Reasoning.Control { + case llm.ReasoningExtraBodyDisable: + return "capabilities.reasoning.control=extra_body_disable would merge a thinking-disable at reasoning=off" + case llm.ReasoningEffortField: + return "capabilities.reasoning.control=effort would send reasoning_effort at reasoning=off, suppressing thinking" + } + for _, k := range thinkingControlExtraKeys { + if _, present := mod.ExtraBody[k]; present { + return fmt.Sprintf("extra_body carries the thinking-control key %q", k) + } + } + return "" +} + // baselineCapability is the per-kind default before any capabilities block: the // OpenAI-compat wire (max_tokens + send temperature), with reasoning off by // omission for cloud and by the ollama "none" switch for local. diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index c7e1d34..1b0f9ee 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -289,6 +289,18 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор Фикс (эскалация `cjk_artifact`→фолбэк-черновик + coverage-гейт вырезания + live-conformance) — **отдельная сессия**, онбординг-промт: [BACKEND_SESSION_PROMPT_SILENT_REFUSALS.md](BACKEND_SESSION_PROMPT_SILENT_REFUSALS.md). Не блокирует шаг 3. +### 2026-07-04 — Сессия 5: Веха 2.5 (старт) — DeepSeek эхо-мина зафиксирована регресс-гейтом + +Валидация точки старта зелёная (`go build/vet/test -race` ~80 тестов; `tmctl report` $0). Первым делом — регресс-гейт §2, чтобы будущая правка `models.yaml` не включила эхо-мину DeepSeek. + +**Решение: fail-fast в конфиг-валидации (сильнее unit-теста — срабатывает на КАЖДОМ `tmctl`-запуске), а не только тест.** Провайдер получает декларативный маркер `echoes_when_thinking_off: true` (заведён на `deepseek`, документирован трассой). `config.echoMineViolation` резолвит каппу модели такого провайдера и валит старт, если thinking подавляется при `reasoning=off` — по ОБЕИМ поверхностям вооружения: (1) `capabilities.reasoning.control` = `extra_body_disable` (мержит `{thinking:disabled}`) или `effort` (шлёт `reasoning_effort`); (2) thinking-ключ, протащенный через model `extra_body` (его контрол-чек бы пропустил — `openAIRequest` мержит его в тело). Инвариант: echo-prone провайдер обязан держать `reasoning.control` = `none|mandatory` и без thinking-disable `extra_body` → thinking остаётся на дефолте (ON). + +**GLM отличается корректно:** маркер только на `deepseek`; `glm-5` (provider `zai`) с `{thinking:{type:disabled}}` остаётся легальным (его вход — русский черновик, CJK нечего эхать). Метаданные валидации-only — на провод не идут, в snapshot НЕ входят (без `--resnapshot`). + +**Тесты (мутационно-проверенные):** `TestDeepSeekEchoMineRejected` — 5 кейсов (2 поверхности + `effort` + GLM-безопасность + дефолт); `TestBoevoyConfigDeepSeekThinkingStaysOn` — гард на реальном `configs/models.yaml` (флаг на месте + `deepseek-v4-flash`→`ReasoningNone`; unmarshal напрямую, чтобы обойти date-бомбу `prices_checked`). Убрать `echoMineViolation` — armed-кейсы перестают падать (мутация ловится). + +Дальше по §7 — контрактные вопросы владельцу перед coverage-гейтом/эскалацией (сегментация 1:1 vs §3.7, скоуп эскалации, скоуп coverage_fail/D10, порядок). + ## Полигон (секция параллельной сессии — записи добавлять сюда)