Fail-fast on any config that disables DeepSeek thinking at reasoning=off, the echo-mine regression gate that keeps thinking on by default
This commit is contained in:
Claude (backend session)
parent
fc05b6c2e7
commit
737d4c6a68
4 changed files with 241 additions and 8 deletions
|
|
@ -20,6 +20,14 @@ providers:
|
|||
base_url: https://api.deepseek.com/v1
|
||||
api_key_env: DEEPSEEK_API_KEY
|
||||
reasoning: subset # thinking внутри completion_tokens
|
||||
# ⚠️ ЭХО-МИНА (трасса 2026-07-04, PROGRESS «Ответ Полигону»): на плотном CJK
|
||||
# DeepSeek с thinking ВЫКЛ воспроизводимо возвращает ИСХОДНИК вместо перевода
|
||||
# (тихий отказ, HTTP 200). Держится только тем, что reasoning:"off" через
|
||||
# ReasoningNone — осознанный no-op (thinking остаётся ON по дефолту). Флаг
|
||||
# включает fail-fast (config.echoMineViolation): нельзя завести deepseek
|
||||
# capabilities.reasoning.off_extra_body:{thinking:{type:disabled}} — включит
|
||||
# эхо в проде. GLM с thinking:disabled безопасен (вход — русский черновик).
|
||||
echoes_when_thinking_off: true
|
||||
timeouts: { attempt_s: 240, max_attempts: 3, backoff_cap_s: 60 }
|
||||
|
||||
zai: # GLM, международный контур (docs.z.ai)
|
||||
|
|
|
|||
157
backend/internal/config/echo_mine_test.go
Normal file
157
backend/internal/config/echo_mine_test.go
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"textmachine/backend/internal/llm"
|
||||
)
|
||||
|
||||
// echo_mine_test.go is the §2 regression gate for the DeepSeek echo mine: on a
|
||||
// provider empirically shown to ECHO the untranslated CJK source when thinking is
|
||||
// OFF (traced 2026-07-04), no model may be configured to disable/suppress thinking
|
||||
// at reasoning=off — that re-arms a silent refusal (HTTP 200, no translation) in
|
||||
// prod. LoadModels must fail-fast on both injection surfaces, while leaving the
|
||||
// SAFE GLM editor (thinking:disabled over a Russian draft) untouched.
|
||||
//
|
||||
// Mutation check (how this guards against a future edit re-arming the mine):
|
||||
// delete the echoMineViolation call in LoadModels (or the field) and the four
|
||||
// "armed" cases below stop failing — the test goes red exactly when the guard is
|
||||
// removed.
|
||||
|
||||
// writeAndLoadModels materializes a models.yaml body and runs it through the real
|
||||
// LoadModels validation (mirrors the config_test.go temp-file pattern).
|
||||
func writeAndLoadModels(t *testing.T, body string) (*Models, error) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "models.yaml")
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return LoadModels(path)
|
||||
}
|
||||
|
||||
func TestDeepSeekEchoMineRejected(t *testing.T) {
|
||||
today := time.Now().UTC().Format("2006-01-02")
|
||||
// provCaps is injected under provider p; modelCaps under model m.
|
||||
tmpl := func(echoProne bool, modelExtra string) string {
|
||||
flag := ""
|
||||
if echoProne {
|
||||
flag = "\n echoes_when_thinking_off: true"
|
||||
}
|
||||
return fmt.Sprintf(`
|
||||
prices_checked: %q
|
||||
default_model: m
|
||||
providers:
|
||||
p:
|
||||
kind: openai
|
||||
base_url: http://x%s
|
||||
models:
|
||||
m:
|
||||
provider: p
|
||||
price: { input_per_m: 1, output_per_m: 2 }%s
|
||||
`, today, flag, modelExtra)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
echoProne bool
|
||||
modelExtra string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
// The exact traced mine: the thinking-disable switch via capabilities.
|
||||
name: "armed via capabilities.reasoning.extra_body_disable",
|
||||
echoProne: true,
|
||||
modelExtra: "\n capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
// The second surface: a thinking-disable smuggled through extra_body,
|
||||
// which openAIRequest merges into the wire body — the control-only guard
|
||||
// would miss this, so it must be caught too.
|
||||
name: "armed via extra_body thinking key",
|
||||
echoProne: true,
|
||||
modelExtra: "\n extra_body: { thinking: { type: disabled } }",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
// reasoning_effort at off also pushes thinking off the default.
|
||||
name: "armed via capabilities.reasoning.effort",
|
||||
echoProne: true,
|
||||
modelExtra: "\n capabilities: { reasoning: { control: effort, off_effort: minimal } }",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
// The SAFE GLM case: an IDENTICAL thinking-disable is legal on a provider
|
||||
// that is NOT echo-prone (its editor input is a Russian draft).
|
||||
name: "safe: same thinking-disable on a non-echo-prone provider",
|
||||
echoProne: false,
|
||||
modelExtra: "\n capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
// The current benign DeepSeek state: no reasoning capability → off is a
|
||||
// no-op → thinking stays at the provider default (ON).
|
||||
name: "safe: echo-prone provider with the default (no) reasoning control",
|
||||
echoProne: true,
|
||||
modelExtra: "",
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, err := writeAndLoadModels(t, tmpl(c.echoProne, c.modelExtra))
|
||||
if c.wantErr {
|
||||
if err == nil || !strings.Contains(err.Error(), "echo mine") {
|
||||
t.Fatalf("expected fail-fast mentioning the echo mine, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("expected the config to load, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBoevoyConfigDeepSeekThinkingStaysOn guards the REAL configs/models.yaml
|
||||
// against a future edit arming the mine. It asserts the three load-bearing facts
|
||||
// directly (unmarshalling raw to dodge the prices_checked staleness date-bomb that
|
||||
// LoadModels enforces): (1) the deepseek provider is still marked echo-prone — the
|
||||
// guard is inert without the flag; (2) deepseek-v4-flash resolves to ReasoningNone
|
||||
// so reasoning="off" stays an off-by-omission (thinking ON by default); (3)
|
||||
// echoMineViolation is empty. Arm the real config and (2)/(3) go red here, while
|
||||
// tmctl itself fails loudly via LoadModels.
|
||||
func TestBoevoyConfigDeepSeekThinkingStaysOn(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "configs", "models.yaml"))
|
||||
if err != nil {
|
||||
t.Fatalf("read boevoy models.yaml: %v", err)
|
||||
}
|
||||
var m Models
|
||||
if err := yaml.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatalf("parse boevoy models.yaml: %v", err)
|
||||
}
|
||||
|
||||
const model = "deepseek-v4-flash"
|
||||
mod, ok := m.Models[model]
|
||||
if !ok {
|
||||
t.Fatalf("boevoy config no longer defines %s — update this guard deliberately", model)
|
||||
}
|
||||
prov := m.Providers[mod.Provider]
|
||||
if !prov.EchoesWhenThinkingOff {
|
||||
t.Fatalf("provider %s must stay marked echoes_when_thinking_off — the echo-mine guard is inert without it", mod.Provider)
|
||||
}
|
||||
if got := m.ResolveCapability(model).Reasoning.Control; got != llm.ReasoningNone {
|
||||
t.Fatalf("%s must resolve to ReasoningNone so reasoning=off stays a no-op (thinking ON), got %q — echo mine armed", model, got)
|
||||
}
|
||||
if why := m.echoMineViolation(model); why != "" {
|
||||
t.Fatalf("%s re-arms the echo mine: %s", model, why)
|
||||
}
|
||||
}
|
||||
|
|
@ -32,13 +32,27 @@ type Models struct {
|
|||
|
||||
// Provider is one backend endpoint.
|
||||
type Provider struct {
|
||||
Kind string `yaml:"kind"` // openai | anthropic | local
|
||||
BaseURL string `yaml:"base_url"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
Reasoning string `yaml:"reasoning"` // subset | additive (openai kind only)
|
||||
CacheTTL string `yaml:"cache_ttl"` // anthropic kind: "", "5m", "1h"
|
||||
Model string `yaml:"model"` // local kind: the backend's own tag
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
Kind string `yaml:"kind"` // openai | anthropic | local
|
||||
BaseURL string `yaml:"base_url"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
Reasoning string `yaml:"reasoning"` // subset | additive (openai kind only)
|
||||
// EchoesWhenThinkingOff marks a provider empirically shown to ECHO the
|
||||
// untranslated CJK source instead of translating when thinking is OFF
|
||||
// (DeepSeek, traced 2026-07-04 — reproducibly on dense CJK, 3/3 retries;
|
||||
// PROGRESS «Ответ Полигону», BACKEND_SESSION_PROMPT_SILENT_REFUSALS §2). Its
|
||||
// benign state is held by an off-by-omission: reasoning="off" resolves through
|
||||
// ReasoningNone which emits NOTHING thinking-related, so the provider default
|
||||
// (thinking ON) stands. A config that DISABLES thinking at reasoning=off on
|
||||
// such a provider re-arms a silent refusal (echo = HTTP 200, no translation)
|
||||
// in prod. LoadModels (echoMineViolation) fail-fasts on exactly that — the
|
||||
// regression gate §2 mandates. It distinguishes the SAFE GLM editor, whose
|
||||
// thinking:disabled is fine because its input is a Russian draft (no CJK to
|
||||
// echo): GLM's provider carries no such flag. Validation-only metadata — it
|
||||
// never reaches the wire, so it is NOT part of the snapshot (no --resnapshot).
|
||||
EchoesWhenThinkingOff bool `yaml:"echoes_when_thinking_off"`
|
||||
CacheTTL string `yaml:"cache_ttl"` // anthropic kind: "", "5m", "1h"
|
||||
Model string `yaml:"model"` // local kind: the backend's own tag
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
// Temperature (local kind): override запроса — ollama чтит request-
|
||||
// температуру поверх Modelfile, и температура облачной роли молча
|
||||
// расстроила бы локальную модель. 0 = наследовать запрос.
|
||||
|
|
@ -68,7 +82,7 @@ type TemperatureCap struct {
|
|||
|
||||
// ReasoningCapCfg declares how the neutral reasoning effort maps to the wire.
|
||||
type ReasoningCapCfg struct {
|
||||
Control string `yaml:"control"` // none | effort | extra_body_disable | mandatory
|
||||
Control string `yaml:"control"` // none | effort | extra_body_disable | mandatory
|
||||
OffEffort string `yaml:"off_effort"` // effort: value sent when reasoning=off
|
||||
OffExtraBody map[string]any `yaml:"off_extra_body"` // extra_body_disable: merged when reasoning=off
|
||||
OnExtraBody map[string]any `yaml:"on_extra_body"` // extra_body_disable: merged when reasoning is on (D6.2)
|
||||
|
|
@ -184,6 +198,12 @@ func LoadModels(path string) (*Models, error) {
|
|||
bad("model %s: non-local models require non-zero input/output prices", name)
|
||||
}
|
||||
validateCapabilities(bad, "model "+name, mod.Capabilities)
|
||||
if why := m.echoMineViolation(name); why != "" {
|
||||
bad("model %s on echo-prone provider %s: %s — this re-arms the DeepSeek echo mine "+
|
||||
"(the provider echoes the untranslated CJK source when thinking is OFF; PROGRESS «Ответ Полигону», §2); "+
|
||||
"an echo-prone provider MUST keep thinking at its default (reasoning.control none|mandatory, and no "+
|
||||
"thinking-disable extra_body) so a silent refusal is never shipped to prod", name, mod.Provider, why)
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, fmt.Errorf("config %s:\n - %s", path, strings.Join(problems, "\n - "))
|
||||
|
|
@ -232,6 +252,42 @@ func (m *Models) ResolveCapability(modelName string) llm.Capability {
|
|||
return resolved
|
||||
}
|
||||
|
||||
// thinkingControlExtraKeys are model-level extra_body keys whose sole purpose is
|
||||
// to toggle a provider's thinking on the wire (GLM/DeepSeek {"thinking":…}, Qwen
|
||||
// {"enable_thinking":…}, a raw {"reasoning_effort":…}). On an echo-prone provider
|
||||
// their mere PRESENCE is the mine, whatever the value: thinking must stay at the
|
||||
// provider default, and these keys exist only to move it off that default.
|
||||
var thinkingControlExtraKeys = []string{"thinking", "enable_thinking", "reasoning_effort"}
|
||||
|
||||
// echoMineViolation reports, for one model, whether its RESOLVED wire shape would
|
||||
// suppress thinking at reasoning=off on an echo-prone provider — the DeepSeek echo
|
||||
// mine (§2). It returns a human cause string, or "" when safe. Both injection
|
||||
// surfaces are covered: the capabilities.reasoning switch (extra_body_disable
|
||||
// merges a thinking-disable at off; effort sends reasoning_effort at off — both
|
||||
// push thinking off the default) AND a thinking-control key smuggled through the
|
||||
// model's top-level extra_body (merged into the wire body by openAIRequest). A
|
||||
// non-echo-prone provider (GLM: its editor input is a Russian draft, nothing CJK
|
||||
// to echo) is never gated, so its {thinking:{type:disabled}} stays legal.
|
||||
func (m *Models) echoMineViolation(name string) string {
|
||||
mod := m.Models[name]
|
||||
prov, ok := m.Providers[mod.Provider]
|
||||
if !ok || !prov.EchoesWhenThinkingOff {
|
||||
return ""
|
||||
}
|
||||
switch m.ResolveCapability(name).Reasoning.Control {
|
||||
case llm.ReasoningExtraBodyDisable:
|
||||
return "capabilities.reasoning.control=extra_body_disable would merge a thinking-disable at reasoning=off"
|
||||
case llm.ReasoningEffortField:
|
||||
return "capabilities.reasoning.control=effort would send reasoning_effort at reasoning=off, suppressing thinking"
|
||||
}
|
||||
for _, k := range thinkingControlExtraKeys {
|
||||
if _, present := mod.ExtraBody[k]; present {
|
||||
return fmt.Sprintf("extra_body carries the thinking-control key %q", k)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// baselineCapability is the per-kind default before any capabilities block: the
|
||||
// OpenAI-compat wire (max_tokens + send temperature), with reasoning off by
|
||||
// omission for cloud and by the ollama "none" switch for local.
|
||||
|
|
|
|||
|
|
@ -289,6 +289,18 @@ keep-alive (Ф1–2), инъекция глоссария (`selective`), пор
|
|||
|
||||
Фикс (эскалация `cjk_artifact`→фолбэк-черновик + coverage-гейт вырезания + live-conformance) — **отдельная сессия**, онбординг-промт: [BACKEND_SESSION_PROMPT_SILENT_REFUSALS.md](BACKEND_SESSION_PROMPT_SILENT_REFUSALS.md). Не блокирует шаг 3.
|
||||
|
||||
### 2026-07-04 — Сессия 5: Веха 2.5 (старт) — DeepSeek эхо-мина зафиксирована регресс-гейтом
|
||||
|
||||
Валидация точки старта зелёная (`go build/vet/test -race` ~80 тестов; `tmctl report` $0). Первым делом — регресс-гейт §2, чтобы будущая правка `models.yaml` не включила эхо-мину DeepSeek.
|
||||
|
||||
**Решение: fail-fast в конфиг-валидации (сильнее unit-теста — срабатывает на КАЖДОМ `tmctl`-запуске), а не только тест.** Провайдер получает декларативный маркер `echoes_when_thinking_off: true` (заведён на `deepseek`, документирован трассой). `config.echoMineViolation` резолвит каппу модели такого провайдера и валит старт, если thinking подавляется при `reasoning=off` — по ОБЕИМ поверхностям вооружения: (1) `capabilities.reasoning.control` = `extra_body_disable` (мержит `{thinking:disabled}`) или `effort` (шлёт `reasoning_effort`); (2) thinking-ключ, протащенный через model `extra_body` (его контрол-чек бы пропустил — `openAIRequest` мержит его в тело). Инвариант: echo-prone провайдер обязан держать `reasoning.control` = `none|mandatory` и без thinking-disable `extra_body` → thinking остаётся на дефолте (ON).
|
||||
|
||||
**GLM отличается корректно:** маркер только на `deepseek`; `glm-5` (provider `zai`) с `{thinking:{type:disabled}}` остаётся легальным (его вход — русский черновик, CJK нечего эхать). Метаданные валидации-only — на провод не идут, в snapshot НЕ входят (без `--resnapshot`).
|
||||
|
||||
**Тесты (мутационно-проверенные):** `TestDeepSeekEchoMineRejected` — 5 кейсов (2 поверхности + `effort` + GLM-безопасность + дефолт); `TestBoevoyConfigDeepSeekThinkingStaysOn` — гард на реальном `configs/models.yaml` (флаг на месте + `deepseek-v4-flash`→`ReasoningNone`; unmarshal напрямую, чтобы обойти date-бомбу `prices_checked`). Убрать `echoMineViolation` — armed-кейсы перестают падать (мутация ловится).
|
||||
|
||||
Дальше по §7 — контрактные вопросы владельцу перед coverage-гейтом/эскалацией (сегментация 1:1 vs §3.7, скоуп эскалации, скоуп coverage_fail/D10, порядок).
|
||||
|
||||
## Полигон
|
||||
(секция параллельной сессии — записи добавлять сюда)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue