textmachine/backend/internal/config/echo_mine_test.go

package config

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"gopkg.in/yaml.v3"

	"textmachine/backend/internal/llm"
)

// echo_mine_test.go is the §2 regression gate for the DeepSeek echo mine: on a
// provider empirically shown to ECHO the untranslated CJK source when thinking is
// OFF (traced 2026-07-04), no model may be configured to disable/suppress thinking
// at reasoning=off — that re-arms a silent refusal (HTTP 200, no translation) in
// prod. LoadModels must fail-fast on both injection surfaces, while leaving the
// SAFE GLM editor (thinking:disabled over a Russian draft) untouched.
//
// Mutation check (how this guards against a future edit re-arming the mine):
// delete the echoMineViolation call in LoadModels (or the field) and the four
// "armed" cases below stop failing — the test goes red exactly when the guard is
// removed.

// writeAndLoadModels materializes a models.yaml body and runs it through the real
// LoadModels validation (mirrors the config_test.go temp-file pattern).
func writeAndLoadModels(t *testing.T, body string) (*Models, error) {
	t.Helper()
	dir := t.TempDir()
	path := filepath.Join(dir, "models.yaml")
	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
		t.Fatal(err)
	}
	return LoadModels(path)
}

func TestDeepSeekEchoMineRejected(t *testing.T) {
	today := time.Now().UTC().Format("2006-01-02")
	// provCaps is injected under provider p; modelCaps under model m.
	tmpl := func(echoProne bool, modelExtra string) string {
		flag := ""
		if echoProne {
			flag = "\n    echoes_when_thinking_off: true"
		}
		return fmt.Sprintf(`
prices_checked: %q
default_model: m
providers:
  p:
    kind: openai
    base_url: http://x%s
models:
  m:
    provider: p
    price: { input_per_m: 1, output_per_m: 2 }%s
`, today, flag, modelExtra)
	}

	cases := []struct {
		name       string
		echoProne  bool
		modelExtra string
		wantErr    bool
	}{
		{
			// The exact traced mine: the thinking-disable switch via capabilities.
			name:       "armed via capabilities.reasoning.extra_body_disable",
			echoProne:  true,
			modelExtra: "\n    capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }",
			wantErr:    true,
		},
		{
			// The second surface: a thinking-disable smuggled through extra_body,
			// which openAIRequest merges into the wire body — the control-only guard
			// would miss this, so it must be caught too.
			name:       "armed via extra_body thinking key",
			echoProne:  true,
			modelExtra: "\n    extra_body: { thinking: { type: disabled } }",
			wantErr:    true,
		},
		{
			// reasoning_effort at off also pushes thinking off the default.
			name:       "armed via capabilities.reasoning.effort",
			echoProne:  true,
			modelExtra: "\n    capabilities: { reasoning: { control: effort, off_effort: minimal } }",
			wantErr:    true,
		},
		{
			// The SAFE GLM case: an IDENTICAL thinking-disable is legal on a provider
			// that is NOT echo-prone (its editor input is a Russian draft).
			name:       "safe: same thinking-disable on a non-echo-prone provider",
			echoProne:  false,
			modelExtra: "\n    capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }",
			wantErr:    false,
		},
		{
			// The current benign DeepSeek state: no reasoning capability → off is a
			// no-op → thinking stays at the provider default (ON).
			name:       "safe: echo-prone provider with the default (no) reasoning control",
			echoProne:  true,
			modelExtra: "",
			wantErr:    false,
		},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			_, err := writeAndLoadModels(t, tmpl(c.echoProne, c.modelExtra))
			if c.wantErr {
				if err == nil || !strings.Contains(err.Error(), "echo mine") {
					t.Fatalf("expected fail-fast mentioning the echo mine, got %v", err)
				}
				return
			}
			if err != nil {
				t.Fatalf("expected the config to load, got %v", err)
			}
		})
	}
}

// TestBoevoyConfigDeepSeekThinkingStaysOn guards the REAL configs/models.yaml
// against a future edit arming the mine. It asserts the three load-bearing facts
// directly (unmarshalling raw to dodge the prices_checked staleness date-bomb that
// LoadModels enforces): (1) the deepseek provider is still marked echo-prone — the
// guard is inert without the flag; (2) deepseek-v4-flash resolves to ReasoningNone
// so reasoning="off" stays an off-by-omission (thinking ON by default); (3)
// echoMineViolation is empty. Arm the real config and (2)/(3) go red here, while
// tmctl itself fails loudly via LoadModels.
func TestBoevoyConfigDeepSeekThinkingStaysOn(t *testing.T) {
	raw, err := os.ReadFile(filepath.Join("..", "..", "configs", "models.yaml"))
	if err != nil {
		t.Fatalf("read boevoy models.yaml: %v", err)
	}
	var m Models
	if err := yaml.Unmarshal(raw, &m); err != nil {
		t.Fatalf("parse boevoy models.yaml: %v", err)
	}

	const model = "deepseek-v4-flash"
	mod, ok := m.Models[model]
	if !ok {
		t.Fatalf("boevoy config no longer defines %s — update this guard deliberately", model)
	}
	prov := m.Providers[mod.Provider]
	if !prov.EchoesWhenThinkingOff {
		t.Fatalf("provider %s must stay marked echoes_when_thinking_off — the echo-mine guard is inert without it", mod.Provider)
	}
	if got := m.ResolveCapability(model).Reasoning.Control; got != llm.ReasoningNone {
		t.Fatalf("%s must resolve to ReasoningNone so reasoning=off stays a no-op (thinking ON), got %q — echo mine armed", model, got)
	}
	if why := m.echoMineViolation(model); why != "" {
		t.Fatalf("%s re-arms the echo mine: %s", model, why)
	}
}