package config import ( "fmt" "os" "path/filepath" "strings" "testing" "time" "gopkg.in/yaml.v3" "textmachine/backend/internal/llm" ) // echo_mine_test.go is the §2 regression gate for the DeepSeek echo mine: on a // provider empirically shown to ECHO the untranslated CJK source when thinking is // OFF (traced 2026-07-04), no model may be configured to disable/suppress thinking // at reasoning=off — that re-arms a silent refusal (HTTP 200, no translation) in // prod. LoadModels must fail-fast on both injection surfaces, while leaving the // SAFE GLM editor (thinking:disabled over a Russian draft) untouched. // // Mutation check (how this guards against a future edit re-arming the mine): // delete the echoMineViolation call in LoadModels (or the field) and the four // "armed" cases below stop failing — the test goes red exactly when the guard is // removed. // writeAndLoadModels materializes a models.yaml body and runs it through the real // LoadModels validation (mirrors the config_test.go temp-file pattern). func writeAndLoadModels(t *testing.T, body string) (*Models, error) { t.Helper() dir := t.TempDir() path := filepath.Join(dir, "models.yaml") if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } return LoadModels(path) } func TestDeepSeekEchoMineRejected(t *testing.T) { today := time.Now().UTC().Format("2006-01-02") // provCaps is injected under provider p; modelCaps under model m. tmpl := func(echoProne bool, modelExtra string) string { flag := "" if echoProne { flag = "\n echoes_when_thinking_off: true" } return fmt.Sprintf(` prices_checked: %q default_model: m providers: p: kind: openai base_url: http://x%s models: m: provider: p price: { input_per_m: 1, output_per_m: 2 }%s `, today, flag, modelExtra) } cases := []struct { name string echoProne bool modelExtra string wantErr bool }{ { // The exact traced mine: the thinking-disable switch via capabilities. name: "armed via capabilities.reasoning.extra_body_disable", echoProne: true, modelExtra: "\n capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }", wantErr: true, }, { // The second surface: a thinking-disable smuggled through extra_body, // which openAIRequest merges into the wire body — the control-only guard // would miss this, so it must be caught too. name: "armed via extra_body thinking key", echoProne: true, modelExtra: "\n extra_body: { thinking: { type: disabled } }", wantErr: true, }, { // The NESTED form: chat_template_kwargs.thinking:false is DeepSeek's // documented V3.1+ disable — the most likely real arming form. A flat // top-level scan misses it; the recursive scan must catch it (ext-review). name: "armed via nested chat_template_kwargs.thinking (DeepSeek V3.1+)", echoProne: true, modelExtra: "\n extra_body: { chat_template_kwargs: { thinking: false } }", wantErr: true, }, { // reasoning_effort at off also pushes thinking off the default. name: "armed via capabilities.reasoning.effort", echoProne: true, modelExtra: "\n capabilities: { reasoning: { control: effort, off_effort: minimal } }", wantErr: true, }, { // The SAFE GLM case: an IDENTICAL thinking-disable is legal on a provider // that is NOT echo-prone (its editor input is a Russian draft). name: "safe: same thinking-disable on a non-echo-prone provider", echoProne: false, modelExtra: "\n capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }", wantErr: false, }, { // The current benign DeepSeek state: no reasoning capability → off is a // no-op → thinking stays at the provider default (ON). name: "safe: echo-prone provider with the default (no) reasoning control", echoProne: true, modelExtra: "", wantErr: false, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { _, err := writeAndLoadModels(t, tmpl(c.echoProne, c.modelExtra)) if c.wantErr { if err == nil || !strings.Contains(err.Error(), "echo mine") { t.Fatalf("expected fail-fast mentioning the echo mine, got %v", err) } return } if err != nil { t.Fatalf("expected the config to load, got %v", err) } }) } } // TestBoevoyConfigDeepSeekThinkingStaysOn guards the REAL configs/models.yaml // against a future edit arming the mine. It asserts the three load-bearing facts // directly (unmarshalling raw to dodge the prices_checked staleness date-bomb that // LoadModels enforces): (1) the deepseek provider is still marked echo-prone — the // guard is inert without the flag; (2) deepseek-v4-flash resolves to ReasoningNone // so reasoning="off" stays an off-by-omission (thinking ON by default); (3) // echoMineViolation is empty. Arm the real config and (2)/(3) go red here, while // tmctl itself fails loudly via LoadModels. func TestBoevoyConfigDeepSeekThinkingStaysOn(t *testing.T) { raw, err := os.ReadFile(filepath.Join("..", "..", "configs", "models.yaml")) if err != nil { t.Fatalf("read boevoy models.yaml: %v", err) } var m Models if err := yaml.Unmarshal(raw, &m); err != nil { t.Fatalf("parse boevoy models.yaml: %v", err) } const model = "deepseek-v4-flash" mod, ok := m.Models[model] if !ok { t.Fatalf("boevoy config no longer defines %s — update this guard deliberately", model) } prov := m.Providers[mod.Provider] if !prov.EchoesWhenThinkingOff { t.Fatalf("provider %s must stay marked echoes_when_thinking_off — the echo-mine guard is inert without it", mod.Provider) } if got := m.ResolveCapability(model).Reasoning.Control; got != llm.ReasoningNone { t.Fatalf("%s must resolve to ReasoningNone so reasoning=off stays a no-op (thinking ON), got %q — echo mine armed", model, got) } if why := m.echoMineViolation(model); why != "" { t.Fatalf("%s re-arms the echo mine: %s", model, why) } } // TestBoevoyConfigEditorDsproAndGrokReasoning pins two invariants. (1) The D39.22 interim editor: // the default editor of both boevoy pipelines is deepseek-v4-pro BILINGUAL (supersedes the D30.1 // glm-5 default; the pere-run did not crown glm-vs-dspro, dspro = judge #1 / cheaper / repairable // defects → owner interim, glm-5 kept as pipeline-arm-glm.yaml reserve) and its reasoning is "off" // — the LOAD-BEARING echo-mine semantic: on deepseek-v4-pro "off" resolves to ReasoningNone (a // NO-OP), so thinking stays ON and the echo-mine is NOT armed. (2) grok-4.3 (still the channel-B // tier / 18+ judge in models.yaml) resolves to an EXPLICIT reasoning off-switch (ReasoningEffortField // + OffEffort "none"), NOT off-by-omission — grok's OMITTED default is "low" (it thinks, billed // additively per xAI), and it must not trip the echo-mine guard. c2's select (judge) stage stays glm-5. func TestBoevoyConfigEditorDsproAndGrokReasoning(t *testing.T) { m, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml")) if err != nil { t.Fatalf("load boevoy models.yaml: %v", err) } // (2) grok-4.3 reasoning invariant (channel B / 18+ judge; unchanged by the editor flip). const grok = "grok-4.3" cap := m.ResolveCapability(grok) if cap.Reasoning.Control != llm.ReasoningEffortField || cap.Reasoning.OffEffort != "none" { t.Errorf("%s must resolve to explicit reasoning_effort:\"none\" (control=effort, off_effort=none), got control=%q off_effort=%q — control:none leaves grok's \"low\" default (thinks, additive billing)", grok, cap.Reasoning.Control, cap.Reasoning.OffEffort) } if why := m.echoMineViolation(grok); why != "" { // The guard must stay silent for grok — NOT because "the editor only sees a Russian draft" (that // rationale died with D30.1: the editor is bilingual), but because xai carries no echo-prone flag. // Pack-17 deliberately did NOT add one: it would outlaw grok's off-switch, whose money consequence // (grok then thinks by default, billed additively) is the owner's call. The exposure is surfaced // loudly instead (pipeline.Runner.sourceEchoExposure) and caught by the echo gate (D19.2). t.Errorf("%s must not trip the echo-mine guard (xai declares no echoes_when_thinking_off): %s", grok, why) } // (1) D39.22 interim editor flip: deepseek-v4-pro bilingual editor, reasoning "off" (a ReasoningNone // no-op — thinking stays ON), in both boevoy pipelines. The glm-5 default is superseded (kept as a // reserve arm); the c2 select (judge) glm-5 slot is a DIFFERENT role and is unaffected. for _, pf := range []string{"pipeline-c1.yaml", "pipeline-c2.yaml"} { p, err := LoadPipeline(filepath.Join("..", "..", "configs", pf), m, "zh-ru", nil) if err != nil { t.Fatalf("load %s: %v", pf, err) } var editModel, editReasoning, selectModel string var editFewShot *bool for _, st := range p.Stages { switch st.Role { case "editor": editModel, editReasoning, editFewShot = st.Model, st.Reasoning, st.FewShot case "judge": selectModel = st.Model } } if editModel != "deepseek-v4-pro" { t.Errorf("%s edit stage model = %q, want deepseek-v4-pro (D39.22 interim editor; glm-5 is the reserve arm)", pf, editModel) } if editReasoning != "off" { t.Errorf("%s edit stage reasoning = %q, want \"off\" — the echo-mine semantic: on deepseek-v4-pro \"off\"=ReasoningNone (no-op) so thinking stays ON, never armed", pf, editReasoning) } // The reasoning editor drops the hand-written few-shot block (D38.5 / exp14 §2а: its own CoT is // disturbed by the examples). This assertion used to live on pipeline-arm-deepseek-pro.yaml — the // arm that swapped the editor TO deepseek-v4-pro. D39.22 made that editor the production one, so // the arm became a copy of c1 and was retired; the guarantee moves here, onto the configs that // actually ship it. if editFewShot == nil || *editFewShot { t.Errorf("%s edit stage must set few_shot:false — the reasoning editor drops the hand-written examples (D38.5), got %v", pf, editFewShot) } if pf == "pipeline-c2.yaml" && selectModel != "glm-5" { t.Errorf("%s select (judge) stage must stay glm-5 (Gemini Phase-2 slot), got %q", pf, selectModel) } } } // TestSwapArmConfigs guards the WS6 editor swap-arms (D39.9 — editor = the weak link). The arms are // CONFIG, not code: each pipeline-arm-*.yaml swaps ONLY the editor model, so a different editor-model // resolves a different stageSnap.Model → a distinct snapshot_W2 (a separate run). This pins: (1) each arm // loads and its editor is the arm model; (2) neither arm re-arms the echo mine; (3) neither keeps the // few-shot block off — both surviving arms run NON-reasoning editors, for which the examples are the // point; (4) the mistral arm's PRE-CONDITION — mistral-large-2512 carries a rate_limit so the WS1 // per-model rate-guard actually throttles it (§6(б) F4). // // ⚠ That every OTHER key matches production is not pinned here and cannot be, because a list of keys can // only check what somebody wrote into it: it is TestAnArmIsTheProductionConfigWithADifferentEditor, which // compares the loaded configs whole. // // ⚠ pipeline-arm-deepseek-pro.yaml is RETIRED (07.09) rather than absent by oversight: D39.22 made // deepseek-v4-pro the production editor, so that arm isolated no variable — it was a second copy of c1 // that had to be kept in step forever, and its own pin read as "the arm is wrong" whenever c1 moved. The // few-shot guarantee it carried moved onto c1/c2 above; the equality gate refuses to let an arm silently // become a copy again. // editorKeepsFewShot records, per EDITOR MODEL, whether that editor keeps the hand-written ---FEWSHOT--- // block. It is a decision, not a property a model exposes: a reasoning editor's own chain of thought is // disturbed by worked examples (D38.5 / exp14 §2а) and drops them; the others keep them, because for a // non-reasoning editor the examples are the point. Keyed on the model so that a new ARM inherits the // decision and a new EDITOR has to state one — an unrecorded editor is refused, never defaulted. var editorKeepsFewShot = map[string]bool{ "deepseek-v4-pro": false, "glm-5": true, "mistral-large-2512": true, } func TestSwapArmConfigs(t *testing.T) { m, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml")) if err != nil { t.Fatalf("load boevoy models.yaml: %v", err) } arms, err := filepath.Glob(filepath.Join("..", "..", "configs", "pipeline-arm-*.yaml")) if err != nil { t.Fatal(err) } if len(arms) == 0 { t.Fatal("no configs/pipeline-arm-*.yaml — this test would be enforcing nothing") } for _, file := range arms { t.Run(filepath.Base(file), func(t *testing.T) { p, lerr := LoadPipeline(file, m, "zh-ru", nil) if lerr != nil { t.Fatalf("load %s: %v", file, lerr) } var edit *Stage for i := range p.Stages { if p.Stages[i].Role == "editor" { edit = &p.Stages[i] } } if edit == nil { t.Fatalf("%s has no editor stage", file) } // (1) the arm file NAMES its editor, and the name is the only handle a deploy flip has: the // slug in pipeline-arm-.yaml must be part of the editor model it resolves. Derived from // the filename rather than listed, so a new arm is covered by being named; without it the // glob-driven checks below would pass an arm whose editor was swapped to another arm's model. slug := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(file), "pipeline-arm-"), ".yaml") if !strings.Contains(edit.Model, slug) { t.Errorf("%s resolves editor %q, which does not carry the file's own slug %q — an arm's name is what a deploy flip selects, so it has to say which editor it is", filepath.Base(file), edit.Model, slug) } // (2) echo-mine safety: the arm editor's model must not re-arm the mine. if why := m.echoMineViolation(edit.Model); why != "" { t.Errorf("%s editor %q re-arms the echo mine: %s", file, edit.Model, why) } // (3) few-shot policy, looked up BY EDITOR MODEL rather than by file: the decision belongs to // the model, so a new arm on a known editor is covered by existing it, and a new EDITOR is // refused until somebody records what it does with the examples. keeps, known := editorKeepsFewShot[edit.Model] if !known { t.Fatalf("%s runs editor %q, whose few-shot decision is not recorded in editorKeepsFewShot — a reasoning editor drops the hand examples and a non-reasoning one keeps them, and defaulting either way would be silent (D38.5)", file, edit.Model) } switch { case keeps && edit.FewShot != nil: t.Errorf("%s: editor %q keeps the few-shot block, so few_shot must be left at its default (nil = ON), got %v", file, edit.Model, *edit.FewShot) case !keeps && (edit.FewShot == nil || *edit.FewShot): t.Errorf("%s: editor %q is a reasoning editor and must set few_shot:false, got %v", file, edit.Model, edit.FewShot) } }) } // (4) mistral rate-guard pre-condition: mistral-large-2512 must carry a rate_limit so the WS1 wave // per-model semaphore actually throttles it (mistral ~48% retry-fails under N-∥ without it). if rl := m.Models["mistral-large-2512"].RateLimit; rl.MaxConcurrency <= 0 { t.Errorf("mistral-large-2512 must configure rate_limit.max_concurrency>0 (WS6 pre-condition — the swap-arm needs the WS1 rate-guard), got %d", rl.MaxConcurrency) } }