textmachine/backend/internal/config/reducedeffort_test.go

package config

import (
	"fmt"
	"os"
	"path/filepath"
	"testing"
	"time"
)

// reducedeffort_test.go pins the thinking ladder Models.ReducedEffort derives from a model's RESOLVED
// capability — the data that already says what "off" means on each wire — so that no per-model table of
// vendor levels has to exist for a retry to ask for less thinking.
func modelsWithEveryReasoningControl(t *testing.T) *Models {
	t.Helper()
	dir := t.TempDir()
	path := filepath.Join(dir, "models.yaml")
	if err := os.WriteFile(path, []byte(fmt.Sprintf(`
prices_checked: %q
default_model: none-model
providers:
  ds: { kind: openai, base_url: http://y, reasoning: subset }
models:
  # control=none (DeepSeek's shape): low|medium|high go out, "" and "off" emit nothing and leave the
  # provider thinking at its OWN default.
  none-model: { provider: ds, price: { input_per_m: 1, output_per_m: 2 } }
  # control=effort (grok/ollama): "off" sends an explicit floor value.
  effort-model:
    provider: ds
    price: { input_per_m: 1, output_per_m: 2 }
    capabilities: { reasoning: { control: effort, off_effort: none } }
  # control=extra_body_disable (GLM): "off" AND "" merge a disable body.
  disable-model:
    provider: ds
    price: { input_per_m: 1, output_per_m: 2 }
    capabilities: { reasoning: { control: extra_body_disable, off_extra_body: { thinking: { type: disabled } } } }
  # control=mandatory (Gemini 3.1 Pro): nothing reasoning-related reaches the wire.
  mandatory-model:
    provider: ds
    price: { input_per_m: 1, output_per_m: 2 }
    capabilities: { reasoning: { control: mandatory } }
`, time.Now().UTC().Format("2006-01-02"))), 0o644); err != nil {
		t.Fatal(err)
	}
	m, err := LoadModels(path)
	if err != nil {
		t.Fatalf("models load: %v", err)
	}
	return m
}

func TestReducedEffortWalksDownTheEmittingSteps(t *testing.T) {
	m := modelsWithEveryReasoningControl(t)
	// Only these two controls put the LEVEL on the wire (llm.Capability.applyToBody emits
	// reasoning_effort for them and for nobody else), so only here can a step change the call.
	for _, model := range []string{"none-model", "effort-model"} {
		if got, ok := m.ReducedEffort(model, "high"); !ok || got != "medium" {
			t.Fatalf("%s: high should step to medium, got %q ok=%t", model, got, ok)
		}
		if got, ok := m.ReducedEffort(model, "medium"); !ok || got != "low" {
			t.Fatalf("%s: medium should step to low, got %q ok=%t", model, got, ok)
		}
		// The ladder stops at the lowest EMITTING step. Below it lies switching a provider's thinking
		// off, which arms the echo mine on dense CJK (D19.1 п.2) — a decision with its own guardrail,
		// never something a retry policy takes on its own.
		if got, ok := m.ReducedEffort(model, "low"); ok {
			t.Fatalf("%s: low is the floor, a retry must not reach for an off-switch, got %q", model, got)
		}
	}
}

// TestAWireThatCannotHearTheLevelGetsNoLadder is the half a code read misses. Under
// extra_body_disable the body says thinking ON or OFF and never which level, so `high` and `medium`
// marshal to the SAME bytes at the same price: a step there is not a cheaper retry, it is a second
// copy of the call that just failed, bought under a different attempt key.
func TestAWireThatCannotHearTheLevelGetsNoLadder(t *testing.T) {
	m := modelsWithEveryReasoningControl(t)
	for _, effort := range []string{"", "off", "low", "medium", "high"} {
		if got, ok := m.ReducedEffort("disable-model", effort); ok {
			t.Fatalf("extra_body_disable carries no level; stepping %q to %q would re-buy the same request", effort, got)
		}
	}
}

// TestOnlyAnUnownedDefaultStepsDownFromOff is the per-control reading, and it is the half the whole
// remedy rests on: `off` is not the bottom of every wire. With control=none nothing is emitted and the
// VENDOR's default stands — thinking ON by that control's definition, `high` on DeepSeek — so an
// explicit `low` replaces a level we do not own with one we do. With control=effort or
// extra_body_disable, `off` already resolves to a floor the capability itself chose, and there is
// nothing beneath it to ask for.
func TestOnlyAnUnownedDefaultStepsDownFromOff(t *testing.T) {
	m := modelsWithEveryReasoningControl(t)
	for _, effort := range []string{"", "off"} {
		if got, ok := m.ReducedEffort("none-model", effort); !ok || got != "low" {
			t.Fatalf("control=none at %q rides the vendor default and must step to an explicit low, got %q ok=%t", effort, got, ok)
		}
		for _, model := range []string{"effort-model", "disable-model"} {
			if got, ok := m.ReducedEffort(model, effort); ok {
				t.Fatalf("%s at %q already sits on its own floor; stepping to %q would claim a reduction that is not one", model, effort, got)
			}
		}
	}
}

// TestMandatoryThinkingHasNoLadder: where a disable attempt is a 400 and the neutral knob is swallowed
// whole, there is no step to take at any level, and pretending otherwise would buy a second identical
// call at the same price.
func TestMandatoryThinkingHasNoLadder(t *testing.T) {
	m := modelsWithEveryReasoningControl(t)
	for _, effort := range []string{"", "off", "low", "medium", "high"} {
		if got, ok := m.ReducedEffort("mandatory-model", effort); ok {
			t.Fatalf("mandatory thinking cannot be sized from here; at %q it offered %q", effort, got)
		}
	}
}

// TestReducedEffortSpeaksOnlyTheEngineVocabulary keeps the ladder inside the ONE word list the loader
// accepts: a step the config could not have been written with would reach the wire from the retry path
// without ever passing that gate.
//
// ⚠ THE EXPECTED WORDS ARE SPELLED OUT HERE, not asked of ValidReasoningEffort. ReducedEffort returns
// `next, ValidReasoningEffort(next)`, so `ok && !ValidReasoningEffort(got)` is identically false and an
// assertion built on it is green for EVERY possible implementation — including one that steps `high` to
// `minimal`. Measured: that exact mutation leaves such an assertion passing. A test of a guard must not
// be written in terms of the guard.
func TestReducedEffortSpeaksOnlyTheEngineVocabulary(t *testing.T) {
	m := modelsWithEveryReasoningControl(t)
	// The ladder may only ever hand back one of these two. Anything else — a vendor word like `xhigh`,
	// an off-switch, or a typo — is a value the loader has never validated.
	allowed := map[string]bool{"medium": true, "low": true}
	for _, model := range []string{"none-model", "effort-model", "disable-model", "mandatory-model"} {
		for _, effort := range []string{"", "off", "low", "medium", "high"} {
			got, ok := m.ReducedEffort(model, effort)
			if !ok {
				if got != "" {
					t.Fatalf("%s at %q: no step exists, so the word must be empty, got %q", model, effort, got)
				}
				continue
			}
			if !allowed[got] {
				t.Fatalf("%s at %q stepped to %q; the only steps this ladder may produce are medium and low", model, effort, got)
			}
			if !ValidReasoningEffort(got) {
				t.Fatalf("%s at %q stepped to %q, which the loader would reject", model, effort, got)
			}
		}
	}
}

// TestAnUnknownModelGetsNoLadder closes the hole that made the RUNNER's pins satisfiable by the wrong
// object. Resolving an unlisted slug yields the OpenAI-compat baseline, so the ladder used to answer
// «step to low» for a model this catalogue has never seen — and for the EMPTY STRING, which is what a
// caller reaching for the wrong field (ResolvedHop instead of ResolvedModel, on a stage with no hop)
// hands over. Measured before this: swapping the runner's argument to the hop left every pin green,
// because both names resolved to the same baseline.
func TestAnUnknownModelGetsNoLadder(t *testing.T) {
	m := modelsWithEveryReasoningControl(t)
	for _, name := range []string{"", "not-in-the-catalogue", "none-model-0813"} {
		for _, effort := range []string{"", "off", "low", "medium", "high"} {
			if got, ok := m.ReducedEffort(name, effort); ok {
				t.Fatalf("model %q is not in the catalogue; there is no declared wire to derive a step "+
					"from, yet %q stepped to %q", name, effort, got)
			}
		}
	}
	// The control: the SAME question about a listed model still answers, or this test would pass on a
	// function that refuses everything.
	if got, ok := m.ReducedEffort("none-model", "off"); !ok || got != "low" {
		t.Fatalf("a catalogued model must still get its step, got %q ok=%t", got, ok)
	}
}