textmachine/backend/internal/config/models_catalog_test.go

package config

import (
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"textmachine/backend/internal/llm"
)

// models_catalog_test.go — the pack-12 point-1 catalog validation test (research/21
// §5.3, goose declarative.rs:396-467 `all_bundled_providers_are_valid`): a $0 go-test
// over the SHIPPED configs/models.yaml that fails on catalog drift BEFORE a paid run.
// LoadModels already fail-fasts most of this at load; the test PINS that the shipped
// catalog currently passes (a stale prices_checked, an undeclared provider, a bad
// reasoning/capability enum, a re-armed echo mine all turn this red), and adds the two
// structural invariants LoadModels does not itself assert: every model-referenced
// non-local provider declares an api_key_env, and every resolved min_max_tokens floor
// is in a sane band.

const shippedModelsYAML = "../../configs/models.yaml"

// maxReasonableMinTokens bounds a per-model max_tokens floor. The real floors are
// 8000 (DeepSeek/Gemini) and 16000 (Kimi); a value above this band is almost
// certainly a typo (a token count written as a price, an extra zero) that would
// over-reserve every call on that model.
const maxReasonableMinTokens = 200000

func TestShippedModelsCatalogValid(t *testing.T) {
	m, err := LoadModels(shippedModelsYAML)
	if err != nil {
		// LoadModels aggregates ALL problems into one error — surface it verbatim so
		// the operator fixes the drift (stale prices, undeclared provider, bad enum,
		// echo mine) before spending a dollar.
		t.Fatalf("shipped configs/models.yaml failed validation (catalog drift before a paid run):\n%v", err)
	}

	// default_model resolves (price-fallback anchor).
	if _, ok := m.Models[m.DefaultModel]; !ok {
		t.Fatalf("default_model %q is not a defined model", m.DefaultModel)
	}

	knownReasoning := map[string]bool{"": true, "subset": true, "additive": true, "additive_total": true}
	for name, prov := range m.Providers {
		switch prov.Kind {
		case "openai", "anthropic", "local":
		default:
			t.Errorf("provider %s: unknown kind %q", name, prov.Kind)
		}
		if prov.Kind == "openai" && !knownReasoning[prov.Reasoning] {
			t.Errorf("provider %s: reasoning %q not in {subset,additive,additive_total}", name, prov.Reasoning)
		}
	}

	for name, mod := range m.Models {
		prov, ok := m.Providers[mod.Provider]
		if !ok {
			t.Errorf("model %s references undeclared provider %q", name, mod.Provider)
			continue
		}
		// Every model-reachable NON-LOCAL provider must declare an api_key_env: a paid
		// model with no key env would surface as a 401 only after reserve/slot charge.
		if prov.Kind != "local" && prov.APIKeyEnv == "" {
			t.Errorf("model %s: non-local provider %s declares no api_key_env", name, mod.Provider)
		}
		// Non-local models must carry non-zero input/output prices (an unknown model
		// must never book at $0 and blind the ceiling).
		if prov.Kind != "local" && (mod.Price.InputPerM <= 0 || mod.Price.OutputPerM <= 0) {
			t.Errorf("model %s: non-local model needs input/output prices > 0 (got in=%.4f out=%.4f)", name, mod.Price.InputPerM, mod.Price.OutputPerM)
		}
		// The RESOLVED max_tokens floor (provider→model layering) is sane: never
		// negative, never absurdly large.
		if floor := m.MinMaxTokens(name); floor < 0 || floor > maxReasonableMinTokens {
			t.Errorf("model %s: resolved min_max_tokens %d is out of the sane band [0,%d]", name, floor, maxReasonableMinTokens)
		}
	}
}

// TestShippedGeminiDeclaresOneSystemMessage is the gate the pack's own fix was missing: the two lines
// in configs/models.yaml are the ONLY thing that makes the system-message join reach the real endpoint,
// and an adversarial pass proved that deleting them left the entire module's tests green. Everything
// else about the fix is exercised against a synthetic fixture provider; this is what ties it to
// production. The failure it guards is a silent HTTP 200 with the glossary dropped — nothing 4xx, no
// gate, a plausible translation that has quietly stopped obeying the memory bank.
func TestShippedGeminiDeclaresOneSystemMessage(t *testing.T) {
	m, err := LoadModels(shippedModelsYAML)
	if err != nil {
		t.Fatalf("shipped configs/models.yaml failed validation:\n%v", err)
	}
	seen := 0
	for name, mod := range m.Models {
		if mod.Provider != "gemini" {
			continue
		}
		seen++
		if got := m.ResolveCapability(name).SystemMessages; got != llm.SystemMessagesSingle {
			t.Errorf("model %s rides the Gemini OpenAI-compat layer, which carries ONE system message; "+
				"resolved SystemMessages = %q, want %q — the memory-bank injection is the second system "+
				"message and is dropped by the endpoint without an error",
				name, got, llm.SystemMessagesSingle)
		}
	}
	if seen == 0 {
		t.Fatal("no model resolves to the gemini provider — the declaration this test guards has nothing to guard; " +
			"if the provider was removed on purpose, remove this test with it")
	}
}

// TestSystemMessagesIsRefusedOnTheAnthropicKind: the axis is an OpenAI-compat wire shape, and the
// Anthropic adapter is built with no Capability at all — so a declaration there reaches nothing, while
// still resolving into the capability the job snapshot carries. Accepted silently it would re-buy a
// book for a line that changed no byte on the wire; refused by kind, the mistake is a load error.
func TestSystemMessagesIsRefusedOnTheAnthropicKind(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "models.yaml")
	if err := os.WriteFile(path, []byte(`
prices_checked: "`+time.Now().UTC().Format("2006-01-02")+`"
default_model: m
providers:
  a:
    kind: anthropic
    api_key_env: X
    capabilities: { system_messages: single }
models:
  m:
    provider: a
    price: { input_per_m: 1, cached_per_m: 1, cache_write_per_m: 0, output_per_m: 1 }
`), 0o644); err != nil {
		t.Fatal(err)
	}
	_, err := LoadModels(path)
	if err == nil {
		t.Fatal("a wire-shape axis the anthropic adapter cannot read must be refused, not silently resolved")
	}
	if !strings.Contains(err.Error(), "capabilities.system_messages") {
		t.Fatalf("the refusal must name the key, got: %v", err)
	}
}

// TestExplicitMultiCostsNothing pins the normalisation the code argues for: writing the DEFAULT out
// loud is how an author records a verified endpoint or overrides a provider's single-slot rule, and it
// must not put a key in the capability the snapshot carries — that would re-buy the book for a line
// that changed no byte on the wire.
func TestExplicitMultiCostsNothing(t *testing.T) {
	dir := t.TempDir()
	write := func(caps string) *Models {
		t.Helper()
		path := filepath.Join(dir, strings.ReplaceAll(caps, " ", "")+"models.yaml")
		if err := os.WriteFile(path, []byte(`
prices_checked: "`+time.Now().UTC().Format("2006-01-02")+`"
default_model: m
providers:
  p:
    kind: openai
    base_url: http://x
    api_key_env: X
`+caps+`
models:
  m:
    provider: p
    price: { input_per_m: 1, cached_per_m: 1, cache_write_per_m: 0, output_per_m: 1 }
`), 0o644); err != nil {
			t.Fatal(err)
		}
		m, err := LoadModels(path)
		if err != nil {
			t.Fatalf("load: %v", err)
		}
		return m
	}
	bare := write("")
	multi := write("    capabilities: { system_messages: multi }\n")
	single := write("    capabilities: { system_messages: single }\n")

	bareJSON, _ := json.Marshal(bare.ResolveCapability("m"))
	multiJSON, _ := json.Marshal(multi.ResolveCapability("m"))
	singleJSON, _ := json.Marshal(single.ResolveCapability("m"))
	if string(bareJSON) != string(multiJSON) {
		t.Fatalf("an explicit `multi` must resolve byte-identically to saying nothing:\n bare  %s\n multi %s", bareJSON, multiJSON)
	}
	if string(singleJSON) == string(bareJSON) {
		t.Fatalf("a declared `single` MUST move the snapshot bytes — that is what makes the flip a loud --resnapshot: %s", singleJSON)
	}
}