package config import ( "encoding/json" "os" "path/filepath" "strings" "testing" "time" "textmachine/backend/internal/llm" ) // models_catalog_test.go — the pack-12 point-1 catalog validation test (research/21 // §5.3, goose declarative.rs:396-467 `all_bundled_providers_are_valid`): a $0 go-test // over the SHIPPED configs/models.yaml that fails on catalog drift BEFORE a paid run. // LoadModels already fail-fasts most of this at load; the test PINS that the shipped // catalog currently passes (a stale prices_checked, an undeclared provider, a bad // reasoning/capability enum, a re-armed echo mine all turn this red), and adds the two // structural invariants LoadModels does not itself assert: every model-referenced // non-local provider declares an api_key_env, and every resolved min_max_tokens floor // is in a sane band. const shippedModelsYAML = "../../configs/models.yaml" // maxReasonableMinTokens bounds a per-model max_tokens floor. The real floors are // 8000 (DeepSeek/Gemini) and 16000 (Kimi); a value above this band is almost // certainly a typo (a token count written as a price, an extra zero) that would // over-reserve every call on that model. const maxReasonableMinTokens = 200000 func TestShippedModelsCatalogValid(t *testing.T) { m, err := LoadModels(shippedModelsYAML) if err != nil { // LoadModels aggregates ALL problems into one error — surface it verbatim so // the operator fixes the drift (stale prices, undeclared provider, bad enum, // echo mine) before spending a dollar. t.Fatalf("shipped configs/models.yaml failed validation (catalog drift before a paid run):\n%v", err) } // default_model resolves (price-fallback anchor). if _, ok := m.Models[m.DefaultModel]; !ok { t.Fatalf("default_model %q is not a defined model", m.DefaultModel) } knownReasoning := map[string]bool{"": true, "subset": true, "additive": true, "additive_total": true} for name, prov := range m.Providers { switch prov.Kind { case "openai", "anthropic", "local": default: t.Errorf("provider %s: unknown kind %q", name, prov.Kind) } if prov.Kind == "openai" && !knownReasoning[prov.Reasoning] { t.Errorf("provider %s: reasoning %q not in {subset,additive,additive_total}", name, prov.Reasoning) } } for name, mod := range m.Models { prov, ok := m.Providers[mod.Provider] if !ok { t.Errorf("model %s references undeclared provider %q", name, mod.Provider) continue } // Every model-reachable NON-LOCAL provider must declare an api_key_env: a paid // model with no key env would surface as a 401 only after reserve/slot charge. if prov.Kind != "local" && prov.APIKeyEnv == "" { t.Errorf("model %s: non-local provider %s declares no api_key_env", name, mod.Provider) } // Non-local models must carry non-zero input/output prices (an unknown model // must never book at $0 and blind the ceiling). if prov.Kind != "local" && (mod.Price.InputPerM <= 0 || mod.Price.OutputPerM <= 0) { t.Errorf("model %s: non-local model needs input/output prices > 0 (got in=%.4f out=%.4f)", name, mod.Price.InputPerM, mod.Price.OutputPerM) } // The RESOLVED max_tokens floor (provider→model layering) is sane: never // negative, never absurdly large. if floor := m.MinMaxTokens(name); floor < 0 || floor > maxReasonableMinTokens { t.Errorf("model %s: resolved min_max_tokens %d is out of the sane band [0,%d]", name, floor, maxReasonableMinTokens) } } } // TestShippedGeminiDeclaresOneSystemMessage is the gate the pack's own fix was missing: the two lines // in configs/models.yaml are the ONLY thing that makes the system-message join reach the real endpoint, // and an adversarial pass proved that deleting them left the entire module's tests green. Everything // else about the fix is exercised against a synthetic fixture provider; this is what ties it to // production. The failure it guards is a silent HTTP 200 with the glossary dropped — nothing 4xx, no // gate, a plausible translation that has quietly stopped obeying the memory bank. func TestShippedGeminiDeclaresOneSystemMessage(t *testing.T) { m, err := LoadModels(shippedModelsYAML) if err != nil { t.Fatalf("shipped configs/models.yaml failed validation:\n%v", err) } seen := 0 for name, mod := range m.Models { if mod.Provider != "gemini" { continue } seen++ if got := m.ResolveCapability(name).SystemMessages; got != llm.SystemMessagesSingle { t.Errorf("model %s rides the Gemini OpenAI-compat layer, which carries ONE system message; "+ "resolved SystemMessages = %q, want %q — the memory-bank injection is the second system "+ "message and is dropped by the endpoint without an error", name, got, llm.SystemMessagesSingle) } } if seen == 0 { t.Fatal("no model resolves to the gemini provider — the declaration this test guards has nothing to guard; " + "if the provider was removed on purpose, remove this test with it") } } // TestSystemMessagesIsRefusedOnTheAnthropicKind: the axis is an OpenAI-compat wire shape, and the // Anthropic adapter is built with no Capability at all — so a declaration there reaches nothing, while // still resolving into the capability the job snapshot carries. Accepted silently it would re-buy a // book for a line that changed no byte on the wire; refused by kind, the mistake is a load error. func TestSystemMessagesIsRefusedOnTheAnthropicKind(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "models.yaml") if err := os.WriteFile(path, []byte(` prices_checked: "`+time.Now().UTC().Format("2006-01-02")+`" default_model: m providers: a: kind: anthropic api_key_env: X capabilities: { system_messages: single } models: m: provider: a price: { input_per_m: 1, cached_per_m: 1, cache_write_per_m: 0, output_per_m: 1 } `), 0o644); err != nil { t.Fatal(err) } _, err := LoadModels(path) if err == nil { t.Fatal("a wire-shape axis the anthropic adapter cannot read must be refused, not silently resolved") } if !strings.Contains(err.Error(), "capabilities.system_messages") { t.Fatalf("the refusal must name the key, got: %v", err) } } // TestExplicitMultiCostsNothing pins the normalisation the code argues for: writing the DEFAULT out // loud is how an author records a verified endpoint or overrides a provider's single-slot rule, and it // must not put a key in the capability the snapshot carries — that would re-buy the book for a line // that changed no byte on the wire. func TestExplicitMultiCostsNothing(t *testing.T) { dir := t.TempDir() write := func(caps string) *Models { t.Helper() path := filepath.Join(dir, strings.ReplaceAll(caps, " ", "")+"models.yaml") if err := os.WriteFile(path, []byte(` prices_checked: "`+time.Now().UTC().Format("2006-01-02")+`" default_model: m providers: p: kind: openai base_url: http://x api_key_env: X `+caps+` models: m: provider: p price: { input_per_m: 1, cached_per_m: 1, cache_write_per_m: 0, output_per_m: 1 } `), 0o644); err != nil { t.Fatal(err) } m, err := LoadModels(path) if err != nil { t.Fatalf("load: %v", err) } return m } bare := write("") multi := write(" capabilities: { system_messages: multi }\n") single := write(" capabilities: { system_messages: single }\n") bareJSON, _ := json.Marshal(bare.ResolveCapability("m")) multiJSON, _ := json.Marshal(multi.ResolveCapability("m")) singleJSON, _ := json.Marshal(single.ResolveCapability("m")) if string(bareJSON) != string(multiJSON) { t.Fatalf("an explicit `multi` must resolve byte-identically to saying nothing:\n bare %s\n multi %s", bareJSON, multiJSON) } if string(singleJSON) == string(bareJSON) { t.Fatalf("a declared `single` MUST move the snapshot bytes — that is what makes the flip a loud --resnapshot: %s", singleJSON) } }