package config import ( "testing" ) // models_catalog_test.go — the pack-12 point-1 catalog validation test (research/21 // §5.3, goose declarative.rs:396-467 `all_bundled_providers_are_valid`): a $0 go-test // over the SHIPPED configs/models.yaml that fails on catalog drift BEFORE a paid run. // LoadModels already fail-fasts most of this at load; the test PINS that the shipped // catalog currently passes (a stale prices_checked, an undeclared provider, a bad // reasoning/capability enum, a re-armed echo mine all turn this red), and adds the two // structural invariants LoadModels does not itself assert: every model-referenced // non-local provider declares an api_key_env, and every resolved min_max_tokens floor // is in a sane band. const shippedModelsYAML = "../../configs/models.yaml" // maxReasonableMinTokens bounds a per-model max_tokens floor. The real floors are // 8000 (DeepSeek/Gemini) and 16000 (Kimi); a value above this band is almost // certainly a typo (a token count written as a price, an extra zero) that would // over-reserve every call on that model. const maxReasonableMinTokens = 200000 func TestShippedModelsCatalogValid(t *testing.T) { m, err := LoadModels(shippedModelsYAML) if err != nil { // LoadModels aggregates ALL problems into one error — surface it verbatim so // the operator fixes the drift (stale prices, undeclared provider, bad enum, // echo mine) before spending a dollar. t.Fatalf("shipped configs/models.yaml failed validation (catalog drift before a paid run):\n%v", err) } // default_model resolves (price-fallback anchor). if _, ok := m.Models[m.DefaultModel]; !ok { t.Fatalf("default_model %q is not a defined model", m.DefaultModel) } knownReasoning := map[string]bool{"": true, "subset": true, "additive": true, "additive_total": true} for name, prov := range m.Providers { switch prov.Kind { case "openai", "anthropic", "local": default: t.Errorf("provider %s: unknown kind %q", name, prov.Kind) } if prov.Kind == "openai" && !knownReasoning[prov.Reasoning] { t.Errorf("provider %s: reasoning %q not in {subset,additive,additive_total}", name, prov.Reasoning) } } for name, mod := range m.Models { prov, ok := m.Providers[mod.Provider] if !ok { t.Errorf("model %s references undeclared provider %q", name, mod.Provider) continue } // Every model-reachable NON-LOCAL provider must declare an api_key_env: a paid // model with no key env would surface as a 401 only after reserve/slot charge. if prov.Kind != "local" && prov.APIKeyEnv == "" { t.Errorf("model %s: non-local provider %s declares no api_key_env", name, mod.Provider) } // Non-local models must carry non-zero input/output prices (an unknown model // must never book at $0 and blind the ceiling). if prov.Kind != "local" && (mod.Price.InputPerM <= 0 || mod.Price.OutputPerM <= 0) { t.Errorf("model %s: non-local model needs input/output prices > 0 (got in=%.4f out=%.4f)", name, mod.Price.InputPerM, mod.Price.OutputPerM) } // The RESOLVED max_tokens floor (provider→model layering) is sane: never // negative, never absurdly large. if floor := m.MinMaxTokens(name); floor < 0 || floor > maxReasonableMinTokens { t.Errorf("model %s: resolved min_max_tokens %d is out of the sane band [0,%d]", name, floor, maxReasonableMinTokens) } } }