73 lines
3.3 KiB
Go
73 lines
3.3 KiB
Go
package config
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
// models_catalog_test.go — the pack-12 point-1 catalog validation test (research/21
|
|
// §5.3, goose declarative.rs:396-467 `all_bundled_providers_are_valid`): a $0 go-test
|
|
// over the SHIPPED configs/models.yaml that fails on catalog drift BEFORE a paid run.
|
|
// LoadModels already fail-fasts most of this at load; the test PINS that the shipped
|
|
// catalog currently passes (a stale prices_checked, an undeclared provider, a bad
|
|
// reasoning/capability enum, a re-armed echo mine all turn this red), and adds the two
|
|
// structural invariants LoadModels does not itself assert: every model-referenced
|
|
// non-local provider declares an api_key_env, and every resolved min_max_tokens floor
|
|
// is in a sane band.
|
|
|
|
const shippedModelsYAML = "../../configs/models.yaml"
|
|
|
|
// maxReasonableMinTokens bounds a per-model max_tokens floor. The real floors are
|
|
// 8000 (DeepSeek/Gemini) and 16000 (Kimi); a value above this band is almost
|
|
// certainly a typo (a token count written as a price, an extra zero) that would
|
|
// over-reserve every call on that model.
|
|
const maxReasonableMinTokens = 200000
|
|
|
|
func TestShippedModelsCatalogValid(t *testing.T) {
|
|
m, err := LoadModels(shippedModelsYAML)
|
|
if err != nil {
|
|
// LoadModels aggregates ALL problems into one error — surface it verbatim so
|
|
// the operator fixes the drift (stale prices, undeclared provider, bad enum,
|
|
// echo mine) before spending a dollar.
|
|
t.Fatalf("shipped configs/models.yaml failed validation (catalog drift before a paid run):\n%v", err)
|
|
}
|
|
|
|
// default_model resolves (price-fallback anchor).
|
|
if _, ok := m.Models[m.DefaultModel]; !ok {
|
|
t.Fatalf("default_model %q is not a defined model", m.DefaultModel)
|
|
}
|
|
|
|
knownReasoning := map[string]bool{"": true, "subset": true, "additive": true, "additive_total": true}
|
|
for name, prov := range m.Providers {
|
|
switch prov.Kind {
|
|
case "openai", "anthropic", "local":
|
|
default:
|
|
t.Errorf("provider %s: unknown kind %q", name, prov.Kind)
|
|
}
|
|
if prov.Kind == "openai" && !knownReasoning[prov.Reasoning] {
|
|
t.Errorf("provider %s: reasoning %q not in {subset,additive,additive_total}", name, prov.Reasoning)
|
|
}
|
|
}
|
|
|
|
for name, mod := range m.Models {
|
|
prov, ok := m.Providers[mod.Provider]
|
|
if !ok {
|
|
t.Errorf("model %s references undeclared provider %q", name, mod.Provider)
|
|
continue
|
|
}
|
|
// Every model-reachable NON-LOCAL provider must declare an api_key_env: a paid
|
|
// model with no key env would surface as a 401 only after reserve/slot charge.
|
|
if prov.Kind != "local" && prov.APIKeyEnv == "" {
|
|
t.Errorf("model %s: non-local provider %s declares no api_key_env", name, mod.Provider)
|
|
}
|
|
// Non-local models must carry non-zero input/output prices (an unknown model
|
|
// must never book at $0 and blind the ceiling).
|
|
if prov.Kind != "local" && (mod.Price.InputPerM <= 0 || mod.Price.OutputPerM <= 0) {
|
|
t.Errorf("model %s: non-local model needs input/output prices > 0 (got in=%.4f out=%.4f)", name, mod.Price.InputPerM, mod.Price.OutputPerM)
|
|
}
|
|
// The RESOLVED max_tokens floor (provider→model layering) is sane: never
|
|
// negative, never absurdly large.
|
|
if floor := m.MinMaxTokens(name); floor < 0 || floor > maxReasonableMinTokens {
|
|
t.Errorf("model %s: resolved min_max_tokens %d is out of the sane band [0,%d]", name, floor, maxReasonableMinTokens)
|
|
}
|
|
}
|
|
}
|