96 lines
3.5 KiB
Go
96 lines
3.5 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// additive_reasoning_test.go is the D13.6 fail-fast gate: a stage that THINKS
|
|
// (reasoning low|medium|high) on an ADDITIVE-billing provider (xAI — reasoning bills on top
|
|
// of completion) must declare reasoning_max_tokens, or LoadPipeline fails loud — the same
|
|
// structural discipline as the echo mine. Reverting the check in LoadPipeline makes the
|
|
// "armed" cases below stop failing.
|
|
func TestReasoningBufferFailFast(t *testing.T) {
|
|
today := time.Now().UTC().Format("2006-01-02")
|
|
dir := t.TempDir()
|
|
// A prompt template LoadPipeline can stat + split.
|
|
promptPath := filepath.Join(dir, "p.md")
|
|
if err := os.WriteFile(promptPath, []byte("sys\n---USER---\n{{text}}"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
modelsPath := filepath.Join(dir, "models.yaml")
|
|
if err := os.WriteFile(modelsPath, []byte(fmt.Sprintf(`
|
|
prices_checked: %q
|
|
default_model: additive-model
|
|
providers:
|
|
xai:
|
|
kind: openai
|
|
base_url: http://x
|
|
reasoning: additive
|
|
ds:
|
|
kind: openai
|
|
base_url: http://y
|
|
reasoning: subset
|
|
models:
|
|
additive-model: { provider: xai, price: { input_per_m: 1, output_per_m: 2 } }
|
|
subset-model: { provider: ds, price: { input_per_m: 1, output_per_m: 2 } }
|
|
`, today)), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
models, err := LoadModels(modelsPath)
|
|
if err != nil {
|
|
t.Fatalf("models load: %v", err)
|
|
}
|
|
|
|
load := func(stage string) error {
|
|
pp := filepath.Join(dir, "pipe.yaml")
|
|
body := fmt.Sprintf("core: C1\nversion: 1\ndefaults: { max_output_ratio: 2, min_max_tokens: 512 }\nstages:\n%s", stage)
|
|
if err := os.WriteFile(pp, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, e := LoadPipeline(pp, models)
|
|
return e
|
|
}
|
|
stage := func(model, reasoning string, buf int, escalateTo string) string {
|
|
s := fmt.Sprintf(" - { name: s, role: translator, model: %s, prompt: %s, prompt_version: v, reasoning: %q", model, promptPath, reasoning)
|
|
if buf > 0 {
|
|
s += fmt.Sprintf(", reasoning_max_tokens: %d", buf)
|
|
}
|
|
if escalateTo != "" {
|
|
s += fmt.Sprintf(", escalate_to: %s", escalateTo)
|
|
}
|
|
return s + " }"
|
|
}
|
|
|
|
cases := []struct {
|
|
name string
|
|
stage string
|
|
wantErr bool
|
|
}{
|
|
{"think-on additive no buffer → fail", stage("additive-model", "high", 0, ""), true},
|
|
{"think-on additive with buffer → ok", stage("additive-model", "medium", 4000, ""), false},
|
|
{"think-on subset no buffer → ok", stage("subset-model", "high", 0, ""), false},
|
|
{"off additive no buffer → ok", stage("additive-model", "off", 0, ""), false},
|
|
// finding #4: "" on an additive provider falls to the grok default (thinks) → MUST fail.
|
|
{"empty-reasoning additive no buffer → fail", stage("additive-model", "", 0, ""), true},
|
|
{"empty-reasoning additive with buffer → ok", stage("additive-model", "", 4000, ""), false},
|
|
{"empty-reasoning subset no buffer → ok", stage("subset-model", "", 0, ""), false},
|
|
{"escalate_to additive think-on no buffer → fail", stage("subset-model", "high", 0, "additive-model"), true},
|
|
{"escalate_to additive empty no buffer → fail", stage("subset-model", "", 0, "additive-model"), true},
|
|
{"escalate_to additive think-on with buffer → ok", stage("subset-model", "high", 4000, "additive-model"), false},
|
|
}
|
|
for _, c := range cases {
|
|
err := load(c.stage)
|
|
if c.wantErr {
|
|
if err == nil || !strings.Contains(err.Error(), "reasoning_max_tokens") {
|
|
t.Errorf("%s: want an error mentioning reasoning_max_tokens, got %v", c.name, err)
|
|
}
|
|
} else if err != nil {
|
|
t.Errorf("%s: want ok, got %v", c.name, err)
|
|
}
|
|
}
|
|
}
|