210 lines
6.9 KiB
Go
210 lines
6.9 KiB
Go
package llm
|
||
|
||
import (
|
||
"bytes"
|
||
"encoding/json"
|
||
"testing"
|
||
)
|
||
|
||
// capability_test.go asserts the exact wire body the D3.1 resolver renders for
|
||
// each real provider quirk (00-provider-quirks): the hardcoded Phase-0 body
|
||
// 400s on Kimi / gpt-5 / Gemini, so the body shape IS the contract.
|
||
|
||
func marshalBody(t *testing.T, r openAIRequest) map[string]any {
|
||
t.Helper()
|
||
data, err := json.Marshal(r)
|
||
if err != nil {
|
||
t.Fatalf("marshal: %v", err)
|
||
}
|
||
var m map[string]any
|
||
if err := json.Unmarshal(data, &m); err != nil {
|
||
t.Fatalf("unmarshal: %v", err)
|
||
}
|
||
return m
|
||
}
|
||
|
||
func mustNum(t *testing.T, m map[string]any, k string, want float64) {
|
||
t.Helper()
|
||
v, ok := m[k]
|
||
if !ok {
|
||
t.Fatalf("key %q absent, want %v", k, want)
|
||
}
|
||
if f, ok := v.(float64); !ok || f != want {
|
||
t.Fatalf("key %q = %v, want %v", k, v, want)
|
||
}
|
||
}
|
||
|
||
func mustStr(t *testing.T, m map[string]any, k, want string) {
|
||
t.Helper()
|
||
if v, _ := m[k].(string); v != want {
|
||
t.Fatalf("key %q = %v, want %q", k, m[k], want)
|
||
}
|
||
}
|
||
|
||
func mustAbsent(t *testing.T, m map[string]any, k string) {
|
||
t.Helper()
|
||
if _, ok := m[k]; ok {
|
||
t.Fatalf("key %q must be absent, got %v", k, m[k])
|
||
}
|
||
}
|
||
|
||
func TestCapabilityWireBody(t *testing.T) {
|
||
base := openAIRequest{
|
||
model: "m",
|
||
messages: []openAIMessage{{Role: "user", Content: "u"}},
|
||
maxTokens: 4096,
|
||
temperature: 0.4,
|
||
}
|
||
|
||
tests := []struct {
|
||
name string
|
||
cap Capability
|
||
effort string
|
||
extra map[string]any
|
||
check func(t *testing.T, m map[string]any)
|
||
}{
|
||
{
|
||
name: "zero cap reproduces the Phase-0 wire",
|
||
effort: "off",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustNum(t, m, "max_tokens", 4096)
|
||
mustNum(t, m, "temperature", 0.4)
|
||
mustAbsent(t, m, "max_completion_tokens")
|
||
mustAbsent(t, m, "reasoning_effort")
|
||
},
|
||
},
|
||
{
|
||
name: "gpt-5: max_completion_tokens, omit temperature, off->minimal",
|
||
cap: Capability{Budget: BudgetMaxCompletionTokens, Temp: TempOmit, Reasoning: ReasoningCap{Control: ReasoningEffortField, OffEffort: "minimal"}},
|
||
effort: "off",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustNum(t, m, "max_completion_tokens", 4096)
|
||
mustAbsent(t, m, "max_tokens")
|
||
mustAbsent(t, m, "temperature")
|
||
mustStr(t, m, "reasoning_effort", "minimal")
|
||
},
|
||
},
|
||
{
|
||
name: "gemini mandatory: swallow off, never disable, temperature sent",
|
||
cap: Capability{Reasoning: ReasoningCap{Control: ReasoningMandatory}},
|
||
effort: "off",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustNum(t, m, "max_tokens", 4096)
|
||
mustNum(t, m, "temperature", 0.4)
|
||
mustAbsent(t, m, "reasoning_effort")
|
||
mustAbsent(t, m, "thinking")
|
||
},
|
||
},
|
||
{
|
||
name: "gemini mandatory swallows a NON-off effort too (escalation at high still 400-safe)",
|
||
cap: Capability{Reasoning: ReasoningCap{Control: ReasoningMandatory}},
|
||
effort: "high",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustAbsent(t, m, "reasoning_effort")
|
||
mustAbsent(t, m, "thinking")
|
||
mustNum(t, m, "temperature", 0.4)
|
||
},
|
||
},
|
||
{
|
||
name: "GLM extra_body_disable: off merges thinking:disabled, no reasoning_effort",
|
||
cap: Capability{Reasoning: ReasoningCap{Control: ReasoningExtraBodyDisable, OffExtraBody: map[string]any{"thinking": map[string]any{"type": "disabled"}}}},
|
||
effort: "off",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustAbsent(t, m, "reasoning_effort")
|
||
th, ok := m["thinking"].(map[string]any)
|
||
if !ok || th["type"] != "disabled" {
|
||
t.Fatalf("thinking disable not merged: %v", m["thinking"])
|
||
}
|
||
},
|
||
},
|
||
{
|
||
name: "GLM extra_body_disable: UNSET effort also disables (Phase-0 unconditional-disable preserved)",
|
||
cap: Capability{Reasoning: ReasoningCap{Control: ReasoningExtraBodyDisable, OffExtraBody: map[string]any{"thinking": map[string]any{"type": "disabled"}}}},
|
||
effort: "",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustAbsent(t, m, "reasoning_effort")
|
||
th, ok := m["thinking"].(map[string]any)
|
||
if !ok || th["type"] != "disabled" {
|
||
t.Fatalf("unset effort must still disable thinking (else GLM ×3 timeouts): %v", m["thinking"])
|
||
}
|
||
},
|
||
},
|
||
{
|
||
name: "GLM think-ON: on merges on_extra_body (D6.2)",
|
||
cap: Capability{Reasoning: ReasoningCap{Control: ReasoningExtraBodyDisable, OnExtraBody: map[string]any{"thinking": map[string]any{"type": "enabled"}}}},
|
||
effort: "high",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
th, ok := m["thinking"].(map[string]any)
|
||
if !ok || th["type"] != "enabled" {
|
||
t.Fatalf("think-ON not merged: %v", m["thinking"])
|
||
}
|
||
},
|
||
},
|
||
{
|
||
name: "kimi force temperature=1 regardless of the request",
|
||
cap: Capability{Temp: TempForce, TempValue: 1},
|
||
effort: "off",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustNum(t, m, "temperature", 1)
|
||
},
|
||
},
|
||
{
|
||
name: "reasoning none passes low/medium/high through",
|
||
cap: Capability{Reasoning: ReasoningCap{Control: ReasoningNone}},
|
||
effort: "medium",
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustStr(t, m, "reasoning_effort", "medium")
|
||
},
|
||
},
|
||
{
|
||
name: "extra fills gaps only, never overrides a resolved key",
|
||
effort: "off",
|
||
extra: map[string]any{"max_tokens": 999, "custom": "x"},
|
||
check: func(t *testing.T, m map[string]any) {
|
||
mustNum(t, m, "max_tokens", 4096) // resolved wins
|
||
mustStr(t, m, "custom", "x") // gap filled
|
||
},
|
||
},
|
||
}
|
||
|
||
for _, tc := range tests {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
r := base
|
||
r.cap = tc.cap
|
||
r.reasoningEffort = tc.effort
|
||
r.extra = tc.extra
|
||
m := marshalBody(t, r)
|
||
if m["stream"] != false {
|
||
t.Fatalf("stream must serialize as false, got %v", m["stream"])
|
||
}
|
||
tc.check(t, m)
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestGLMMigrationWireIdentical proves the D3.1 migration of GLM's thinking
|
||
// disable from model.extra_body to capabilities.reasoning.off_extra_body leaves
|
||
// the WIRE body byte-identical — the snapshot changes (an intended one-time
|
||
// resnapshot) but not the request, so no live GLM call diverges (D5.2).
|
||
func TestGLMMigrationWireIdentical(t *testing.T) {
|
||
msgs := []openAIMessage{{Role: "system", Content: "s"}, {Role: "user", Content: "u"}}
|
||
legacy := openAIRequest{ // Phase-0 glm-5: default cap + extra_body disable switch
|
||
model: "glm-5", messages: msgs, maxTokens: 4096, temperature: 0.4, reasoningEffort: "off",
|
||
extra: map[string]any{"thinking": map[string]any{"type": "disabled"}},
|
||
}
|
||
migrated := openAIRequest{ // Phase-1 glm-5: disable via capabilities.reasoning.off_extra_body
|
||
model: "glm-5", messages: msgs, maxTokens: 4096, temperature: 0.4, reasoningEffort: "off",
|
||
cap: Capability{Reasoning: ReasoningCap{Control: ReasoningExtraBodyDisable, OffExtraBody: map[string]any{"thinking": map[string]any{"type": "disabled"}}}},
|
||
}
|
||
lb, err := json.Marshal(legacy)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
mb, err := json.Marshal(migrated)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !bytes.Equal(lb, mb) {
|
||
t.Fatalf("GLM migration changed the wire body:\n legacy=%s\n migrated=%s", lb, mb)
|
||
}
|
||
}
|