239 lines
11 KiB
Go
239 lines
11 KiB
Go
package llm
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"errors"
|
||
"log/slog"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// pack12_transport_test.go — drift-guards and behaviour pins for the research/21
|
||
// transport-hygiene pack (points 2, 3, 5, 6, 8a). Each is wire-neutral: none asserts
|
||
// on the request BODY, only on retry/terminal classification, backoff parsing, usage
|
||
// accounting and jitter bounds.
|
||
|
||
// TestRetryableStatusMap pins the ENTIRE status→retryable map (point 8а): a new status
|
||
// class cannot silently change retry policy without failing this test. 429 and 5xx are
|
||
// transient; a credits-exhausted 429 body is terminal; every other non-2xx is terminal.
|
||
func TestRetryableStatusMap(t *testing.T) {
|
||
plain := []byte(`{"error":{"message":"slow down"}}`)
|
||
quota := []byte(`{"error":{"code":"insufficient_quota","message":"you exceeded your current quota"}}`)
|
||
cases := []struct {
|
||
status int
|
||
body []byte
|
||
want bool
|
||
}{
|
||
{http.StatusBadRequest, plain, false}, // 400
|
||
{http.StatusUnauthorized, plain, false}, // 401
|
||
{http.StatusPaymentRequired, plain, false}, // 402 — credits, terminal
|
||
{http.StatusForbidden, plain, false}, // 403
|
||
{http.StatusNotFound, plain, false}, // 404
|
||
{http.StatusRequestTimeout, plain, false}, // 408 — we do NOT retry (codex-style avoided)
|
||
{http.StatusConflict, plain, false}, // 409
|
||
{http.StatusUnprocessableEntity, plain, false}, // 422 (Mistral extra_forbidden class)
|
||
{http.StatusTooManyRequests, plain, true}, // 429 plain → retry
|
||
{http.StatusTooManyRequests, quota, false}, // 429 credits exhausted → terminal
|
||
{http.StatusInternalServerError, plain, true}, // 500
|
||
{http.StatusBadGateway, plain, true}, // 502
|
||
{http.StatusServiceUnavailable, plain, true}, // 503
|
||
{529, plain, true}, // Anthropic overloaded (≥500)
|
||
}
|
||
for _, c := range cases {
|
||
if got := retryableStatus(c.status, c.body); got != c.want {
|
||
t.Errorf("retryableStatus(%d) = %v, want %v", c.status, got, c.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestQuota429IsTerminal (point 2): a 429 whose body marks credits exhausted fails loud
|
||
// in ONE attempt (no 3× retries under a held USD reservation), while a plain 429 still
|
||
// retries. Both narrow markers are exercised.
|
||
func TestQuota429IsTerminal(t *testing.T) {
|
||
for _, marker := range []string{"insufficient_quota", "quota_exceeded"} {
|
||
var calls atomic.Int32
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
calls.Add(1)
|
||
w.WriteHeader(http.StatusTooManyRequests)
|
||
w.Write([]byte(`{"error":{"code":"` + marker + `","message":"credits gone"}}`))
|
||
}))
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "test", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
_, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
var se *HTTPStatusError
|
||
if !errors.As(err, &se) || se.Status != http.StatusTooManyRequests {
|
||
t.Fatalf("%s: want typed 429, got %v", marker, err)
|
||
}
|
||
if calls.Load() != 1 {
|
||
t.Fatalf("%s: a credits-exhausted 429 must be terminal (1 call), got %d", marker, calls.Load())
|
||
}
|
||
srv.Close()
|
||
}
|
||
|
||
// A plain 429 (no credits marker) still retries to exhaustion.
|
||
var calls atomic.Int32
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
calls.Add(1)
|
||
w.WriteHeader(http.StatusTooManyRequests)
|
||
w.Write([]byte(`{"error":{"message":"rate limited, slow down"}}`))
|
||
}))
|
||
defer srv.Close()
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "test", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
_, _ = c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if calls.Load() != 3 {
|
||
t.Fatalf("a plain 429 must stay retryable (3 attempts), got %d", calls.Load())
|
||
}
|
||
}
|
||
|
||
// TestParseRetryAfterFractionalAndMs (point 3): Retry-After-Ms wins; fractional and
|
||
// integer seconds parse; an HTTP-date parses; anything else is 0. The old strconv.Atoi
|
||
// dropped "1.5"/ms → 0 → blind backoff.
|
||
func TestParseRetryAfterFractionalAndMs(t *testing.T) {
|
||
mk := func(kv ...string) http.Header {
|
||
h := http.Header{}
|
||
for i := 0; i+1 < len(kv); i += 2 {
|
||
h.Set(kv[i], kv[i+1])
|
||
}
|
||
return h
|
||
}
|
||
cases := []struct {
|
||
name string
|
||
h http.Header
|
||
want time.Duration
|
||
}{
|
||
{"ms wins over seconds", mk("Retry-After-Ms", "1500", "Retry-After", "9"), 1500 * time.Millisecond},
|
||
{"fractional ms", mk("Retry-After-Ms", "250.5"), 250500 * time.Microsecond},
|
||
{"fractional seconds", mk("Retry-After", "1.5"), 1500 * time.Millisecond},
|
||
{"integer seconds", mk("Retry-After", "3"), 3 * time.Second},
|
||
{"empty", mk(), 0},
|
||
{"garbage", mk("Retry-After", "soon"), 0},
|
||
{"zero ms ignored, falls to seconds", mk("Retry-After-Ms", "0", "Retry-After", "2"), 2 * time.Second},
|
||
}
|
||
for _, c := range cases {
|
||
if got := parseRetryAfter(c.h); got != c.want {
|
||
t.Errorf("%s: parseRetryAfter = %v, want %v", c.name, got, c.want)
|
||
}
|
||
}
|
||
// HTTP-date form resolves to a positive duration near the delta.
|
||
future := time.Now().Add(4 * time.Second).UTC().Format(http.TimeFormat)
|
||
if d := parseRetryAfter(mk("Retry-After", future)); d <= 0 || d > 5*time.Second {
|
||
t.Errorf("HTTP-date Retry-After = %v, want ~4s", d)
|
||
}
|
||
}
|
||
|
||
// TestAdditiveReasoningIdentityGuard (point 6): xAI additive reasoning is surfaced only
|
||
// when the identity total==prompt+completion+reasoning holds (or total is unreported);
|
||
// a broken identity means reasoning is already inside completion → treat as subset (0)
|
||
// and WARN, so the ledger never double-bills the (latent) 30–44% overcount.
|
||
func TestAdditiveReasoningIdentityGuard(t *testing.T) {
|
||
serve := func(body string) *httptest.Server {
|
||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { openAIOK(t, w, body) }))
|
||
}
|
||
call := func(srv *httptest.Server, logger *slog.Logger) *LLMResponse {
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "xai", BaseURL: srv.URL, Profile: fastProfile(), Reasoning: ReasoningAdditive}, logger)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return resp
|
||
}
|
||
|
||
// Identity holds: total=100 == prompt10 + completion40 + reasoning50 → surface 50.
|
||
srv := serve(`{"id":"a","choices":[{"message":{"content":"x"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":10,"completion_tokens":40,"total_tokens":100,"completion_tokens_details":{"reasoning_tokens":50}}}`)
|
||
if r := call(srv, nil); r.Usage.ReasoningTokens != 50 {
|
||
t.Fatalf("identity holds → reasoning must be 50, got %d", r.Usage.ReasoningTokens)
|
||
}
|
||
srv.Close()
|
||
|
||
// Identity BROKEN: total=60 ≠ 10+40+50=100 → reasoning already in completion → 0 + WARN.
|
||
var buf bytes.Buffer
|
||
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||
srv = serve(`{"id":"b","choices":[{"message":{"content":"x"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":10,"completion_tokens":40,"total_tokens":60,"completion_tokens_details":{"reasoning_tokens":50}}}`)
|
||
if r := call(srv, logger); r.Usage.ReasoningTokens != 0 {
|
||
t.Fatalf("broken identity → reasoning must be 0 (subset guard), got %d", r.Usage.ReasoningTokens)
|
||
}
|
||
srv.Close()
|
||
if !strings.Contains(buf.String(), "additive-reasoning identity broken") {
|
||
t.Fatalf("a broken identity must WARN as a quirk candidate, logs:\n%s", buf.String())
|
||
}
|
||
|
||
// Total unreported (0): unverifiable → keep the known xAI additive default.
|
||
srv = serve(`{"id":"c","choices":[{"message":{"content":"x"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":10,"completion_tokens":40,"completion_tokens_details":{"reasoning_tokens":50}}}`)
|
||
if r := call(srv, nil); r.Usage.ReasoningTokens != 50 {
|
||
t.Fatalf("total unreported → keep additive default 50, got %d", r.Usage.ReasoningTokens)
|
||
}
|
||
srv.Close()
|
||
}
|
||
|
||
// TestNormalizeOpenAIFinish pins point 8b (composite finish) AND the vendor-checked GLM
|
||
// mapping at the adapter boundary (docs.z.ai verified 2026-07-24). The Gemini composite
|
||
// `content_filter: PROHIBITED_CONTENT` and GLM's `sensitive` both normalize to the neutral
|
||
// content_filter, so the downstream classifier's exact match still fires — exact matchers
|
||
// are dead on those wires (D22.8а / research/21 §1.18б). Every other value — `network_error`,
|
||
// `model_context_window_exceeded`, an unknown, a lookalike without the colon — stays RAW so
|
||
// it remains FAIL-LOUD (never a silently-mapped clean stop, the litellm unknown→stop trap).
|
||
func TestNormalizeOpenAIFinish(t *testing.T) {
|
||
cases := map[string]string{
|
||
"content_filter: PROHIBITED_CONTENT": FinishContentFilter, // Gemini composite (point 8b)
|
||
"content_filter:SAFETY": FinishContentFilter, // composite, no space
|
||
"sensitive": FinishContentFilter, // GLM content filter (vendor-check)
|
||
"content_filter": FinishContentFilter, // bare, unchanged
|
||
"content_filtered_ok": "content_filtered_ok", // lookalike WITHOUT the colon → raw
|
||
"network_error": "network_error", // raw — fail-loud, NOT stop
|
||
"model_context_window_exceeded": "model_context_window_exceeded", // raw (deferred GLM-overflow item)
|
||
"stop": "stop",
|
||
"length": "length",
|
||
"": "",
|
||
}
|
||
for in, want := range cases {
|
||
if got := normalizeOpenAIFinish(in); got != want {
|
||
t.Errorf("normalizeOpenAIFinish(%q) = %q, want %q", in, got, want)
|
||
}
|
||
}
|
||
|
||
// End-to-end: a Gemini composite content_filter completion surfaces as neutral
|
||
// content_filter through the adapter.
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
openAIOK(t, w, `{"id":"g","choices":[{"message":{"content":"x"},"finish_reason":"content_filter: PROHIBITED_CONTENT"}],
|
||
"usage":{"prompt_tokens":10,"completion_tokens":1}}`)
|
||
}))
|
||
defer srv.Close()
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "gemini", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp.FinishReason != FinishContentFilter {
|
||
t.Fatalf("Gemini composite content_filter must surface as neutral content_filter, got %q", resp.FinishReason)
|
||
}
|
||
}
|
||
|
||
// TestProportionalJitterBounds (point 5): symmetric jitter stays within ±25% and never
|
||
// drives a wait non-positive; positive-only jitter (for an honoured Retry-After) never
|
||
// shortens the wait below the server hint.
|
||
func TestProportionalJitterBounds(t *testing.T) {
|
||
d := 20 * time.Second
|
||
for i := 0; i < 2000; i++ {
|
||
sym := proportionalJitter(d, true)
|
||
if sym < -d/4 || sym > d/4 {
|
||
t.Fatalf("symmetric jitter %v out of ±25%% of %v", sym, d)
|
||
}
|
||
if d+sym <= 0 {
|
||
t.Fatalf("jitter drove the wait non-positive: %v", d+sym)
|
||
}
|
||
pos := proportionalJitter(d, false)
|
||
if pos < 0 || pos > d/4 {
|
||
t.Fatalf("positive-only jitter %v out of [0,25%%] of %v", pos, d)
|
||
}
|
||
}
|
||
if got := proportionalJitter(0, true); got != 0 {
|
||
t.Fatalf("zero backoff must yield zero jitter, got %v", got)
|
||
}
|
||
}
|