322 lines
14 KiB
Go
322 lines
14 KiB
Go
package llm
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"log/slog"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// Scenario tests of the wire contract via httptest — donor style (real
|
||
// HTTP round-trips, not transport mocks).
|
||
|
||
func fastProfile() RetryProfile {
|
||
return RetryProfile{AttemptTimeout: 2 * time.Second, MaxAttempts: 3, BackoffBase: time.Millisecond, BackoffCap: 5 * time.Millisecond}
|
||
}
|
||
|
||
func openAIOK(t *testing.T, w http.ResponseWriter, body string) {
|
||
t.Helper()
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if _, err := w.Write([]byte(body)); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
func TestOpenAICompatHappyPath(t *testing.T) {
|
||
var gotBody map[string]any
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if r.URL.Path != "/chat/completions" {
|
||
t.Errorf("path = %s", r.URL.Path)
|
||
}
|
||
if got := r.Header.Get("Authorization"); got != "Bearer k" {
|
||
t.Errorf("auth = %q", got)
|
||
}
|
||
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
openAIOK(t, w, `{"id":"r1","model":"m-actual","choices":[{"message":{"content":"привет"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":20,"prompt_tokens_details":{"cached_tokens":40}}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "test", BaseURL: srv.URL, APIKey: "k", Profile: fastProfile(),
|
||
ExtraBody: map[string]any{"thinking": map[string]any{"type": "disabled"}}}, nil)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{
|
||
Model: "m",
|
||
Messages: []Message{{Role: "system", Content: "s"}, {Role: "user", Content: "u"}},
|
||
MaxTokens: 256,
|
||
Temperature: 0.3,
|
||
})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp.Text != "привет" || resp.Model != "m-actual" || resp.FinishReason != FinishStop {
|
||
t.Fatalf("resp = %+v", resp)
|
||
}
|
||
if resp.Usage.PromptTokens != 100 || resp.Usage.CachedTokens != 40 || resp.Usage.CompletionTokens != 20 {
|
||
t.Fatalf("usage = %+v", resp.Usage)
|
||
}
|
||
if resp.Usage.ReasoningTokens != 0 {
|
||
t.Fatalf("subset semantics must keep reasoning 0, got %d", resp.Usage.ReasoningTokens)
|
||
}
|
||
// ExtraBody from models.yaml is merged into the request body.
|
||
if _, ok := gotBody["thinking"]; !ok {
|
||
t.Fatalf("extra_body not merged into wire body: %v", gotBody)
|
||
}
|
||
if gotBody["stream"] != false {
|
||
t.Fatalf("stream must be false, got %v", gotBody["stream"])
|
||
}
|
||
}
|
||
|
||
func TestDeepSeekCacheFieldsVariant(t *testing.T) {
|
||
// DeepSeek usage variant: prompt_cache_hit_tokens instead of
|
||
// prompt_tokens_details — otherwise the cache experiment would show a false zero.
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
openAIOK(t, w, `{"id":"r2","choices":[{"message":{"content":"x"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":100,"completion_tokens":5,"prompt_cache_hit_tokens":64,"prompt_cache_miss_tokens":36}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "deepseek", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp.Usage.CachedTokens != 64 {
|
||
t.Fatalf("deepseek cache-hit tokens not parsed: %+v", resp.Usage)
|
||
}
|
||
}
|
||
|
||
func TestAdditiveReasoningSemantics(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
openAIOK(t, w, `{"id":"r3","choices":[{"message":{"content":"x"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":10,"completion_tokens":5,"completion_tokens_details":{"reasoning_tokens":50}}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
// additive (xAI): reasoning on top of completion.
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "xai", BaseURL: srv.URL, Profile: fastProfile(), Reasoning: ReasoningAdditive}, nil)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp.Usage.ReasoningTokens != 50 {
|
||
t.Fatalf("additive semantics must surface reasoning tokens, got %+v", resp.Usage)
|
||
}
|
||
|
||
// subset: same body, but reasoning stays 0 (otherwise double billing).
|
||
c2 := NewOpenAICompatClient(OpenAICompatConfig{Name: "openai", BaseURL: srv.URL, Profile: fastProfile(), Reasoning: ReasoningSubset}, nil)
|
||
resp2, err := c2.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp2.Usage.ReasoningTokens != 0 {
|
||
t.Fatalf("subset semantics must keep reasoning 0, got %+v", resp2.Usage)
|
||
}
|
||
}
|
||
|
||
// TestAdditiveTotalReasoningSemantics covers the Gemini case (live-probe 2026-07-10): thinking
|
||
// tokens are billed as output but appear ONLY in total_tokens — completion_tokens excludes them
|
||
// and there is no reasoning_tokens field. The adapter must derive reasoning = total − prompt −
|
||
// completion so the mandatory-thinking spend is not invisible to the ledger. Reverting the
|
||
// ReasoningAdditiveTotal branch drops reasoning to 0 and fails this test.
|
||
func TestAdditiveTotalReasoningSemantics(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
// Mirrors the real Gemini shape: completion=2, prompt=22, total=847 → 823 hidden thinking,
|
||
// NO completion_tokens_details.reasoning_tokens field at all.
|
||
openAIOK(t, w, `{"id":"g1","model":"gemini-3.1-pro-preview","choices":[{"message":{"content":"391"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":22,"completion_tokens":2,"total_tokens":847}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "gemini", BaseURL: srv.URL, Profile: fastProfile(), Reasoning: ReasoningAdditiveTotal}, nil)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 8192})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp.Usage.ReasoningTokens != 823 {
|
||
t.Fatalf("additive_total must derive hidden thinking = total−prompt−completion = 823, got %+v", resp.Usage)
|
||
}
|
||
if resp.Usage.CompletionTokens != 2 {
|
||
t.Fatalf("completion tokens must stay the visible 2 (thinking is separate reasoning), got %+v", resp.Usage)
|
||
}
|
||
|
||
// A spec provider whose total == prompt+completion must NOT derive a phantom reasoning count
|
||
// (guard the >0 branch — never inflate the ceiling).
|
||
srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
openAIOK(t, w, `{"id":"g2","choices":[{"message":{"content":"x"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`)
|
||
}))
|
||
defer srv2.Close()
|
||
c2 := NewOpenAICompatClient(OpenAICompatConfig{Name: "gemini", BaseURL: srv2.URL, Profile: fastProfile(), Reasoning: ReasoningAdditiveTotal}, nil)
|
||
resp2, err := c2.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp2.Usage.ReasoningTokens != 0 {
|
||
t.Fatalf("total==prompt+completion must derive 0 reasoning, got %+v", resp2.Usage)
|
||
}
|
||
}
|
||
|
||
func TestRetryOn429HonoursRetryAfter(t *testing.T) {
|
||
var calls atomic.Int32
|
||
var firstRetryAt, secondCallAt time.Time
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
n := calls.Add(1)
|
||
if n == 1 {
|
||
firstRetryAt = time.Now()
|
||
w.Header().Set("Retry-After", "1")
|
||
w.WriteHeader(http.StatusTooManyRequests)
|
||
return
|
||
}
|
||
secondCallAt = time.Now()
|
||
openAIOK(t, w, `{"id":"r","choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "test", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp.Text != "ok" || calls.Load() != 2 {
|
||
t.Fatalf("text=%q calls=%d", resp.Text, calls.Load())
|
||
}
|
||
// Retry-After: 1s must override the profile's millisecond backoff.
|
||
if wait := secondCallAt.Sub(firstRetryAt); wait < 900*time.Millisecond {
|
||
t.Fatalf("Retry-After not honoured: waited only %v", wait)
|
||
}
|
||
}
|
||
|
||
func TestTerminal4xxNoRetry(t *testing.T) {
|
||
var calls atomic.Int32
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
calls.Add(1)
|
||
w.WriteHeader(http.StatusBadRequest)
|
||
w.Write([]byte(`{"error":"bad model"}`))
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "test", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
_, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err == nil {
|
||
t.Fatal("terminal 4xx must fail")
|
||
}
|
||
var se *HTTPStatusError
|
||
if !errors.As(err, &se) || se.Status != 400 {
|
||
t.Fatalf("want typed 400, got %v", err)
|
||
}
|
||
if calls.Load() != 1 {
|
||
t.Fatalf("terminal 4xx must not retry, calls=%d", calls.Load())
|
||
}
|
||
}
|
||
|
||
func TestEmptyContent2xxIsSuccess(t *testing.T) {
|
||
// 2xx with empty content — a billed call: the transport returns success,
|
||
// the money is booked, the gates deal with the empty text.
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
openAIOK(t, w, `{"id":"r","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":0}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "test", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil {
|
||
t.Fatalf("empty-content 2xx must be a success (billed call): %v", err)
|
||
}
|
||
if resp.Text != "" || resp.Usage.PromptTokens != 10 {
|
||
t.Fatalf("resp = %+v", resp)
|
||
}
|
||
}
|
||
|
||
func TestRetryOn5xxThenSuccess(t *testing.T) {
|
||
var calls atomic.Int32
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if calls.Add(1) < 3 {
|
||
w.WriteHeader(http.StatusBadGateway)
|
||
return
|
||
}
|
||
openAIOK(t, w, `{"id":"r","choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "test", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err != nil || resp.Text != "ok" {
|
||
t.Fatalf("resp=%v err=%v", resp, err)
|
||
}
|
||
if calls.Load() != 3 {
|
||
t.Fatalf("calls = %d", calls.Load())
|
||
}
|
||
}
|
||
|
||
// TestRetryLoopLogsWillRetryOnlyWhileAttemptsRemain pins the retryLoop logging contract
|
||
// (D23.4, httpllm.go:87-89): a "will retry" WARN is emitted ONLY when an attempt actually
|
||
// remains — carrying the REAL wait (retry_in), which Retry-After can silently stretch —
|
||
// and NEVER before the terminal exhausted error. Before the fix the last failure also
|
||
// logged "will retry" right before giving up. slog-capture, no live provider assertion.
|
||
func TestRetryLoopLogsWillRetryOnlyWhileAttemptsRemain(t *testing.T) {
|
||
var buf bytes.Buffer
|
||
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
||
var calls atomic.Int32
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
calls.Add(1)
|
||
w.WriteHeader(http.StatusBadGateway) // always 5xx → retryable, runs to exhaustion
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "prov", BaseURL: srv.URL, Profile: fastProfile()}, logger)
|
||
_, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
if err == nil {
|
||
t.Fatal("a persistent 5xx must exhaust to an error")
|
||
}
|
||
if calls.Load() != 3 { // fastProfile MaxAttempts=3
|
||
t.Fatalf("want 3 attempts, got %d", calls.Load())
|
||
}
|
||
logs := buf.String()
|
||
// Exactly MaxAttempts-1 (2) retry warnings: after attempts 1 and 2, never after the
|
||
// last one (the loop breaks on att+1>=MaxAttempts before the WARN).
|
||
if n := strings.Count(logs, "attempt failed, will retry"); n != 2 {
|
||
t.Fatalf("want exactly 2 'will retry' logs (none on the terminal attempt), got %d:\n%s", n, logs)
|
||
}
|
||
// Each retry log carries the real wait it is about to sleep.
|
||
if !strings.Contains(logs, "retry_in=") {
|
||
t.Fatalf("a retry log must carry retry_in (the actual wait, Retry-After-aware):\n%s", logs)
|
||
}
|
||
// The final outcome is the exhausted error, not another "will retry".
|
||
if !strings.Contains(err.Error(), "exhausted") {
|
||
t.Fatalf("terminal error must say exhausted, got %v", err)
|
||
}
|
||
}
|
||
|
||
// TestBilledDecodeErrorReBillCap pins the pack-12 point-9 cap (research/21 §1.18в): a
|
||
// 2xx with an unreadable body has ALREADY billed, so it is retried AT MOST ONCE — a
|
||
// second billed-decode is terminal, never the full MaxAttempts. Before the cap a
|
||
// provider emitting persistent 2xx garbage burned all 3 attempts (a paid retry storm);
|
||
// now the runner receives the typed BilledDecodeError after 2 calls and settles the
|
||
// estimate. The one retry is preserved so a transient proxy break still gets a second shot.
|
||
func TestBilledDecodeErrorReBillCap(t *testing.T) {
|
||
var calls atomic.Int32
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
calls.Add(1)
|
||
w.Write([]byte(`{"truncated`)) // body truncated by a proxy — undecodable, but billed
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{Name: "test", BaseURL: srv.URL, Profile: fastProfile()}, nil)
|
||
_, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
|
||
var bde *BilledDecodeError
|
||
if err == nil || !errors.As(err, &bde) {
|
||
t.Fatalf("want BilledDecodeError, got %v", err)
|
||
}
|
||
if calls.Load() != 2 { // fastProfile MaxAttempts=3, but the billed-decode cap stops at 2
|
||
t.Fatalf("a billed 2xx-garbage must be re-billed at most once (2 calls), got %d", calls.Load())
|
||
}
|
||
}
|