137 lines
4.1 KiB
Go
137 lines
4.1 KiB
Go
package llm
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"testing"
|
|
)
|
|
|
|
// fakeLLM — interface stand-in (donor style): a fixed response or an error.
|
|
type fakeLLM struct {
|
|
resp *LLMResponse
|
|
err error
|
|
calls int
|
|
}
|
|
|
|
func (f *fakeLLM) Complete(_ context.Context, _ LLMRequest) (*LLMResponse, error) {
|
|
f.calls++
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
return f.resp, nil
|
|
}
|
|
|
|
func quietLogger() *slog.Logger { return slog.New(slog.DiscardHandler) }
|
|
|
|
// newTestFailover assembles the decorator without a prober (healthy is set by
|
|
// hand via noteProbe) — a deterministic breaker machine, like the donor's
|
|
// failover_test.
|
|
func newTestFailover(primary, fallback LLMClient) *failoverClient {
|
|
return &failoverClient{
|
|
primary: primary, fallback: fallback,
|
|
log: quietLogger(), legTimeout: 1e9, needOK: 1,
|
|
}
|
|
}
|
|
|
|
func TestFailoverUnhealthyGoesStraightToCloud(t *testing.T) {
|
|
local := &fakeLLM{resp: &LLMResponse{Text: "local"}}
|
|
cloud := &fakeLLM{resp: &LLMResponse{Text: "cloud"}}
|
|
f := newTestFailover(local, cloud) // healthy=false at start
|
|
|
|
resp, err := f.Complete(context.Background(), LLMRequest{})
|
|
if err != nil || resp.Text != "cloud" {
|
|
t.Fatalf("resp=%v err=%v", resp, err)
|
|
}
|
|
if local.calls != 0 {
|
|
t.Fatal("unhealthy leg must not be tried")
|
|
}
|
|
}
|
|
|
|
func TestFailoverHealthyServesLocal(t *testing.T) {
|
|
local := &fakeLLM{resp: &LLMResponse{Text: "local", Model: "local-model"}}
|
|
cloud := &fakeLLM{resp: &LLMResponse{Text: "cloud"}}
|
|
f := newTestFailover(local, cloud)
|
|
f.noteProbe(true)
|
|
|
|
resp, err := f.Complete(context.Background(), LLMRequest{})
|
|
if err != nil || resp.Text != "local" {
|
|
t.Fatalf("resp=%v err=%v", resp, err)
|
|
}
|
|
if cloud.calls != 0 {
|
|
t.Fatal("cloud must not be billed when local serves")
|
|
}
|
|
}
|
|
|
|
func TestFailoverEmptyLocalRetriesOnCloud(t *testing.T) {
|
|
// An empty 2xx from the local leg (thinking ate the budget) — free, so we
|
|
// retry on the cloud; the breaker does NOT trip.
|
|
local := &fakeLLM{resp: &LLMResponse{Text: " "}}
|
|
cloud := &fakeLLM{resp: &LLMResponse{Text: "cloud"}}
|
|
f := newTestFailover(local, cloud)
|
|
f.noteProbe(true)
|
|
|
|
resp, err := f.Complete(context.Background(), LLMRequest{})
|
|
if err != nil || resp.Text != "cloud" {
|
|
t.Fatalf("resp=%v err=%v", resp, err)
|
|
}
|
|
if !f.isHealthy() {
|
|
t.Fatal("empty content must not trip the breaker")
|
|
}
|
|
}
|
|
|
|
func TestFailoverTerminal4xxFailsLoud(t *testing.T) {
|
|
// A terminal 4xx from the local leg — a config error (wrong model tag);
|
|
// the cloud must not mask it.
|
|
local := &fakeLLM{err: &HTTPStatusError{Provider: "local", Status: 404, Body: "no such model"}}
|
|
cloud := &fakeLLM{resp: &LLMResponse{Text: "cloud"}}
|
|
f := newTestFailover(local, cloud)
|
|
f.noteProbe(true)
|
|
|
|
_, err := f.Complete(context.Background(), LLMRequest{})
|
|
var se *HTTPStatusError
|
|
if err == nil || !errors.As(err, &se) || se.Status != 404 {
|
|
t.Fatalf("want loud 404, got %v", err)
|
|
}
|
|
if cloud.calls != 0 {
|
|
t.Fatal("terminal 4xx must not be masked by the cloud")
|
|
}
|
|
}
|
|
|
|
func TestFailover5xxTripsBreakerAndFallsBack(t *testing.T) {
|
|
local := &fakeLLM{err: &HTTPStatusError{Provider: "local", Status: 502, Body: "boom"}}
|
|
cloud := &fakeLLM{resp: &LLMResponse{Text: "cloud"}}
|
|
f := newTestFailover(local, cloud)
|
|
f.noteProbe(true)
|
|
|
|
resp, err := f.Complete(context.Background(), LLMRequest{})
|
|
if err != nil || resp.Text != "cloud" {
|
|
t.Fatalf("resp=%v err=%v", resp, err)
|
|
}
|
|
if f.isHealthy() {
|
|
t.Fatal("5xx must trip the breaker")
|
|
}
|
|
// Anti-flapping: after a request-trip one probe success is not enough.
|
|
f.noteProbe(true)
|
|
if f.isHealthy() {
|
|
t.Fatal("one probe success must not re-close a request-tripped breaker")
|
|
}
|
|
f.noteProbe(true)
|
|
if !f.isHealthy() {
|
|
t.Fatal("two consecutive probe successes must re-close the breaker")
|
|
}
|
|
}
|
|
|
|
func Test429FallsBackLikeTimeout(t *testing.T) {
|
|
// 429 — a busy single-slot GPU, falls back to the cloud like a timeout (not
|
|
// like a terminal 4xx).
|
|
local := &fakeLLM{err: &HTTPStatusError{Provider: "local", Status: 429, Body: "busy"}}
|
|
cloud := &fakeLLM{resp: &LLMResponse{Text: "cloud"}}
|
|
f := newTestFailover(local, cloud)
|
|
f.noteProbe(true)
|
|
|
|
resp, err := f.Complete(context.Background(), LLMRequest{})
|
|
if err != nil || resp.Text != "cloud" {
|
|
t.Fatalf("resp=%v err=%v", resp, err)
|
|
}
|
|
}
|