237 lines
8.6 KiB
Go
237 lines
8.6 KiB
Go
package llm
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"io"
|
||
"log/slog"
|
||
"net/http"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
// failover.go is the local-first routing decorator over the LLMClient seam:
|
||
// prefer the (free) home backend while it is reachable, fall back to the cloud
|
||
// transparently when it is not. Порт vojo failover.go; конструктор принимает
|
||
// явные параметры вместо толстого *Config, пробер использует no-proxy
|
||
// транспорт (та же прокси-ловушка, что у provider_local).
|
||
//
|
||
// Channel-isolation caveat (03-implementation-notes §3.5): the fallback leg is
|
||
// wired by the CALLER per role — for NSFW channel B (Фаза 2) the fallback must
|
||
// itself be a permissive provider or absent (fail loud), otherwise an empty
|
||
// answer from the local abliterated model would silently ship the flagged
|
||
// chunk to a restricted cloud provider.
|
||
//
|
||
// Health is decided by an ACTIVE background probe plus a request-path circuit
|
||
// breaker, both feeding one healthy flag:
|
||
// - unhealthy → requests go straight to the cloud;
|
||
// - healthy → try local under its own deadline; on transport/5xx/timeout
|
||
// trip the breaker and retry the SAME request on the cloud;
|
||
// - recovery is probe-driven ONLY (anti-flapping): a tripped breaker needs
|
||
// tripProbesToClose consecutive probe successes to close.
|
||
//
|
||
// A terminal 4xx from the local leg (except 429) does NOT fall back: that is a
|
||
// config/request error (wrong model tag, malformed body) and the cloud would
|
||
// only mask it. 429 (a busy single-slot GPU) falls back like a timeout.
|
||
|
||
const (
|
||
// localProbeTimeout bounds one health probe: a bare GET /models is answered
|
||
// from the server's process memory, no inference — slower means the path,
|
||
// not the model, is struggling.
|
||
localProbeTimeout = 3 * time.Second
|
||
// tripProbesToClose is how many consecutive probe successes re-close a
|
||
// breaker tripped by a REQUEST failure (probe can pass while requests fail:
|
||
// a cold model being loaded, a half-alive path).
|
||
tripProbesToClose = 2
|
||
// neverOnlineWarnAfter: consecutive probe failures — without the leg EVER
|
||
// having come online since boot — that trigger a one-time WARN, so an
|
||
// enabled-but-dead local leg can't silently serve 100% of traffic from the
|
||
// paid cloud.
|
||
neverOnlineWarnAfter = 3
|
||
)
|
||
|
||
// FailoverConfig wires the decorator.
|
||
type FailoverConfig struct {
|
||
LegTimeout time.Duration // per-request deadline for the local leg
|
||
ProbeURL string // <local base URL>/models
|
||
ProbeKey string // optional bearer for the probe (usually empty)
|
||
ProbeInterval time.Duration // background probe period
|
||
}
|
||
|
||
type failoverClient struct {
|
||
primary LLMClient // local backend (free, preferred)
|
||
fallback LLMClient // cloud (paid)
|
||
log *slog.Logger
|
||
|
||
legTimeout time.Duration
|
||
probeURL string
|
||
probeKey string
|
||
interval time.Duration
|
||
httpc *http.Client
|
||
|
||
// Breaker state. healthy starts false: until the first probe passes,
|
||
// traffic is served by the cloud, so a boot with the home box offline never
|
||
// eats the probe timeout on user requests.
|
||
mu sync.Mutex
|
||
healthy bool
|
||
consecOK int
|
||
needOK int
|
||
consecFail int
|
||
everHealthy bool
|
||
warnedDown bool
|
||
}
|
||
|
||
// NewFailoverClient wraps primary (local) and fallback (cloud) into one
|
||
// LLMClient and starts the background health probe, which lives until ctx is
|
||
// cancelled (process lifetime).
|
||
func NewFailoverClient(ctx context.Context, primary, fallback LLMClient, cfg FailoverConfig, logger *slog.Logger) LLMClient {
|
||
if cfg.LegTimeout <= 0 {
|
||
cfg.LegTimeout = 300 * time.Second // полигон: чанки 63–278 с; 45 с vojo непригодны
|
||
}
|
||
if cfg.ProbeInterval <= 0 {
|
||
cfg.ProbeInterval = 30 * time.Second
|
||
}
|
||
c := &failoverClient{
|
||
primary: primary,
|
||
fallback: fallback,
|
||
log: logger,
|
||
legTimeout: cfg.LegTimeout,
|
||
probeURL: cfg.ProbeURL,
|
||
probeKey: cfg.ProbeKey,
|
||
interval: cfg.ProbeInterval,
|
||
httpc: NoProxyClient(),
|
||
needOK: 1,
|
||
}
|
||
logger.Info("local llm failover enabled; cloud serves until the first successful probe", "probe_url", c.probeURL)
|
||
go c.probeLoop(ctx)
|
||
return c
|
||
}
|
||
|
||
func (c *failoverClient) Complete(ctx context.Context, req LLMRequest) (*LLMResponse, error) {
|
||
if !c.isHealthy() {
|
||
c.log.DebugContext(ctx, "local llm unhealthy; serving from cloud")
|
||
return c.fallback.Complete(ctx, req)
|
||
}
|
||
|
||
lctx, cancel := context.WithTimeout(ctx, c.legTimeout)
|
||
resp, err := c.primary.Complete(lctx, req)
|
||
cancel()
|
||
if err == nil {
|
||
if strings.TrimSpace(resp.Text) != "" {
|
||
return resp, nil
|
||
}
|
||
// A 2xx with EMPTY content from the local leg (thinking ate the token
|
||
// budget, a quirk of the abliterated model). The local call is free, so
|
||
// retry the turn on the cloud. The backend is up: not a breaker trip.
|
||
// INFO, не DEBUG: каждый такой ретрай — реальные облачные деньги на роли,
|
||
// задуманной бесплатной; un-budgeted local→cloud обязан быть виден (D4.3).
|
||
c.log.InfoContext(ctx, "local llm returned empty content; retrying on cloud (paid)")
|
||
return c.fallback.Complete(ctx, req)
|
||
}
|
||
if ctx.Err() != nil {
|
||
// The WHOLE request budget is gone, not just the local leg — a cloud
|
||
// call would die the same way; don't burn a paid attempt or trip the
|
||
// breaker over our own cancellation.
|
||
return nil, err
|
||
}
|
||
var se *HTTPStatusError
|
||
if errors.As(err, &se) && se.Status >= 400 && se.Status < 500 && se.Status != http.StatusTooManyRequests {
|
||
return nil, err // terminal 4xx: local config error — fail loud, never mask with the cloud
|
||
}
|
||
|
||
c.trip()
|
||
// WARN с причиной: сам trip() логирует переход БЕЗ err (он под мьютексом и не
|
||
// знает запроса), а на дефолтном INFO оператор иначе видел бы «tripped offline»
|
||
// с выброшенной причиной — первый вопрос дебага канала B (пакет №4).
|
||
c.log.WarnContext(ctx, "local llm failed; falling back to cloud", "err", err)
|
||
resp, ferr := c.fallback.Complete(ctx, req)
|
||
if ferr != nil {
|
||
c.log.WarnContext(ctx, "local llm and cloud fallback both failed", "local_err", err, "cloud_err", ferr)
|
||
return nil, ferr
|
||
}
|
||
return resp, nil
|
||
}
|
||
|
||
func (c *failoverClient) isHealthy() bool {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
return c.healthy
|
||
}
|
||
|
||
// trip opens the breaker after an in-request local failure. Recovery is
|
||
// probe-driven only.
|
||
func (c *failoverClient) trip() {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
if c.healthy {
|
||
c.log.Info("local llm tripped offline by a request failure; serving from cloud until probes recover")
|
||
}
|
||
c.healthy = false
|
||
c.consecOK = 0
|
||
c.needOK = tripProbesToClose
|
||
}
|
||
|
||
// noteProbe folds one probe result into the breaker. A failed probe opens it
|
||
// but does not raise needOK: plain down/up recovers on the next success, only
|
||
// a request-path trip demands the longer streak.
|
||
func (c *failoverClient) noteProbe(ok bool) {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
if !ok {
|
||
if c.healthy {
|
||
c.log.Info("local llm probe failed; serving from cloud")
|
||
}
|
||
c.healthy = false
|
||
c.consecOK = 0
|
||
c.consecFail++
|
||
if !c.everHealthy && !c.warnedDown && c.consecFail >= neverOnlineWarnAfter {
|
||
c.warnedDown = true
|
||
c.log.Warn("local llm has never come online since boot; ALL traffic is served by the paid cloud — check the local base URL", "probe_url", c.probeURL)
|
||
}
|
||
return
|
||
}
|
||
c.consecFail = 0
|
||
c.consecOK++
|
||
if !c.healthy && c.consecOK >= c.needOK {
|
||
c.healthy = true
|
||
c.everHealthy = true
|
||
c.needOK = 1
|
||
c.log.Info("local llm online; serving from local backend")
|
||
}
|
||
}
|
||
|
||
func (c *failoverClient) probeLoop(ctx context.Context) {
|
||
t := time.NewTicker(c.interval)
|
||
defer t.Stop()
|
||
for {
|
||
c.noteProbe(c.probe(ctx)) // first probe runs immediately, not an interval late
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-t.C:
|
||
}
|
||
}
|
||
}
|
||
|
||
// probe GETs the OpenAI models listing on the local base URL — cheap (no
|
||
// inference) and it proves the whole path end to end. It does NOT prove the
|
||
// model is resident in VRAM; keeping it warm is OLLAMA_KEEP_ALIVE=-1's job.
|
||
func (c *failoverClient) probe(ctx context.Context) bool {
|
||
pctx, cancel := context.WithTimeout(ctx, localProbeTimeout)
|
||
defer cancel()
|
||
req, err := http.NewRequestWithContext(pctx, http.MethodGet, c.probeURL, nil)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
if c.probeKey != "" {
|
||
req.Header.Set("Authorization", "Bearer "+c.probeKey)
|
||
}
|
||
resp, err := c.httpc.Do(req)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
defer resp.Body.Close()
|
||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) // drain for connection reuse
|
||
return resp.StatusCode >= 200 && resp.StatusCode < 300
|
||
}
|