package llm import ( "context" "errors" "io" "log/slog" "net/http" "strings" "sync" "time" ) // failover.go is the local-first routing decorator over the LLMClient seam: // prefer the (free) home backend while it is reachable, fall back to the cloud // transparently when it is not. Порт vojo failover.go; конструктор принимает // явные параметры вместо толстого *Config, пробер использует no-proxy // транспорт (та же прокси-ловушка, что у provider_local). // // Channel-isolation caveat (03-implementation-notes §3.5): the fallback leg is // wired by the CALLER per role — for NSFW channel B (Фаза 2) the fallback must // itself be a permissive provider or absent (fail loud), otherwise an empty // answer from the local abliterated model would silently ship the flagged // chunk to a restricted cloud provider. // // Health is decided by an ACTIVE background probe plus a request-path circuit // breaker, both feeding one healthy flag: // - unhealthy → requests go straight to the cloud; // - healthy → try local under its own deadline; on transport/5xx/timeout // trip the breaker and retry the SAME request on the cloud; // - recovery is probe-driven ONLY (anti-flapping): a tripped breaker needs // tripProbesToClose consecutive probe successes to close. // // A terminal 4xx from the local leg (except 429) does NOT fall back: that is a // config/request error (wrong model tag, malformed body) and the cloud would // only mask it. 429 (a busy single-slot GPU) falls back like a timeout. const ( // localProbeTimeout bounds one health probe: a bare GET /models is answered // from the server's process memory, no inference — slower means the path, // not the model, is struggling. localProbeTimeout = 3 * time.Second // tripProbesToClose is how many consecutive probe successes re-close a // breaker tripped by a REQUEST failure (probe can pass while requests fail: // a cold model being loaded, a half-alive path). tripProbesToClose = 2 // neverOnlineWarnAfter: consecutive probe failures — without the leg EVER // having come online since boot — that trigger a one-time WARN, so an // enabled-but-dead local leg can't silently serve 100% of traffic from the // paid cloud. neverOnlineWarnAfter = 3 ) // FailoverConfig wires the decorator. type FailoverConfig struct { LegTimeout time.Duration // per-request deadline for the local leg ProbeURL string // /models ProbeKey string // optional bearer for the probe (usually empty) ProbeInterval time.Duration // background probe period } type failoverClient struct { primary LLMClient // local backend (free, preferred) fallback LLMClient // cloud (paid) log *slog.Logger legTimeout time.Duration probeURL string probeKey string interval time.Duration httpc *http.Client // Breaker state. healthy starts false: until the first probe passes, // traffic is served by the cloud, so a boot with the home box offline never // eats the probe timeout on user requests. mu sync.Mutex healthy bool consecOK int needOK int consecFail int everHealthy bool warnedDown bool } // NewFailoverClient wraps primary (local) and fallback (cloud) into one // LLMClient and starts the background health probe, which lives until ctx is // cancelled (process lifetime). func NewFailoverClient(ctx context.Context, primary, fallback LLMClient, cfg FailoverConfig, logger *slog.Logger) LLMClient { if cfg.LegTimeout <= 0 { cfg.LegTimeout = 300 * time.Second // полигон: чанки 63–278 с; 45 с vojo непригодны } if cfg.ProbeInterval <= 0 { cfg.ProbeInterval = 30 * time.Second } c := &failoverClient{ primary: primary, fallback: fallback, log: logger, legTimeout: cfg.LegTimeout, probeURL: cfg.ProbeURL, probeKey: cfg.ProbeKey, interval: cfg.ProbeInterval, httpc: NoProxyClient(), needOK: 1, } logger.Info("local llm failover enabled; cloud serves until the first successful probe", "probe_url", c.probeURL) go c.probeLoop(ctx) return c } func (c *failoverClient) Complete(ctx context.Context, req LLMRequest) (*LLMResponse, error) { if !c.isHealthy() { c.log.DebugContext(ctx, "local llm unhealthy; serving from cloud") return c.fallback.Complete(ctx, req) } lctx, cancel := context.WithTimeout(ctx, c.legTimeout) resp, err := c.primary.Complete(lctx, req) cancel() if err == nil { if strings.TrimSpace(resp.Text) != "" { return resp, nil } // A 2xx with EMPTY content from the local leg (thinking ate the token // budget, a quirk of the abliterated model). The local call is free, so // retry the turn on the cloud. The backend is up: not a breaker trip. c.log.DebugContext(ctx, "local llm returned empty content; retrying on cloud") return c.fallback.Complete(ctx, req) } if ctx.Err() != nil { // The WHOLE request budget is gone, not just the local leg — a cloud // call would die the same way; don't burn a paid attempt or trip the // breaker over our own cancellation. return nil, err } var se *HTTPStatusError if errors.As(err, &se) && se.Status >= 400 && se.Status < 500 && se.Status != http.StatusTooManyRequests { return nil, err // terminal 4xx: local config error — fail loud, never mask with the cloud } c.trip() c.log.DebugContext(ctx, "local llm failed; falling back to cloud", "err", err) resp, ferr := c.fallback.Complete(ctx, req) if ferr != nil { c.log.WarnContext(ctx, "local llm and cloud fallback both failed", "local_err", err, "cloud_err", ferr) return nil, ferr } return resp, nil } func (c *failoverClient) isHealthy() bool { c.mu.Lock() defer c.mu.Unlock() return c.healthy } // trip opens the breaker after an in-request local failure. Recovery is // probe-driven only. func (c *failoverClient) trip() { c.mu.Lock() defer c.mu.Unlock() if c.healthy { c.log.Info("local llm tripped offline by a request failure; serving from cloud until probes recover") } c.healthy = false c.consecOK = 0 c.needOK = tripProbesToClose } // noteProbe folds one probe result into the breaker. A failed probe opens it // but does not raise needOK: plain down/up recovers on the next success, only // a request-path trip demands the longer streak. func (c *failoverClient) noteProbe(ok bool) { c.mu.Lock() defer c.mu.Unlock() if !ok { if c.healthy { c.log.Info("local llm probe failed; serving from cloud") } c.healthy = false c.consecOK = 0 c.consecFail++ if !c.everHealthy && !c.warnedDown && c.consecFail >= neverOnlineWarnAfter { c.warnedDown = true c.log.Warn("local llm has never come online since boot; ALL traffic is served by the paid cloud — check the local base URL", "probe_url", c.probeURL) } return } c.consecFail = 0 c.consecOK++ if !c.healthy && c.consecOK >= c.needOK { c.healthy = true c.everHealthy = true c.needOK = 1 c.log.Info("local llm online; serving from local backend") } } func (c *failoverClient) probeLoop(ctx context.Context) { t := time.NewTicker(c.interval) defer t.Stop() for { c.noteProbe(c.probe(ctx)) // first probe runs immediately, not an interval late select { case <-ctx.Done(): return case <-t.C: } } } // probe GETs the OpenAI models listing on the local base URL — cheap (no // inference) and it proves the whole path end to end. It does NOT prove the // model is resident in VRAM; keeping it warm is OLLAMA_KEEP_ALIVE=-1's job. func (c *failoverClient) probe(ctx context.Context) bool { pctx, cancel := context.WithTimeout(ctx, localProbeTimeout) defer cancel() req, err := http.NewRequestWithContext(pctx, http.MethodGet, c.probeURL, nil) if err != nil { return false } if c.probeKey != "" { req.Header.Set("Authorization", "Bearer "+c.probeKey) } resp, err := c.httpc.Do(req) if err != nil { return false } defer resp.Body.Close() _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) // drain for connection reuse return resp.StatusCode >= 200 && resp.StatusCode < 300 }