114 lines
4.5 KiB
Go
114 lines
4.5 KiB
Go
package llm
|
||
|
||
import (
|
||
"context"
|
||
"log/slog"
|
||
"net/http"
|
||
)
|
||
|
||
// provider_local.go is the adapter for a self-hosted OpenAI-compatible
|
||
// inference backend (ollama / llama-server on the owner's GTX 1070). Порт
|
||
// vojo provider_local.go; главное отличие — явный no-proxy транспорт.
|
||
//
|
||
// Прокси-грабли стенда (memory textmachine-local-stand + валидация): в env
|
||
// прописан webshare-прокси, NO_PROXY=<local> — виндовая нотация, которую Go
|
||
// не понимает. Go stdlib сам НЕ проксирует loopback (127.0.0.1/localhost), но
|
||
// llama-server на стенде слушает WSL-gateway-адреса вида 172.18.0.1 — такой
|
||
// запрос ушёл бы на прокси и получил 403, нога failover навсегда осталась бы
|
||
// unhealthy, молча оплачивая облако. Поэтому транспорт локального провайдера
|
||
// (и его health-пробы в failover.go) строится с Proxy: nil безусловно.
|
||
|
||
// LocalConfig configures the local backend adapter.
|
||
type LocalConfig struct {
|
||
BaseURL string // e.g. http://127.0.0.1:11434/v1 or http://172.18.0.1:11434/v1
|
||
APIKey string // usually empty — no Authorization header then
|
||
// Model is the adapter's own tag (e.g. huihui_ai/qwen3-abliterated:8b):
|
||
// the pipeline asks for role-model names that don't exist locally. The
|
||
// response's Model reports what actually served, so billing prices it at
|
||
// the local $0 entry.
|
||
Model string
|
||
// Temperature overrides the request's (ollama honours a request temperature
|
||
// over the Modelfile; passing the cloud role's value through would mistune
|
||
// the local model). 0 = keep the request's.
|
||
Temperature float64
|
||
// MaxTokens overrides the request's: ollama's cap covers thinking AND
|
||
// answer together (unlike xAI, where thinking bills on top), so a
|
||
// cloud-sized budget lets a thinking model burn it all on reasoning and
|
||
// return empty content. 0 = inherit the request's.
|
||
MaxTokens int
|
||
Profile RetryProfile
|
||
// Cap is the resolved wire shape (capability.go). If its reasoning control
|
||
// is unset, NewLocalClient defaults it to the ollama off-switch (map "off"
|
||
// onto reasoning_effort:"none").
|
||
Cap Capability
|
||
}
|
||
|
||
// NoProxyClient is an http.Client that bypasses any environment proxy.
|
||
// Exported so the failover prober uses the same transport discipline.
|
||
func NoProxyClient() *http.Client {
|
||
return &http.Client{Transport: &http.Transport{Proxy: nil}}
|
||
}
|
||
|
||
type localClient struct {
|
||
http *openAIClient
|
||
model string
|
||
temp float64
|
||
maxTok int
|
||
cap Capability
|
||
}
|
||
|
||
// NewLocalClient builds the local-backend adapter.
|
||
func NewLocalClient(cfg LocalConfig, logger *slog.Logger) LLMClient {
|
||
cap := cfg.Cap
|
||
if cap.Reasoning.Control == "" {
|
||
// ollama's /v1 maps reasoning_effort onto Qwen3 thinking; "none" is its
|
||
// explicit off-switch (verified in vojo). Default it here so a zero-Cap
|
||
// caller keeps the off→"none" behavior; BuildClient passes the same via
|
||
// the resolver, so the snapshot's stageSnap capability matches the wire.
|
||
cap.Reasoning = ReasoningCap{Control: ReasoningEffortField, OffEffort: "none"}
|
||
}
|
||
return &localClient{
|
||
http: newOpenAIClient("local", cfg.BaseURL, cfg.APIKey, cfg.Profile, nil, NoProxyClient(), logger),
|
||
model: cfg.Model,
|
||
temp: cfg.Temperature,
|
||
maxTok: cfg.MaxTokens,
|
||
cap: cap,
|
||
}
|
||
}
|
||
|
||
func (c *localClient) Complete(ctx context.Context, req LLMRequest) (*LLMResponse, error) {
|
||
maxTok := req.MaxTokens
|
||
if c.maxTok > 0 {
|
||
maxTok = c.maxTok
|
||
}
|
||
temp := req.Temperature
|
||
if c.temp > 0 {
|
||
temp = c.temp
|
||
}
|
||
resp, err := c.http.complete(ctx, openAIRequest{
|
||
model: c.model,
|
||
messages: toOpenAIMessages(req.Messages),
|
||
maxTokens: maxTok,
|
||
temperature: temp,
|
||
stream: false,
|
||
reasoningEffort: req.ReasoningEffort, // neutral; cap maps off→"none" for ollama
|
||
responseFormat: jsonResponseFormat(req.JSONOnly),
|
||
cap: c.cap,
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return &LLMResponse{
|
||
Text: resp.Text(),
|
||
Usage: Usage{
|
||
PromptTokens: resp.Usage.PromptTokens,
|
||
CachedTokens: resp.Usage.cacheRead(),
|
||
CompletionTokens: resp.Usage.CompletionTokens,
|
||
// ReasoningTokens deliberately 0: ollama counts thinking INSIDE
|
||
// completion_tokens (subset semantics). It's all $0 regardless.
|
||
},
|
||
Model: c.model, // the local model answered, whatever the pipeline asked for
|
||
FinishReason: resp.FinishReason(),
|
||
ProviderRequestID: resp.ID,
|
||
}, nil
|
||
}
|