90 lines
3.7 KiB
Go
90 lines
3.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
)
|
|
|
|
// provider_local.go is the thin adapter for the self-hosted inference backend (the
|
|
// home ollama/llama-server behind the SSH tunnel — docs/plans/local_llm_backend.md).
|
|
// Both speak the OpenAI Chat Completions wire format, so like xAI this is a shell
|
|
// over the shared openAIClient transport (httpllm.go). It differs from the cloud
|
|
// adapters in three deliberate ways:
|
|
//
|
|
// - The MODEL is the adapter's own (LOCAL_LLM_MODEL), never the cascade's: the
|
|
// cascade asks for grok-* names that don't exist locally. The response's Model
|
|
// reports what actually served, so billing prices it at the local $0 entry.
|
|
// - The TEMPERATURE is the adapter's own (LOCAL_LLM_TEMP, Qwen3's non-thinking 0.7):
|
|
// ollama honours a request temperature over the Modelfile, so passing through the
|
|
// cascade's XAI_TEMPERATURE (0.6) would silently mistune the local model. The
|
|
// remaining Qwen3 samplers (top_k/min_p/repeat_penalty) can't cross ollama's /v1
|
|
// layer at all and are baked into the Modelfile at home.
|
|
// - MAX TOKENS has a local override (LOCAL_LLM_MAX_OUTPUT_TOKENS): unlike xAI —
|
|
// where thinking bills on top of max_tokens — ollama's cap covers thinking AND
|
|
// answer together, so the cloud-sized MAX_OUTPUT_TOKENS would let a thinking
|
|
// route (reasoning_effort low/high) burn the whole budget on reasoning and
|
|
// return empty content. Local tokens are free; the override defaults roomier.
|
|
//
|
|
// ReasoningEffort passes through unchanged: ollama's /v1 maps it onto Qwen3 thinking
|
|
// ("none" = off — the casual grok_direct route; "low"/"high" = on — verified live),
|
|
// so the cascade's existing per-route efforts drive local thinking with no new knob.
|
|
// ConvID (the x-grok-conv-id cache-routing header) and Tools are xAI concepts with no
|
|
// local equivalent and are deliberately not forwarded.
|
|
type localClient struct {
|
|
http *openAIClient
|
|
model string
|
|
temp float64
|
|
maxTok int // 0 = inherit the request's MaxTokens
|
|
}
|
|
|
|
// NewLocalLLMClient builds the local-backend adapter. Returns the neutral LLMClient
|
|
// so the bot holds no vendor type. The API key is optional (a bare local endpoint
|
|
// sends no Authorization header — see openAIClient.attempt).
|
|
func NewLocalLLMClient(cfg *Config, logger *slog.Logger) LLMClient {
|
|
return &localClient{
|
|
http: newOpenAIClient("local", cfg.LocalLLMBaseURL, cfg.LocalLLMAPIKey, nil, logger),
|
|
model: cfg.LocalLLMModel,
|
|
temp: cfg.LocalLLMTemp,
|
|
maxTok: cfg.LocalLLMMaxTok,
|
|
}
|
|
}
|
|
|
|
func (c *localClient) Complete(ctx context.Context, req LLMRequest) (*LLMResponse, error) {
|
|
msgs := make([]openAIMessage, len(req.Messages))
|
|
for i, m := range req.Messages {
|
|
msgs[i] = openAIMessage{Role: m.Role, Content: m.Content}
|
|
}
|
|
maxTok := req.MaxTokens
|
|
if c.maxTok > 0 {
|
|
maxTok = c.maxTok
|
|
}
|
|
var respFormat any
|
|
if req.JSONOnly {
|
|
respFormat = map[string]string{"type": "json_object"}
|
|
}
|
|
resp, err := c.http.complete(ctx, openAIRequest{
|
|
Model: c.model,
|
|
Messages: msgs,
|
|
MaxTokens: maxTok,
|
|
Temperature: c.temp,
|
|
Stream: false,
|
|
ReasoningEffort: req.ReasoningEffort,
|
|
ResponseFormat: respFormat,
|
|
}, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &LLMResponse{
|
|
Text: resp.Text(),
|
|
Usage: Usage{
|
|
PromptTokens: resp.Usage.PromptTokens,
|
|
CachedTokens: resp.Usage.PromptTokensDetails.CachedTokens,
|
|
CompletionTokens: resp.Usage.CompletionTokens,
|
|
// ReasoningTokens deliberately left 0: ollama counts thinking INSIDE
|
|
// completion_tokens (subset semantics — see llm.go), and the details
|
|
// field is absent anyway. It's all $0 regardless.
|
|
},
|
|
Model: c.model, // the local model answered, whatever the cascade asked for
|
|
ProviderRequestID: resp.ID,
|
|
}, nil
|
|
}
|