package llm import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "log/slog" "math/rand" "net/http" "strconv" "strings" "time" "golang.org/x/net/http2" "textmachine/backend/internal/obs" ) // httpllm.go is the shared OpenAI-compatible Chat Completions transport: one // HTTP+retry implementation reused by every OpenAI-compatible adapter // (DeepSeek, GLM, Kimi, xAI, ru-aggregators, llama-server). A port of vojo's // httpllm.go with Phase-0 edits (03-implementation-notes §2 item 2): // // - all timeouts/retries are client PARAMETERS from the models.yaml profile, // not constants: the donor's hardcoded 60-second per-attempt deadline would // have killed real 63–278 s chunks and retried them three times for free; // - Retry-After on a 429 is read and honoured (the donor capped backoff at 8 s // — a hammer against a bulk chapter run's rate-limits), but is itself capped // by maxRetryAfterWait — otherwise a hostile «Retry-After: 7200» hangs the // stage for hours with a held reservation; // - the response body is read through a LimitReader (the donor capped only in // the gemini-native adapter). // // A documented departure from the donor: the one-shot self-heal of the // reasoning_effort parameter (a model returning 400 was remembered and had the // parameter stripped) is NOT ported — we set reasoning per-stage in config with // fail-fast validation, and a provider's 400 must be fixed in config, not swallowed. // RetryProfile bounds one client's retry loop. Zero fields fall back to // defaults; the profile comes from models.yaml per provider (per-role // overrides — Phase 1). type RetryProfile struct { AttemptTimeout time.Duration // deadline for ONE HTTP attempt MaxAttempts int BackoffBase time.Duration // first backoff; doubles per attempt BackoffCap time.Duration } func (p RetryProfile) withDefaults() RetryProfile { if p.AttemptTimeout <= 0 { // The eval measured 63–278 s per chunk on a local model; cloud is faster, // but the default must survive a long chunk. Provider specifics live in // models.yaml; this is only a safety ceiling. p.AttemptTimeout = 300 * time.Second } if p.MaxAttempts <= 0 { p.MaxAttempts = 3 } if p.BackoffBase <= 0 { p.BackoffBase = 500 * time.Millisecond } if p.BackoffCap <= 0 { p.BackoffCap = 30 * time.Second } return p } // h2ReadIdleTimeout / h2PingTimeout tune the cloud transport's HTTP/2 keepalive // (research/21 §1.5, modelled on the official grok-build's 15s/5s): on a long // thinking-heavy editor call NO body byte flows until the whole JSON is ready, so a // proxy/LB can idle-close the socket MID-generation. An HTTP/2 PING keeps the // connection alive independently of DATA frames; a dead connection then surfaces as // a clean retryable transport error instead of a mid-generation RST that re-bills the // call. These are transport keepalive frames only — they never touch the request // body, so the wire and the job snapshot are unchanged (pack invariant: zero wire bytes). const ( h2ReadIdleTimeout = 15 * time.Second h2PingTimeout = 5 * time.Second ) // keepAliveHTTPClient builds the cloud transport with HTTP/2 PING keepalive. It // clones http.DefaultTransport (proxy-from-env, dial/TLS timeouts, ForceAttemptHTTP2) // and layers tuned h2 ping settings via http2.ConfigureTransports — so an HTTPS // provider gets keepalive while a plaintext test server (httptest, no ALPN) still // speaks HTTP/1.1 over the same base transport, unchanged. On the (unexpected) // ConfigureTransports error the plain cloned transport is used as-is: keepalive is a // reliability bonus, never a hard dependency. The local provider passes its OWN // no-proxy client (httpc != nil), so it is untouched — localhost needs no h2 keepalive. func keepAliveHTTPClient() *http.Client { base := http.DefaultTransport.(*http.Transport).Clone() if h2, err := http2.ConfigureTransports(base); err == nil && h2 != nil { h2.ReadIdleTimeout = h2ReadIdleTimeout h2.PingTimeout = h2PingTimeout } return &http.Client{Transport: base} } // maxResponseBytes caps one completion body read. A translated chapter chunk // is ~10–50 KiB; 16 MiB leaves two orders of magnitude of headroom while // keeping a misbehaving endpoint from exhausting memory. const maxResponseBytes = 16 << 20 // maxRetryAfterWait bounds an honoured Retry-After: the provider's hint wins // over our backoff schedule, but the HINT is capped to this — a stage holding a // USD reservation must stay interruptible-by-timeout, not wedged for hours. The // realized sleep may exceed this by the de-sync jitter (≤+25%, nextBackoff): jitter // is added AFTER the cap ON PURPOSE, else N parallel waves that all see the same // Retry-After≥5min would collapse to the identical instant — the thundering herd the // jitter exists to break. The wait stays ctx-interruptible either way. const maxRetryAfterWait = 5 * time.Minute // retryLoop is the ONE retry engine shared by every transport (OpenAI-compat // and the native Anthropic adapter): exponential backoff with cap, Retry-After // override, jitter, ctx-done select. A single policy instance — so a retry // fix doesn't "drift" to one provider while forgetting another. // // Logs are a product for the operator (package №4, pains cleared by a smoke run): // each attempt's start is visible at DEBUG (otherwise "hung on a timeout" is // indistinguishable from "died"), WARN carries the ACTUAL wait before the next // attempt (Retry-After can silently stretch it to 5 min against the expected // backoff_cap) and says "will retry" ONLY when an attempt actually remains — // previously the last failure too was logged "will retry" before the exhausted error. func retryLoop[T any](ctx context.Context, profile RetryProfile, name string, log *slog.Logger, attempt func() (T, bool, error)) (T, error) { var zero T var lastErr error for att := 0; att < profile.MaxAttempts; att++ { if log != nil { log.DebugContext(ctx, name+" attempt start", "attempt", att+1, "max", profile.MaxAttempts, "attempt_timeout", profile.AttemptTimeout.String()) } resp, retryable, err := attempt() if err == nil { return resp, nil } lastErr = err if ctx.Err() != nil { return zero, ctx.Err() } if !retryable { return zero, err } if att+1 >= profile.MaxAttempts { break // attempts exhausted — no retry remains, no sleep } backoff := nextBackoff(profile, att, lastErr) if log != nil { log.WarnContext(ctx, name+" attempt failed, will retry", "attempt", att+1, "max", profile.MaxAttempts, "retry_in", backoff.Round(time.Millisecond).String(), "err", err) } select { case <-ctx.Done(): return zero, ctx.Err() case <-time.After(backoff): } } return zero, fmt.Errorf("%s: exhausted %d attempts: %w", name, profile.MaxAttempts, lastErr) } // nextBackoff computes the wait before attempt att+1 (0-based att just failed): // exponential base< 20 { shift = 20 // guard the shift against overflow at large max_attempts } backoff := profile.BackoffBase << uint(shift) if backoff <= 0 || backoff > profile.BackoffCap { backoff = profile.BackoffCap } if ra := retryAfterOf(lastErr); ra > 0 { if ra > maxRetryAfterWait { ra = maxRetryAfterWait } // Honour the server's hint as a FLOOR and de-sync parallel waves with // POSITIVE-only proportional jitter (up to +25%): N calls that hit the same // 429 must not all retry at the same instant (thundering herd), but we never // retry BEFORE the provider said to. return ra + proportionalJitter(ra, false) } // Exponential backoff: symmetric ±25% proportional jitter (research/21 §1.18а) // de-syncs parallel waves far better than the old fixed 0–250ms window, which // barely moved a 30s backoff. return backoff + proportionalJitter(backoff, true) } // proportionalJitter returns a jitter offset for a backoff of duration d. symmetric // spreads it over [−25%, +25%] (mean-preserving, for exponential backoff); otherwise // over [0, +25%] (for an honoured Retry-After, which must never be shortened). d ≤ 0 // yields 0. func proportionalJitter(d time.Duration, symmetric bool) time.Duration { spread := int64(d) / 4 // 25% if spread <= 0 { return 0 } if symmetric { return time.Duration(rand.Int63n(2*spread+1) - spread) } return time.Duration(rand.Int63n(spread + 1)) } // openAIClient performs OpenAI-compatible /chat/completions calls with retry. type openAIClient struct { name string // provider label for logs/errors ("deepseek", "glm", "local") base string key string http *http.Client profile RetryProfile headers map[string]string // extra static headers (provider-specific), may be nil log *slog.Logger } // newOpenAIClient builds the shared transport. httpc may be nil (default // client); the local provider passes an explicit no-proxy client (the stand's // proxy gotcha: env-proxy intercepts non-loopback local addresses like 172.x). func newOpenAIClient(name, base, key string, profile RetryProfile, headers map[string]string, httpc *http.Client, logger *slog.Logger) *openAIClient { if httpc == nil { httpc = keepAliveHTTPClient() } return &openAIClient{ name: name, base: base, key: key, http: httpc, profile: profile.withDefaults(), headers: headers, log: logger, } } // --- OpenAI-compatible wire types ------------------------------------------------- type openAIMessage struct { Role string `json:"role"` Content string `json:"content"` } // openAIRequest is the neutral pre-wire request. Its body is assembled by // MarshalJSON THROUGH the per-model Capability resolver (capability.go), not // from fixed json tags: OpenAI-compatible providers share an endpoint but not a // body — the budget key, whether temperature is emitted, and the reasoning // switch all vary per model (D3.1). The Phase-0 fixed body (max_tokens + // always-send temperature + reasoning_effort) 400s on Kimi / gpt-5 / Gemini; // the resolver renders the correct shape and is itself folded into the job // snapshot, so changing it invalidates checkpoints rather than false-hitting. type openAIRequest struct { model string messages []openAIMessage maxTokens int temperature float64 stream bool // reasoningEffort is the NEUTRAL effort ("" | off | low | medium | high); // cap decides how — and whether — it reaches the wire. reasoningEffort string responseFormat any cap Capability // extra carries per-model provider-specific fields from models.yaml. Merged // LAST and only into gaps — a resolved standard key always wins. extra map[string]any } // MarshalJSON assembles the wire body via the capability resolver. Resolved // standard keys are authoritative; config-supplied extra_body fills only gaps. func (r openAIRequest) MarshalJSON() ([]byte, error) { m := map[string]any{ "model": r.model, "messages": r.messages, "stream": r.stream, } r.cap.applyToBody(m, r.maxTokens, r.temperature, r.reasoningEffort) if r.responseFormat != nil { m["response_format"] = r.responseFormat } mergeBody(m, r.extra) return json.Marshal(m) } type openAIUsage struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` // TotalTokens is prompt+completion for spec providers, but for Gemini via the // OpenAI-compat layer it ALSO carries the mandatory thinking tokens, which do NOT // appear in completion_tokens and have no reasoning_tokens field (live-probe // 2026-07-10: completion=2, prompt=22, total=847 → 823 hidden thinking). The // ReasoningAdditiveTotal adapter derives thinking = total − prompt − completion. TotalTokens int `json:"total_tokens"` PromptTokensDetails struct { CachedTokens int `json:"cached_tokens"` } `json:"prompt_tokens_details"` // xAI reports reasoning tokens here SEPARATELY from completion_tokens and // bills them at the output rate; OpenAI-spec providers count them inside // completion_tokens. The adapter decides which semantics apply. CompletionTokensDetails struct { ReasoningTokens int `json:"reasoning_tokens"` } `json:"completion_tokens_details"` // DeepSeek historically reports cache usage in its own top-level fields // instead of prompt_tokens_details. Parse both so the Phase-0 cache // experiment can't show a false zero (03-implementation-notes §3.5). PromptCacheHitTokens int `json:"prompt_cache_hit_tokens"` PromptCacheMissTokens int `json:"prompt_cache_miss_tokens"` } // cacheRead returns the cache-hit token count whichever field the provider // used. func (u openAIUsage) cacheRead() int { if u.PromptTokensDetails.CachedTokens > 0 { return u.PromptTokensDetails.CachedTokens } return u.PromptCacheHitTokens } type openAIResponse struct { ID string `json:"id"` Model string `json:"model"` Choices []struct { Message struct { Content string `json:"content"` } `json:"message"` FinishReason string `json:"finish_reason"` } `json:"choices"` Usage openAIUsage `json:"usage"` } func (r *openAIResponse) Text() string { if len(r.Choices) == 0 { return "" } return r.Choices[0].Message.Content } func (r *openAIResponse) FinishReason() string { if len(r.Choices) == 0 { return "" } // OpenAI-compat wire values map 1:1 onto the neutral constants. return r.Choices[0].FinishReason } // complete calls Chat Completions with retry on transient failures (429 / 5xx / // network, exponential backoff + jitter, Retry-After honoured). Non-retryable // 4xx fail immediately. On exhaustion the caller releases the reservation, so // a transient failure is never silently swallowed. func (c *openAIClient) complete(ctx context.Context, reqBody openAIRequest) (*openAIResponse, error) { payload, err := json.Marshal(reqBody) if err != nil { return nil, err } billedDecodeSeen := 0 return retryLoop(ctx, c.profile, c.name, c.log, func() (*openAIResponse, bool, error) { resp, retryable, err := c.attempt(ctx, payload) // Cap billed-decode re-bills at ONE (research/21 §1.18в): an undecodable 2xx // has ALREADY billed, so re-running it under the full MaxAttempts turns a // provider emitting 2xx garbage into a paid retry STORM (grok-build's // "laundering a serialization error into retryable gave a full-budget storm"). // The first occurrence retries once (a transient proxy break is worth one shot); // a second billed-decode is terminal, so the runner settles at the estimate. if err != nil && retryable { var bde *BilledDecodeError if errors.As(err, &bde) { billedDecodeSeen++ if billedDecodeSeen > 1 { return resp, false, err } } } return resp, retryable, err }) } // attempt performs one HTTP call. Returns retryable=true for 429/5xx and // network errors, false for other non-2xx (terminal 4xx). The per-attempt // deadline bounds a single hung connection; the overall per-request deadline // (set by the caller via ctx) bounds the whole retry loop. func (c *openAIClient) attempt(ctx context.Context, payload []byte) (*openAIResponse, bool, error) { attemptCtx, cancel := context.WithTimeout(ctx, c.profile.AttemptTimeout) defer cancel() req, err := http.NewRequestWithContext(attemptCtx, http.MethodPost, c.base+"/chat/completions", bytes.NewReader(payload)) if err != nil { return nil, false, err } req.Header.Set("Content-Type", "application/json") // A local backend (ollama / llama-server) usually runs without auth; an // empty key means "no Authorization header", not "Bearer " with an empty token. if c.key != "" { req.Header.Set("Authorization", "Bearer "+c.key) } for k, v := range c.headers { req.Header.Set(k, v) } resp, err := c.http.Do(req) if err != nil { // Network error / timeout — retryable (unless the parent ctx is done). A // per-attempt deadline is annotated with the configured timeout: the bare // «context deadline exceeded» doesn't tell the operator WHOSE deadline it is — // the attempt timeout (cured by timeouts.attempt_s) or the whole run cancelled. if errors.Is(attemptCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil { err = fmt.Errorf("attempt timed out after %s (timeouts.attempt_s): %w", c.profile.AttemptTimeout, err) } return nil, ctx.Err() == nil, err } defer resp.Body.Close() // Read one byte past the limit to DISTINGUISH truncation from a whole body. data, _ := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) truncated := len(data) > maxResponseBytes if truncated { data = data[:maxResponseBytes] } obs.LogLLMExchange(ctx, c.log, c.name, payload, resp.StatusCode, data) if resp.StatusCode < 200 || resp.StatusCode >= 300 { retryable := retryableStatus(resp.StatusCode, data) e := &HTTPStatusError{Provider: c.name, Status: resp.StatusCode, Body: snippet(data)} if retryable { e.RetryAfter = parseRetryAfter(resp.Header) // only a retryable status will honour it } return nil, retryable, e } var out openAIResponse if err := json.Unmarshal(data, &out); err != nil { // A 2xx with an unreadable body: the provider has ALREADY charged. We type it // so the runner conservatively settles at the estimate. A truncated (>16 MiB) // body is deterministic — retrying is pointless (each retry is a new billed // 2xx), so terminal; a broken connection / garbage is retryable. if truncated { return nil, false, &BilledDecodeError{Provider: c.name, Err: fmt.Errorf("response exceeded %d bytes (truncated, not retried): %w", maxResponseBytes, err)} } return nil, true, &BilledDecodeError{Provider: c.name, Err: err} } // A 2xx is a billed call even when the model returns empty content // (content filter, finish_reason=length with no text). Return it as a // success so the caller settles the real cost via the ledger instead of // releasing the reservation and losing the spend — which would let empty // replies bypass the book/day ceilings. Gates deal with the empty text. return &out, false, nil } // retryableStatus is the ONE source of the retry/terminal split for a non-2xx // response — pinned by a drift-guard test (research/21 §1.18б) so a new status // class can't silently change retry policy. Only 429 and 5xx are transient; every // other non-2xx is a terminal config/request error that must fail loud. The one // nuance is a CREDITS-EXHAUSTED 429: a body that marks the account/quota as spent // is terminal, not transient — retrying under a held USD reservation only burns // wall-clock (opencode/goose both classify quota/credits as fatal). A 402 // (Payment Required) is already terminal by falling through to the default. func retryableStatus(status int, body []byte) bool { switch { case status == http.StatusTooManyRequests: return !isQuotaExhausted(body) // plain rate-limit → retry; credits gone → terminal case status >= 500: return true default: return false // terminal 4xx (incl. 402 Payment Required) } } // isQuotaExhausted reports whether a 429 body marks the account's credits/quota as // spent — a NARROW, high-confidence marker set (research/21 §1.4), deliberately NOT // a broad regex battery (the aider-style «exact matchers dead» brittleness this pack // explicitly avoids). `insufficient_quota` is the OpenAI-family error code; // `quota_exceeded` covers the others. Retrying either never succeeds — the operator // must top up — so the wave fails loud in one attempt instead of 3 + honoured-backoff. func isQuotaExhausted(body []byte) bool { b := strings.ToLower(string(body)) return strings.Contains(b, "insufficient_quota") || strings.Contains(b, "quota_exceeded") } // HTTPStatusError is a non-2xx completion failure, carrying the status code as // a typed field so callers (the failover decorator) can classify it // structurally — a terminal 4xx is a config/request error that must fail loud, // not be masked by a fallback — instead of parsing the message. type HTTPStatusError struct { Provider string Status int Body string RetryAfter time.Duration // from the Retry-After header; 0 = absent } func (e *HTTPStatusError) Error() string { return fmt.Sprintf("%s http %d: %s", e.Provider, e.Status, e.Body) } // BilledDecodeError is a 2xx whose body could not be decoded: the provider has // billed the call, but usage/text are unknown. The runner treats an exhausted // retry chain ending in this error as BILLED (settle at the reservation // estimate) rather than releasing the reservation. type BilledDecodeError struct { Provider string Err error } func (e *BilledDecodeError) Error() string { return fmt.Sprintf("%s: 2xx body decode failed (call IS billed): %v", e.Provider, e.Err) } func (e *BilledDecodeError) Unwrap() error { return e.Err } func retryAfterOf(err error) time.Duration { var se *HTTPStatusError if errors.As(err, &se) { return se.RetryAfter } return 0 } // toOpenAIMessages maps neutral messages onto the wire. CacheBoundary is // meaningless here: OpenAI-compatible caches (DeepSeek: prefix-match from token // 0 in blocks of 64 tokens) are automatic; the stable-prefix ORDER the // assembler produced is all that matters — which is also why JOINING the system // run under SystemMessagesSingle costs no cache: the stable prefix keeps its // bytes and its position, it merely stops being its own message. // // mode is the endpoint's system-message cardinality (Capability.SystemMessages). // Under SystemMessagesSingle the LEADING system run is joined into one message so // an endpoint with one system slot receives the memory-bank injection instead of // silently dropping it; a system message AFTER a non-system turn is refused loud, // exactly as the Anthropic adapter refuses it — our assembler never produces one, // and a join that quietly re-ordered a conversation would be a worse lie than the // one this function exists to stop. Under SystemMessagesMulti (the default) the // mapping is one-for-one, byte-identical to the wire before this axis existed. func toOpenAIMessages(msgs []Message, mode SystemMessagesMode) ([]openAIMessage, error) { if mode != SystemMessagesSingle { out := make([]openAIMessage, len(msgs)) for i, m := range msgs { out[i] = openAIMessage{Role: m.Role, Content: m.Content} } return out, nil } var systemRun []string out := make([]openAIMessage, 0, len(msgs)) inSystemPrefix := true for _, m := range msgs { if m.Role == "system" { if !inSystemPrefix { return nil, fmt.Errorf("llm: this endpoint carries a single system message and got one after a non-system turn; joining it would re-order the conversation") } systemRun = append(systemRun, m.Content) continue } if inSystemPrefix { inSystemPrefix = false if len(systemRun) > 0 { out = append(out, openAIMessage{Role: "system", Content: strings.Join(systemRun, systemJoinSeparator)}) } } out = append(out, openAIMessage{Role: m.Role, Content: m.Content}) } // A message list that is system-only (no user turn) still has to ship its join. if inSystemPrefix && len(systemRun) > 0 { out = append(out, openAIMessage{Role: "system", Content: strings.Join(systemRun, systemJoinSeparator)}) } return out, nil } // jsonResponseFormat returns the response_format value for JSONOnly requests // (nil otherwise, so the field serializes away). func jsonResponseFormat(jsonOnly bool) any { if jsonOnly { return map[string]string{"type": "json_object"} } return nil } // parseRetryAfter reads a provider's honoured backoff hint from the response // headers (research/21 §1.13). It accepts, in priority order: // - Retry-After-Ms (millisecond precision; opencode/openai-go/anthropic-sdk-go // all read it FIRST — some providers only send this); // - Retry-After as FRACTIONAL or integer seconds (the old strconv.Atoi rejected // "1.5"/"0.5" → 0 → blind exponential backoff, ignoring a sub-second hint); // - Retry-After as an HTTP-date. // // The value is still bounded by maxRetryAfterWait in nextBackoff — this only widens // what we can PARSE, never how long we will actually wait. func parseRetryAfter(h http.Header) time.Duration { if ms := strings.TrimSpace(h.Get("Retry-After-Ms")); ms != "" { if n, err := strconv.ParseFloat(ms, 64); err == nil && n > 0 { return time.Duration(n * float64(time.Millisecond)) } } v := strings.TrimSpace(h.Get("Retry-After")) if v == "" { return 0 } // Fractional or integer seconds. ParseFloat("120") is 120, so this also covers // the integer form the old Atoi handled. if secs, err := strconv.ParseFloat(v, 64); err == nil && secs > 0 { return time.Duration(secs * float64(time.Second)) } if t, err := http.ParseTime(v); err == nil { if d := time.Until(t); d > 0 { return d } } return 0 } func snippet(b []byte) string { const max = 300 if len(b) > max { return string(b[:max]) + "…" } return string(b) }