textmachine/backend/internal/llm/httpllm.go

package llm

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"math/rand"
	"net/http"
	"net/http/httptrace"
	"strconv"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/http2"

	"textmachine/backend/internal/obs"
)

// httpllm.go is the shared OpenAI-compatible Chat Completions transport: one
// HTTP+retry implementation reused by every OpenAI-compatible adapter
// (DeepSeek, GLM, Kimi, xAI, ru-aggregators, llama-server). A port of vojo's
// httpllm.go with Phase-0 edits (03-implementation-notes §2 item 2):
//
//   - all timeouts/retries are client PARAMETERS from the models.yaml profile,
//     not constants: the donor's hardcoded 60-second per-attempt deadline would
//     have killed real 63–278 s chunks and retried them three times for free;
//   - Retry-After on a 429 is read and honoured (the donor capped backoff at 8 s
//     — a hammer against a bulk chapter run's rate-limits), but is itself capped
//     by maxRetryAfterWait — otherwise a hostile «Retry-After: 7200» hangs the
//     stage for hours with a held reservation;
//   - the response body is read through a LimitReader (the donor capped only in
//     the gemini-native adapter).
//
// A documented departure from the donor: the one-shot self-heal of the
// reasoning_effort parameter (a model returning 400 was remembered and had the
// parameter stripped) is NOT ported — we set reasoning per-stage in config with
// fail-fast validation, and a provider's 400 must be fixed in config, not swallowed.

// RetryProfile bounds one client's retry loop. Zero fields fall back to
// defaults; the profile comes from models.yaml per provider (per-role
// overrides — Phase 1).
type RetryProfile struct {
	AttemptTimeout time.Duration // FLOOR for one HTTP attempt's deadline (see deadlineFor)
	MaxAttempts    int
	BackoffBase    time.Duration // first backoff; doubles per attempt
	BackoffCap     time.Duration
	// The three fields below turn the per-attempt deadline from a constant into a function of the
	// budget the call carries (attemptcut.go). All three are optional and all three are DATA: a
	// provider the repository has never seen gets a working deadline from the vendor default and a
	// startup warning naming what to measure, with no Go edit.
	//
	// TokensPerSecFloor is the slowest generation speed this provider has been OBSERVED to hold —
	// below the p10 of its own request_log, rounded down. Unset ⇒ the vendor default.
	TokensPerSecFloor float64
	// QueueSlack is how long the VENDOR documents a request may wait before generation starts. It is
	// added whole rather than amortized: the wait is not proportional to the budget.
	QueueSlack time.Duration
	// AttemptMax bounds the derived deadline — the longest one call may hold a reservation. 0 = unbounded.
	AttemptMax time.Duration
}

func (p RetryProfile) withDefaults() RetryProfile {
	if p.AttemptTimeout <= 0 {
		// The eval measured 63–278 s per chunk on a local model; cloud is faster,
		// but the default must survive a long chunk. Provider specifics live in
		// models.yaml; this is only a safety ceiling.
		p.AttemptTimeout = 300 * time.Second
	}
	if p.MaxAttempts <= 0 {
		p.MaxAttempts = 3
	}
	if p.BackoffBase <= 0 {
		p.BackoffBase = 500 * time.Millisecond
	}
	if p.BackoffCap <= 0 {
		p.BackoffCap = 30 * time.Second
	}
	return p
}

// h2ReadIdleTimeout / h2PingTimeout tune the cloud transport's HTTP/2 keepalive
// (research/21 §1.5, modelled on the official grok-build's 15s/5s): on a long
// thinking-heavy editor call NO body byte flows until the whole JSON is ready, so a
// proxy/LB can idle-close the socket MID-generation. An HTTP/2 PING keeps the
// connection alive independently of DATA frames; a dead connection then surfaces as
// a clean retryable transport error instead of a mid-generation RST that re-bills the
// call. These are transport keepalive frames only — they never touch the request
// body, so the wire and the job snapshot are unchanged (pack invariant: zero wire bytes).
const (
	h2ReadIdleTimeout = 15 * time.Second
	h2PingTimeout     = 5 * time.Second
)

// keepAliveHTTPClient builds the cloud transport with HTTP/2 PING keepalive. It
// clones http.DefaultTransport (proxy-from-env, dial/TLS timeouts, ForceAttemptHTTP2)
// and layers tuned h2 ping settings via http2.ConfigureTransports — so an HTTPS
// provider gets keepalive while a plaintext test server (httptest, no ALPN) still
// speaks HTTP/1.1 over the same base transport, unchanged. On the (unexpected)
// ConfigureTransports error the plain cloned transport is used as-is: keepalive is a
// reliability bonus, never a hard dependency. The local provider passes its OWN
// no-proxy client (httpc != nil), so it is untouched — localhost needs no h2 keepalive.
func keepAliveHTTPClient() *http.Client {
	c, _ := buildCloudClient()
	return c
}

// buildCloudClient is the construction itself, handing back BOTH the client and the h2 transport it
// installed the bounds on. The second return is what makes the bounds checkable at all: reading them off
// a finished client is impossible (ConfigureTransports answers an error the second time), so a check
// written against a client would have to re-derive them on a transport of its own and would then be
// asserting about a transport nobody uses.
func buildCloudClient() (*http.Client, *http2.Transport) {
	base := http.DefaultTransport.(*http.Transport).Clone()
	h2 := tuneHTTP2(base)
	return &http.Client{Transport: base, CheckRedirect: doNotFollowRedirects}, h2
}

// tuneHTTP2 installs the keepalive pair and RETURNS the h2 transport it configured, so a test can read
// what was actually set rather than re-deriving it. Asking ConfigureTransports a second time answers an
// error and no transport, so a check written against an already-built client asserts nothing at all.
// nil on the (unexpected) configure error: keepalive is a reliability bonus, never a hard dependency.
func tuneHTTP2(base *http.Transport) *http2.Transport {
	h2, err := http2.ConfigureTransports(base)
	if err != nil || h2 == nil {
		return nil
	}
	h2.ReadIdleTimeout = h2ReadIdleTimeout
	h2.PingTimeout = h2PingTimeout
	return h2
}

// doNotFollowRedirects stops the client at a 3xx instead of chasing it, so the redirect surfaces as the
// terminal non-2xx it is and the operator is told the base_url is wrong.
//
// ⛔ IT IS A MONEY GUARD BEFORE IT IS A HYGIENE ONE. `Do` spans the WHOLE redirect chain, and the
// delivery trace does not reset between its legs: the first leg reaching a redirector sets WroteRequest
// (and GotFirstResponseByte) for good, so a second leg whose connect is REFUSED still looked delivered.
// Measured through the ledger: $0.001056 booked for a `connect: connection refused` that never put a byte
// on any wire, with AfterHeaders true and zero bytes from the target. A stale `http://` or a normalised
// slash in base_url is enough to trigger it.
//
// The second reason is the one a security review would raise first: net/http drops Authorization only
// across a host change it considers unsafe, and a provider endpoint that redirects is a misconfiguration
// in every case — there is no shape in which silently following one is what an operator wanted.
func doNotFollowRedirects(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }

// maxResponseBytes caps one completion body read. A translated chapter chunk
// is ~10–50 KiB; 16 MiB leaves two orders of magnitude of headroom while
// keeping a misbehaving endpoint from exhausting memory.
const maxResponseBytes = 16 << 20

// maxRetryAfterWait bounds an honoured Retry-After: the provider's hint wins
// over our backoff schedule, but the HINT is capped to this — a stage holding a
// USD reservation must stay interruptible-by-timeout, not wedged for hours. The
// realized sleep may exceed this by the de-sync jitter (≤+25%, nextBackoff): jitter
// is added AFTER the cap ON PURPOSE, else N parallel waves that all see the same
// Retry-After≥5min would collapse to the identical instant — the thundering herd the
// jitter exists to break. The wait stays ctx-interruptible either way.
const maxRetryAfterWait = 5 * time.Minute

// retryLoop is the ONE retry engine shared by every transport (OpenAI-compat
// and the native Anthropic adapter): exponential backoff with cap, Retry-After
// override, jitter, ctx-done select. A single policy instance — so a retry
// fix doesn't "drift" to one provider while forgetting another.
//
// Logs are a product for the operator (package №4, pains cleared by a smoke run):
// each attempt's start is visible at DEBUG (otherwise "hung on a timeout" is
// indistinguishable from "died"), WARN carries the ACTUAL wait before the next
// attempt (Retry-After can silently stretch it to 5 min against the expected
// backoff_cap) and says "will retry" ONLY when an attempt actually remains —
// previously the last failure too was logged "will retry" before the exhausted error.
func retryLoop[T any](ctx context.Context, profile RetryProfile, name string, log *slog.Logger, attempt func() (T, bool, error)) (T, error) {
	var zero T
	var lastErr error
	// ⛔ A DELIVERED CUT MUST OUTLIVE THE ATTEMPT THAT MADE IT. Only the LAST error left this loop, so a
	// chain that cut a delivered request and then met a terminal 4xx — a 400/401/403/413 on the retry,
	// entirely reachable — returned the status alone: the runner saw «the request never went out», gave
	// the reservation back and booked $0 for a generation the provider had made, leaving the position
	// with no mark either. The first cut is kept and joined onto whatever ends the chain, so the money
	// and the reason a caller finally reports are both true of one error.
	var owedCut error
	for att := 0; att < profile.MaxAttempts; att++ {
		if log != nil {
			log.DebugContext(ctx, name+" attempt start", "attempt", att+1, "max", profile.MaxAttempts,
				"attempt_timeout", profile.AttemptTimeout.String())
		}
		resp, retryable, err := attempt()
		if err == nil {
			return resp, nil
		}
		lastErr = err
		owedCut = moreOwed(owedCut, err)
		if ctx.Err() != nil {
			return zero, chainError(cancelledDuring(ctx.Err(), err), owedCut)
		}
		if !retryable {
			return zero, chainError(err, owedCut)
		}
		if att+1 >= profile.MaxAttempts {
			break // attempts exhausted — no retry remains, no sleep
		}
		backoff := nextBackoff(profile, att, lastErr)
		if log != nil {
			log.WarnContext(ctx, name+" attempt failed, will retry", "attempt", att+1, "max", profile.MaxAttempts,
				"retry_in", backoff.Round(time.Millisecond).String(), "err", err)
		}
		select {
		case <-ctx.Done():
			// ⛔ THE SECOND CANCELLATION EXIT, and it used to throw the evidence away where the first one
			// no longer does. A run stopped DURING a backoff has an attempt behind it that may already
			// have been delivered and billed — a cut connection, an undecodable 2xx — and returning a
			// bare ctx.Err() here left the runner with nothing to settle and the chunk with no mark at
			// all. Measured: a delivered connection_lost plus a stop inside the backoff booked $0 and
			// wrote no chunk_status row. The window is the whole sleep, up to a minute on the shipping
			// config, and it opens precisely on the runs where a provider is flapping and an operator is
			// therefore reaching for the stop.
			//
			// ⚠ THIS CLOSES THE MONEY HALF ONLY. What leaves here carries the cut with the strongest
			// money claim, which is deliberately NOT the stop when an earlier paid break outranks it — so
			// the runner's mark cannot be decided by this error's first cause. The mark asks its own
			// question (pipeline's recordCancelledStage: was the run stopped, and did ANY cut deliver),
			// and a guard that read the first cause instead left the position with no row at all.
			return zero, chainError(cancelledDuring(ctx.Err(), lastErr), owedCut)
		case <-time.After(backoff):
		}
	}
	return zero, chainError(fmt.Errorf("%s: exhausted %d attempts: %w", name, profile.MaxAttempts, lastErr), owedCut)
}

// ⛔ THE ERROR IS A SCALAR AND THE CHAIN IS A LIST, and every defect this pair exists to stop came from
// that mismatch. A retry chain can deliver, and be billed for, an attempt that is NOT the attempt whose
// error ends it: a cut, then a 503, then a stop. Whatever one error the loop finally returns is the ONLY
// thing the caller settles from — so it must carry the cut the chain owes money for, from every exit,
// not from the two that happened to be written with it in mind.
//
// moreOwed is the accumulator, and it keeps the FIRST cut deliberately rather than ranking them here.
//
// ⚠ RANKING BELONGS AT THE EXIT, and putting it here as well would be a guard that cannot fire. A chain
// holds at most TWO cuts: only CutByConnection is retryable (AttemptCutError.retryable), and the
// delivered-cut cap makes the second one terminal (`retryable && deliveredCutSeen > 1` → not retryable).
// So whenever two cuts exist, the second one IS the error ending the chain and is in `chainError`'s hand
// already; whenever only one exists, there is nothing to rank. A billable-beats-free branch here would
// read like a money guard and never execute — the shape this pack has spent a shift removing.
func moreOwed(kept, candidate error) error {
	var cc *AttemptCutError
	if kept != nil || !errors.As(candidate, &cc) {
		return kept // already holding one, or not a cut at all — a 503 owes nobody anything
	}
	return candidate
}

// chainError attaches what the chain owes to whatever error ends it. It returns `final` untouched when
// nothing is owed, when the owed cut IS what ended it (joining an error to itself prints the sentence
// twice), or when `final` already carries a cut with a money claim at least as strong.
//
// ⚠ THE TEST IS THE MONEY, NOT THE TYPE. The version this replaces asked `errors.As(final, &cut)` and
// returned early on any cut at all — true while money was drawn on `Delivered`, false the moment it was
// drawn on `Billable`: a later FREE cut then erased an earlier PAID one and the engine booked $0.
func chainError(final, owed error) error {
	if owed == nil || errors.Is(final, owed) {
		return final
	}
	var fc *AttemptCutError
	if errors.As(final, &fc) {
		var oc *AttemptCutError
		if errors.As(owed, &oc) && oc.Billable && !fc.Billable {
			// ⛔ THE OWED CUT GOES FIRST, and the order is the whole assertion. `errors.As` hands back the
			// FIRST match it meets walking the tree, so joining the free cut ahead of the paid one leaves
			// the caller settling from the free one — the very masking this branch exists to undo.
			return errors.Join(owed, final)
		}
		return final
	}
	return errors.Join(final, owed)
}

// nextBackoff computes the wait before attempt att+1 (0-based att just failed):
// exponential base<<att with cap, overridden by an honoured Retry-After (bounded
// by maxRetryAfterWait), plus jitter. Extracted from the loop so the WARN above
// can log the REAL wait it is about to sleep.
func nextBackoff(profile RetryProfile, att int, lastErr error) time.Duration {
	shift := att
	if shift > 20 {
		shift = 20 // guard the shift against overflow at large max_attempts
	}
	backoff := profile.BackoffBase << uint(shift)
	if backoff <= 0 || backoff > profile.BackoffCap {
		backoff = profile.BackoffCap
	}
	if ra := retryAfterOf(lastErr); ra > 0 {
		if ra > maxRetryAfterWait {
			ra = maxRetryAfterWait
		}
		// Honour the server's hint as a FLOOR and de-sync parallel waves with
		// POSITIVE-only proportional jitter (up to +25%): N calls that hit the same
		// 429 must not all retry at the same instant (thundering herd), but we never
		// retry BEFORE the provider said to.
		return ra + proportionalJitter(ra, false)
	}
	// Exponential backoff: symmetric ±25% proportional jitter (research/21 §1.18а)
	// de-syncs parallel waves far better than the old fixed 0–250ms window, which
	// barely moved a 30s backoff.
	return backoff + proportionalJitter(backoff, true)
}

// proportionalJitter returns a jitter offset for a backoff of duration d. symmetric
// spreads it over [−25%, +25%] (mean-preserving, for exponential backoff); otherwise
// over [0, +25%] (for an honoured Retry-After, which must never be shortened). d ≤ 0
// yields 0.
func proportionalJitter(d time.Duration, symmetric bool) time.Duration {
	spread := int64(d) / 4 // 25%
	if spread <= 0 {
		return 0
	}
	if symmetric {
		return time.Duration(rand.Int63n(2*spread+1) - spread)
	}
	return time.Duration(rand.Int63n(spread + 1))
}

// openAIClient performs OpenAI-compatible /chat/completions calls with retry.
type openAIClient struct {
	name    string // provider label for logs/errors ("deepseek", "glm", "local")
	base    string
	key     string
	http    *http.Client
	profile RetryProfile
	headers map[string]string // extra static headers (provider-specific), may be nil
	log     *slog.Logger
	// floorWarned fires the «this provider has no measured speed» notice ONCE per client rather than
	// once per call: the fact is about the configuration, and a wave of forty chunks would otherwise
	// print it forty times and teach the operator to scroll past it.
	floorWarned sync.Once
}

// newOpenAIClient builds the shared transport. httpc may be nil (default
// client); the local provider passes an explicit no-proxy client (the stand's
// proxy gotcha: env-proxy intercepts non-loopback local addresses like 172.x).
func newOpenAIClient(name, base, key string, profile RetryProfile, headers map[string]string, httpc *http.Client, logger *slog.Logger) *openAIClient {
	if httpc == nil {
		httpc = keepAliveHTTPClient()
	}
	return &openAIClient{
		name:    name,
		base:    base,
		key:     key,
		http:    httpc,
		profile: profile.withDefaults(),
		headers: headers,
		log:     logger,
	}
}

// --- OpenAI-compatible wire types -------------------------------------------------

type openAIMessage struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

// openAIRequest is the neutral pre-wire request. Its body is assembled by
// MarshalJSON THROUGH the per-model Capability resolver (capability.go), not
// from fixed json tags: OpenAI-compatible providers share an endpoint but not a
// body — the budget key, whether temperature is emitted, and the reasoning
// switch all vary per model (D3.1). The Phase-0 fixed body (max_tokens +
// always-send temperature + reasoning_effort) 400s on Kimi / gpt-5 / Gemini;
// the resolver renders the correct shape and is itself folded into the job
// snapshot, so changing it invalidates checkpoints rather than false-hitting.
type openAIRequest struct {
	model       string
	messages    []openAIMessage
	maxTokens   int
	temperature float64
	stream      bool
	// reasoningEffort is the NEUTRAL effort ("" | off | low | medium | high);
	// cap decides how — and whether — it reaches the wire.
	reasoningEffort string
	responseFormat  any
	cap             Capability
	// extra carries per-model provider-specific fields from models.yaml. Merged
	// LAST and only into gaps — a resolved standard key always wins.
	extra map[string]any
}

// MarshalJSON assembles the wire body via the capability resolver. Resolved
// standard keys are authoritative; config-supplied extra_body fills only gaps.
func (r openAIRequest) MarshalJSON() ([]byte, error) {
	m := map[string]any{
		"model":    r.model,
		"messages": r.messages,
		"stream":   r.stream,
	}
	r.cap.applyToBody(m, r.maxTokens, r.temperature, r.reasoningEffort)
	if r.responseFormat != nil {
		m["response_format"] = r.responseFormat
	}
	mergeBody(m, r.extra)
	return json.Marshal(m)
}

type openAIUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	// TotalTokens is prompt+completion for spec providers, but for Gemini via the
	// OpenAI-compat layer it ALSO carries the mandatory thinking tokens, which do NOT
	// appear in completion_tokens and have no reasoning_tokens field (live-probe
	// 2026-07-10: completion=2, prompt=22, total=847 → 823 hidden thinking). The
	// ReasoningAdditiveTotal adapter derives thinking = total − prompt − completion.
	TotalTokens         int `json:"total_tokens"`
	PromptTokensDetails struct {
		CachedTokens int `json:"cached_tokens"`
	} `json:"prompt_tokens_details"`
	// xAI reports reasoning tokens here SEPARATELY from completion_tokens and
	// bills them at the output rate; OpenAI-spec providers count them inside
	// completion_tokens. The adapter decides which semantics apply.
	CompletionTokensDetails struct {
		ReasoningTokens int `json:"reasoning_tokens"`
	} `json:"completion_tokens_details"`
	// DeepSeek historically reports cache usage in its own top-level fields
	// instead of prompt_tokens_details. Parse both so the Phase-0 cache
	// experiment can't show a false zero (03-implementation-notes §3.5).
	PromptCacheHitTokens  int `json:"prompt_cache_hit_tokens"`
	PromptCacheMissTokens int `json:"prompt_cache_miss_tokens"`
}

// cacheRead returns the cache-hit token count whichever field the provider
// used.
func (u openAIUsage) cacheRead() int {
	if u.PromptTokensDetails.CachedTokens > 0 {
		return u.PromptTokensDetails.CachedTokens
	}
	return u.PromptCacheHitTokens
}

type openAIResponse struct {
	ID      string `json:"id"`
	Model   string `json:"model"`
	Choices []struct {
		Message struct {
			Content string `json:"content"`
		} `json:"message"`
		FinishReason string `json:"finish_reason"`
	} `json:"choices"`
	Usage openAIUsage `json:"usage"`
}

func (r *openAIResponse) Text() string {
	if len(r.Choices) == 0 {
		return ""
	}
	return r.Choices[0].Message.Content
}

func (r *openAIResponse) FinishReason() string {
	if len(r.Choices) == 0 {
		return ""
	}
	// OpenAI-compat wire values map 1:1 onto the neutral constants.
	return r.Choices[0].FinishReason
}

// complete calls Chat Completions with retry on transient failures (429 / 5xx /
// network, exponential backoff + jitter, Retry-After honoured). Non-retryable
// 4xx fail immediately. On exhaustion the caller releases the reservation, so
// a transient failure is never silently swallowed.
func (c *openAIClient) complete(ctx context.Context, reqBody openAIRequest) (*openAIResponse, error) {
	payload, err := json.Marshal(reqBody)
	if err != nil {
		return nil, err
	}
	billedDecodeSeen := 0
	deliveredCutSeen := 0
	// ⛔ TWO DIFFERENT COUNTS, and they were one field. `deliveredCutSeen` is a RETRY CAP keyed on cuts.
	// `askedToGenerate` is what the operator is told — how many times this request reached the provider
	// inside one chain — and a chain delivers in more ways than by being cut: an undecodable 2xx that
	// already billed, a terminal 4xx, a 503. Counting only the cuts made the ledger line understate a
	// mixed chain while calling itself «the provider was asked N times».
	askedToGenerate := 0
	// ⛔ EVERY CUT THE CHAIN MADE IS RE-STAMPED, not only the newest. The count is a property of the
	// CHAIN, and chainError deliberately hands up an EARLIER cut when that is the one that owes money —
	// carrying a number frozen at the moment it was born. A chain that cut once and was then answered
	// two 503s delivered three times and reported one, under-stating the very gap the line exists to
	// show. The later deliveries are not cuts, so nothing in the error tree knows about them; only this
	// counter does.
	var chainCuts []*AttemptCutError
	return retryLoop(ctx, c.profile, c.name, c.log, func() (*openAIResponse, bool, error) {
		resp, retryable, err := c.attempt(ctx, payload, reqBody.maxTokens)
		if deliveredAttempt(resp, err) {
			askedToGenerate++
		}
		defer func() {
			for _, cc := range chainCuts {
				cc.Deliveries = askedToGenerate
			}
		}()
		// Cap a DELIVERED cut's re-calls at ONE, for the same reason and by the same shape as the
		// billed-decode cap above — but keyed on DELIVERY rather than on a 2xx. That is the whole
		// correction: on a provider that answers 200 while the request is still queued (DeepSeek
		// documents exactly that), «did a 2xx arrive» says nothing about whether a generation was
		// bought, while «did the request go out» says it exactly. A broken connection after delivery
		// is worth one more call; a second is a dead provider, not a flaky socket.
		if err != nil {
			var cut *AttemptCutError
			if errors.As(err, &cut) {
				deliveredCutSeen++
				// The provider has now been asked to generate this many times, and the caller is told:
				// one settle will cover all of them, because the store writes spend only through a
				// checkpoint and they share one key.
				chainCuts = append(chainCuts, cut)
				if retryable && deliveredCutSeen > 1 {
					return resp, false, err
				}
			}
		}
		// Cap billed-decode re-bills at ONE (research/21 §1.18в): an undecodable 2xx
		// has ALREADY billed, so re-running it under the full MaxAttempts turns a
		// provider emitting 2xx garbage into a paid retry STORM (grok-build's
		// "laundering a serialization error into retryable gave a full-budget storm").
		// The first occurrence retries once (a transient proxy break is worth one shot);
		// a second billed-decode is terminal, so the runner settles at the estimate.
		if err != nil && retryable {
			var bde *BilledDecodeError
			if errors.As(err, &bde) {
				billedDecodeSeen++
				if billedDecodeSeen > 1 {
					return resp, false, err
				}
			}
		}
		return resp, retryable, err
	})
}

// deliveredAttempt reports whether ONE attempt's request reached the provider — the question the
// operator's «asked N times» line answers, and a different question from «did it end in a cut».
//
// A status of any kind is delivery by definition: the peer had to read the request to answer it. So is a
// 2xx whose body would not parse — that one has already billed. A cut says so itself. What is NOT a
// delivery is everything that failed before the bytes left: a refused connect, a DNS failure, a write
// that died mid-body.
func deliveredAttempt(resp *openAIResponse, err error) bool {
	if err == nil {
		return resp != nil
	}
	var cut *AttemptCutError
	if errors.As(err, &cut) {
		return cut.Delivered
	}
	var hse *HTTPStatusError
	var bde *BilledDecodeError
	return errors.As(err, &hse) || errors.As(err, &bde)
}

// attempt performs one HTTP call. Returns retryable=true for 429/5xx and
// network errors, false for other non-2xx (terminal 4xx). The per-attempt
// deadline is derived from THIS call's output budget (attemptcut.go); the
// overall per-request deadline (set by the caller via ctx) bounds the whole retry loop.
//
// maxTokens is the budget the body already carries. It is passed rather than re-parsed so the
// deadline and the request can never describe different calls.
func (c *openAIClient) attempt(ctx context.Context, payload []byte, maxTokens int) (*openAIResponse, bool, error) {
	deadline := c.attemptDeadline(ctx, maxTokens)
	attemptCtx, cancel := context.WithTimeout(ctx, deadline)
	defer cancel()

	// The delivery facts are collected by the transport itself: a request is DELIVERED once its bytes
	// are written, which is knowable before any reply exists and is the boundary the money is drawn on.
	var tr deliveryTrace
	started := time.Now()
	req, err := http.NewRequestWithContext(httptrace.WithClientTrace(attemptCtx, tr.clientTrace()),
		http.MethodPost, c.base+"/chat/completions", bytes.NewReader(payload))
	if err != nil {
		return nil, false, err
	}
	req.Header.Set("Content-Type", "application/json")
	// A local backend (ollama / llama-server) usually runs without auth; an
	// empty key means "no Authorization header", not "Bearer " with an empty token.
	if c.key != "" {
		req.Header.Set("Authorization", "Bearer "+c.key)
	}
	for k, v := range c.headers {
		req.Header.Set(k, v)
	}

	resp, err := c.http.Do(req)
	if err != nil {
		// A DELIVERED request that never answered: the provider has it and is (or was) working, so
		// this is a money event and carries its cause. An UNDELIVERED one stays exactly what it was —
		// a plain retryable transport failure whose reservation is released, the one case where
		// «nothing was bought» is true by construction.
		if cut := c.cutError(ctx, attemptCtx, &tr, started, nil, err, false); cut != nil {
			return nil, cut.retryable(), cut
		}
		// The bare «context deadline exceeded» doesn't tell the operator WHOSE deadline it is —
		// this call's own (derived from its budget) or the whole run cancelled.
		if errors.Is(attemptCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil {
			err = fmt.Errorf("attempt timed out after %s (derived from max_tokens=%d; timeouts.attempt_s is its floor) before the request was delivered: %w", deadline, maxTokens, err)
		}
		return nil, ctx.Err() == nil, err
	}
	defer resp.Body.Close()
	// Read one byte past the limit to DISTINGUISH truncation from a whole body — for the >16 MiB case
	// alone. Whether the read COMPLETED is a separate question with a separate answer, and dropping
	// readErr here is what made a body our own deadline cut short (small) indistinguishable from a
	// whole one, so it went down the retryable branch and bought the same generation twice.
	data, readErr := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1))
	truncated := len(data) > maxResponseBytes
	if truncated {
		data = data[:maxResponseBytes]
	}

	obs.LogLLMExchange(ctx, c.log, c.name, payload, resp.StatusCode, data)

	// ⛔ THE STATUS LINE IS ASKED BEFORE THE READ ERROR, and the order is the money. A non-2xx says the
	// provider refused or failed — nothing was generated and nothing is owed — so a body cut short
	// under it is a detail of a failure, not a purchase. Reading them the other way round would settle
	// an estimate for every 4xx whose tiny body happened to land on the deadline. The partial body is
	// still what the retry/terminal split reads (a truncated marker just fails to match and the status
	// stays retryable, the conservative direction), and the read error rides in the message so a
	// misclassified 429 leaves a trace instead of none.
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		retryable := retryableStatus(resp.StatusCode, data)
		e := &HTTPStatusError{Provider: c.name, Status: resp.StatusCode, Body: snippet(data)}
		if readErr != nil {
			e.Body = snippet(data) + fmt.Sprintf(" [body read incomplete: %v]", readErr)
		}
		if retryable {
			e.RetryAfter = parseRetryAfter(resp.Header) // only a retryable status will honour it
		}
		return nil, retryable, e
	}

	if readErr != nil {
		// A 2xx whose body we did not receive whole. Headers had arrived, so the request was written
		// by definition — but ask the trace rather than assume it, and let an undelivered
		// impossibility fall through to the old shape instead of settling money on a deduction.
		if cut := c.cutError(ctx, attemptCtx, &tr, started, data, readErr, true); cut != nil {
			return nil, cut.retryable(), cut
		}
		return nil, ctx.Err() == nil, readErr
	}

	var out openAIResponse
	if err := json.Unmarshal(data, &out); err != nil {
		// A 2xx with an unreadable body: the provider has ALREADY charged. We type it
		// so the runner conservatively settles at the estimate. A truncated (>16 MiB)
		// body is deterministic — retrying is pointless (each retry is a new billed
		// 2xx), so terminal; a broken connection / garbage is retryable.
		if truncated {
			return nil, false, &BilledDecodeError{Provider: c.name, Err: fmt.Errorf("response exceeded %d bytes (truncated, not retried): %w", maxResponseBytes, err)}
		}
		return nil, true, &BilledDecodeError{Provider: c.name, Err: err}
	}
	// A 2xx is a billed call even when the model returns empty content
	// (content filter, finish_reason=length with no text). Return it as a
	// success so the caller settles the real cost via the ledger instead of
	// releasing the reservation and losing the spend — which would let empty
	// replies bypass the book/day ceilings. Gates deal with the empty text.
	return &out, false, nil
}

// retryableStatus is the ONE source of the retry/terminal split for a non-2xx
// response — pinned by a drift-guard test (research/21 §1.18б) so a new status
// class can't silently change retry policy. Only 429 and 5xx are transient; every
// other non-2xx is a terminal config/request error that must fail loud. The one
// nuance is a CREDITS-EXHAUSTED 429: a body that marks the account/quota as spent
// is terminal, not transient — retrying under a held USD reservation only burns
// wall-clock (opencode/goose both classify quota/credits as fatal). A 402
// (Payment Required) is already terminal by falling through to the default.
func retryableStatus(status int, body []byte) bool {
	switch {
	case status == http.StatusTooManyRequests:
		return !isQuotaExhausted(body) // plain rate-limit → retry; credits gone → terminal
	case status >= 500:
		return true
	default:
		return false // terminal 4xx (incl. 402 Payment Required)
	}
}

// isQuotaExhausted reports whether a 429 body marks the account's credits/quota as
// spent — a NARROW, high-confidence marker set (research/21 §1.4), deliberately NOT
// a broad regex battery (the aider-style «exact matchers dead» brittleness this pack
// explicitly avoids). `insufficient_quota` is the OpenAI-family error code;
// `quota_exceeded` covers the others. Retrying either never succeeds — the operator
// must top up — so the wave fails loud in one attempt instead of 3 + honoured-backoff.
func isQuotaExhausted(body []byte) bool {
	b := strings.ToLower(string(body))
	return strings.Contains(b, "insufficient_quota") || strings.Contains(b, "quota_exceeded")
}

// HTTPStatusError is a non-2xx completion failure, carrying the status code as
// a typed field so callers (the failover decorator) can classify it
// structurally — a terminal 4xx is a config/request error that must fail loud,
// not be masked by a fallback — instead of parsing the message.
type HTTPStatusError struct {
	Provider   string
	Status     int
	Body       string
	RetryAfter time.Duration // from the Retry-After header; 0 = absent
}

func (e *HTTPStatusError) Error() string {
	return fmt.Sprintf("%s http %d: %s", e.Provider, e.Status, e.Body)
}

// BilledDecodeError is a 2xx whose body could not be decoded: the provider has
// billed the call, but usage/text are unknown. The runner treats an exhausted
// retry chain ending in this error as BILLED (settle at the reservation
// estimate) rather than releasing the reservation.
type BilledDecodeError struct {
	Provider string
	Err      error
}

func (e *BilledDecodeError) Error() string {
	return fmt.Sprintf("%s: 2xx body decode failed (call IS billed): %v", e.Provider, e.Err)
}

func (e *BilledDecodeError) Unwrap() error { return e.Err }

func retryAfterOf(err error) time.Duration {
	var se *HTTPStatusError
	if errors.As(err, &se) {
		return se.RetryAfter
	}
	return 0
}

// toOpenAIMessages maps neutral messages onto the wire. CacheBoundary is
// meaningless here: OpenAI-compatible caches (DeepSeek: prefix-match from token
// 0 in blocks of 64 tokens) are automatic; the stable-prefix ORDER the
// assembler produced is all that matters — which is also why JOINING the system
// run under SystemMessagesSingle costs no cache: the stable prefix keeps its
// bytes and its position, it merely stops being its own message.
//
// mode is the endpoint's system-message cardinality (Capability.SystemMessages).
// Under SystemMessagesSingle the LEADING system run is joined into one message so
// an endpoint with one system slot receives the memory-bank injection instead of
// silently dropping it; a system message AFTER a non-system turn is refused loud,
// exactly as the Anthropic adapter refuses it — our assembler never produces one,
// and a join that quietly re-ordered a conversation would be a worse lie than the
// one this function exists to stop. Under SystemMessagesMulti (the default) the
// mapping is one-for-one, byte-identical to the wire before this axis existed.
func toOpenAIMessages(msgs []Message, mode SystemMessagesMode) ([]openAIMessage, error) {
	if mode != SystemMessagesSingle {
		out := make([]openAIMessage, len(msgs))
		for i, m := range msgs {
			out[i] = openAIMessage{Role: m.Role, Content: m.Content}
		}
		return out, nil
	}
	var systemRun []string
	out := make([]openAIMessage, 0, len(msgs))
	inSystemPrefix := true
	for _, m := range msgs {
		if m.Role == "system" {
			if !inSystemPrefix {
				return nil, fmt.Errorf("llm: this endpoint carries a single system message and got one after a non-system turn; joining it would re-order the conversation")
			}
			systemRun = append(systemRun, m.Content)
			continue
		}
		if inSystemPrefix {
			inSystemPrefix = false
			if len(systemRun) > 0 {
				out = append(out, openAIMessage{Role: "system", Content: strings.Join(systemRun, systemJoinSeparator)})
			}
		}
		out = append(out, openAIMessage{Role: m.Role, Content: m.Content})
	}
	// A message list that is system-only (no user turn) still has to ship its join.
	if inSystemPrefix && len(systemRun) > 0 {
		out = append(out, openAIMessage{Role: "system", Content: strings.Join(systemRun, systemJoinSeparator)})
	}
	return out, nil
}

// jsonResponseFormat returns the response_format value for JSONOnly requests
// (nil otherwise, so the field serializes away).
func jsonResponseFormat(jsonOnly bool) any {
	if jsonOnly {
		return map[string]string{"type": "json_object"}
	}
	return nil
}

// parseRetryAfter reads a provider's honoured backoff hint from the response
// headers (research/21 §1.13). It accepts, in priority order:
//   - Retry-After-Ms (millisecond precision; opencode/openai-go/anthropic-sdk-go
//     all read it FIRST — some providers only send this);
//   - Retry-After as FRACTIONAL or integer seconds (the old strconv.Atoi rejected
//     "1.5"/"0.5" → 0 → blind exponential backoff, ignoring a sub-second hint);
//   - Retry-After as an HTTP-date.
//
// The value is still bounded by maxRetryAfterWait in nextBackoff — this only widens
// what we can PARSE, never how long we will actually wait.
func parseRetryAfter(h http.Header) time.Duration {
	if ms := strings.TrimSpace(h.Get("Retry-After-Ms")); ms != "" {
		if n, err := strconv.ParseFloat(ms, 64); err == nil && n > 0 {
			return time.Duration(n * float64(time.Millisecond))
		}
	}
	v := strings.TrimSpace(h.Get("Retry-After"))
	if v == "" {
		return 0
	}
	// Fractional or integer seconds. ParseFloat("120") is 120, so this also covers
	// the integer form the old Atoi handled.
	if secs, err := strconv.ParseFloat(v, 64); err == nil && secs > 0 {
		return time.Duration(secs * float64(time.Second))
	}
	if t, err := http.ParseTime(v); err == nil {
		if d := time.Until(t); d > 0 {
			return d
		}
	}
	return 0
}

func snippet(b []byte) string {
	const max = 300
	if len(b) > max {
		return string(b[:max]) + "…"
	}
	return string(b)
}