textmachine/backend/internal/llm/reasoningvisibility_test.go

package llm

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

// serveUsage answers one OpenAI-compat completion with the given usage block verbatim, so a test can
// say exactly which fields the PROVIDER sent — including the difference between a field carrying 0 and
// a field that is not there at all.
func serveUsage(t *testing.T, usage string) *httptest.Server {
	t.Helper()
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		openAIOK(t, w, `{"id":"r1","choices":[{"message":{"content":"x"},"finish_reason":"stop"}],"usage":`+usage+`}`)
	}))
	t.Cleanup(srv.Close)
	return srv
}

func completeWith(t *testing.T, srv *httptest.Server, sem ReasoningSemantics) *LLMResponse {
	t.Helper()
	c := NewOpenAICompatClient(OpenAICompatConfig{Name: "p", BaseURL: srv.URL, Profile: fastProfile(), Reasoning: sem}, nil)
	resp, err := c.Complete(context.Background(), LLMRequest{Model: "m", Messages: []Message{{Role: "user", Content: "u"}}, MaxTokens: 16})
	if err != nil {
		t.Fatal(err)
	}
	return resp
}

// TestSubsetReasoningIsRecordedWithoutBeingBilled is the thinking share of an already-paid completion:
// on a subset-billing provider the number the provider reports must arrive as ReasoningInCompletion and
// must NOT arrive as ReasoningTokens, which ledger.CostUSD adds to the completion.
//
// The measured shape this stands on: DeepSeek fills completion_tokens_details on every reply, and the
// identity total == prompt + completion holds on all of it — the thinking is already paid for inside
// completion_tokens. Surfacing it as ReasoningTokens would bill it a second time.
func TestSubsetReasoningIsRecordedWithoutBeingBilled(t *testing.T) {
	srv := serveUsage(t, `{"prompt_tokens":10,"completion_tokens":8496,"total_tokens":8506,
		"completion_tokens_details":{"reasoning_tokens":8496}}`)
	resp := completeWith(t, srv, ReasoningSubset)

	if resp.Usage.ReasoningInCompletion == nil {
		t.Fatal("a reported thinking count must be recorded, not dropped: ReasoningInCompletion is nil")
	}
	if got := *resp.Usage.ReasoningInCompletion; got != 8496 {
		t.Fatalf("ReasoningInCompletion = %d, want the provider's 8496", got)
	}
	if resp.Usage.ReasoningTokens != 0 {
		t.Fatalf("a subset provider's thinking is already inside completion_tokens; billing it again would double-charge it: ReasoningTokens = %d", resp.Usage.ReasoningTokens)
	}
	if resp.Usage.CompletionTokens != 8496 {
		t.Fatalf("CompletionTokens = %d, want the provider's 8496 untouched", resp.Usage.CompletionTokens)
	}
}

// TestAReportedZeroIsNotTheSameAsNoAnswer is the whole point of the pointer. A subset provider returns
// a genuine 0 for a call that did not think, and a provider that reports no such field returns nothing
// at all; spelling both as 0 is what made `reasoning_tokens` read as a measured number for the life of
// the project while nobody had asked the question.
func TestAReportedZeroIsNotTheSameAsNoAnswer(t *testing.T) {
	reportedZero := completeWith(t, serveUsage(t,
		`{"prompt_tokens":10,"completion_tokens":1617,"total_tokens":1627,"completion_tokens_details":{"reasoning_tokens":0}}`), ReasoningSubset)
	if reportedZero.Usage.ReasoningInCompletion == nil {
		t.Fatal("a provider that REPORTED zero thinking has answered the question; that answer must not read as silence")
	}
	if got := *reportedZero.Usage.ReasoningInCompletion; got != 0 {
		t.Fatalf("a reported zero must stay zero, got %d", got)
	}

	noField := completeWith(t, serveUsage(t,
		`{"prompt_tokens":10,"completion_tokens":1617,"total_tokens":1627}`), ReasoningSubset)
	if noField.Usage.ReasoningInCompletion != nil {
		t.Fatalf("a provider that reported nothing must leave the question unanswered, got %d", *noField.Usage.ReasoningInCompletion)
	}

	emptyDetails := completeWith(t, serveUsage(t,
		`{"prompt_tokens":10,"completion_tokens":1617,"total_tokens":1627,"completion_tokens_details":{}}`), ReasoningSubset)
	if emptyDetails.Usage.ReasoningInCompletion != nil {
		t.Fatalf("a details block without the key is still no answer, got %d", *emptyDetails.Usage.ReasoningInCompletion)
	}
}

// TestAdditiveBillingLeavesTheSubsetFieldEmpty keeps the two questions apart from the other side: where
// thinking bills ON TOP, the count is money and belongs to ReasoningTokens alone. A number in both
// fields would be one call's thinking described twice, and the next reader summing them would overcount
// exactly the share this pack exists to make visible.
func TestAdditiveBillingLeavesTheSubsetFieldEmpty(t *testing.T) {
	body := `{"prompt_tokens":10,"completion_tokens":5,"total_tokens":65,"completion_tokens_details":{"reasoning_tokens":50}}`

	additive := completeWith(t, serveUsage(t, body), ReasoningAdditive)
	if additive.Usage.ReasoningTokens != 50 {
		t.Fatalf("additive billing must surface the count as money, got %d", additive.Usage.ReasoningTokens)
	}
	if additive.Usage.ReasoningInCompletion != nil {
		t.Fatalf("additive thinking is NOT inside the completion; the subset field must stay empty, got %d", *additive.Usage.ReasoningInCompletion)
	}

	total := completeWith(t, serveUsage(t,
		`{"prompt_tokens":22,"completion_tokens":2,"total_tokens":847}`), ReasoningAdditiveTotal)
	if total.Usage.ReasoningTokens != 823 {
		t.Fatalf("additive_total must derive 823 from the total, got %d", total.Usage.ReasoningTokens)
	}
	if total.Usage.ReasoningInCompletion != nil {
		t.Fatalf("additive_total thinking is outside completion_tokens; the subset field must stay empty, got %d", *total.Usage.ReasoningInCompletion)
	}
}

// TestAnUnansweredThinkingShareAddsNoBytes pins the marshalling half of the field's contract. Usage is
// serialised verbatim into every checkpoint's usage_json, so a field that emitted `"…":null` on every
// call would change the bytes of every checkpoint a provider that reports nothing ever writes — the
// byte-identity discipline Capability.MinMaxTokens and SystemMessages carry two files away, and which
// the comment on this field claims for itself.
//
// Measured before this test existed: deleting the tag survived all four packages, the golden included.
// Nothing HASHES usage_json, which is exactly why nothing went red — and exactly why the claim needed a
// pin of its own rather than a neighbour's.
func TestAnUnansweredThinkingShareAddsNoBytes(t *testing.T) {
	silent, err := json.Marshal(Usage{PromptTokens: 10, CompletionTokens: 8496})
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(silent), "ReasoningInCompletion") {
		t.Fatalf("a provider that reported nothing must add no bytes at all, got %s", silent)
	}

	// A MEASURED zero is an answer and must survive to disk; `omitempty` omits a nil pointer, never a
	// pointer to zero, and this is the assertion that says so out loud.
	zero := 0
	measured, err := json.Marshal(Usage{PromptTokens: 10, CompletionTokens: 1617, ReasoningInCompletion: &zero})
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(string(measured), `"ReasoningInCompletion":0`) {
		t.Fatalf("a measured zero must reach the checkpoint as a zero, got %s", measured)
	}

	// And it round-trips: an old checkpoint decodes to nil, a new one to the number it carried.
	var back Usage
	if err := json.Unmarshal(silent, &back); err != nil {
		t.Fatal(err)
	}
	if back.ReasoningInCompletion != nil {
		t.Fatalf("a checkpoint written without the field must decode as unanswered, got %d", *back.ReasoningInCompletion)
	}
	if err := json.Unmarshal(measured, &back); err != nil {
		t.Fatal(err)
	}
	if back.ReasoningInCompletion == nil || *back.ReasoningInCompletion != 0 {
		t.Fatalf("a measured zero must decode as a measured zero, got %v", back.ReasoningInCompletion)
	}
}