150 lines
7.1 KiB
Go
150 lines
7.1 KiB
Go
package ledger
|
|
|
|
import (
|
|
"math"
|
|
"testing"
|
|
|
|
"textmachine/backend/internal/llm"
|
|
)
|
|
|
|
func approx(t *testing.T, got, want float64, msg string) {
|
|
t.Helper()
|
|
if math.Abs(got-want) > 1e-12 {
|
|
t.Fatalf("%s: got %v, want %v", msg, got, want)
|
|
}
|
|
}
|
|
|
|
// The money formula — the core of the unit economics; we check every term, including
|
|
// the new one (cache write), and the degradation to the vojo formula without cache fields.
|
|
func TestCostUSD(t *testing.T) {
|
|
price := ModelPrice{InputPerM: 2.0, CachedPerM: 0.2, CacheWritePerM: 2.5, OutputPerM: 10.0}
|
|
|
|
// Anthropic-style call: 10k of total input = 6k uncached + 3k cache-read +
|
|
// 1k cache-write; 2k output.
|
|
u := llm.Usage{PromptTokens: 10_000, CachedTokens: 3_000, CacheCreationTokens: 1_000, CompletionTokens: 2_000}
|
|
want := 6_000*2.0/1e6 + 3_000*0.2/1e6 + 1_000*2.5/1e6 + 2_000*10.0/1e6
|
|
approx(t, CostUSD(price, u), want, "anthropic-style usage")
|
|
|
|
// OpenAI-compatible call without cache fields — the vojo formula.
|
|
u2 := llm.Usage{PromptTokens: 10_000, CompletionTokens: 2_000}
|
|
approx(t, CostUSD(price, u2), 10_000*2.0/1e6+2_000*10.0/1e6, "plain usage")
|
|
|
|
// Reasoning tokens are billed at the output rate ON TOP OF completion (xAI).
|
|
u3 := llm.Usage{PromptTokens: 1_000, CompletionTokens: 500, ReasoningTokens: 1_500}
|
|
approx(t, CostUSD(price, u3), 1_000*2.0/1e6+2_000*10.0/1e6, "additive reasoning")
|
|
|
|
// Guard against a provider that reported cached > prompt: the uncached term
|
|
// is clamped to 0 rather than going negative.
|
|
u4 := llm.Usage{PromptTokens: 100, CachedTokens: 500, CompletionTokens: 10}
|
|
want4 := 0*2.0/1e6 + 500*0.2/1e6 + 10*10.0/1e6
|
|
approx(t, CostUSD(price, u4), want4, "cached > prompt guard")
|
|
|
|
// Negative usage numbers (a provider accounting bug) must not REDUCE
|
|
// the cost — every term is clamped to ≥0.
|
|
u5 := llm.Usage{PromptTokens: -1000, CachedTokens: -50, CompletionTokens: -200, ReasoningTokens: -5}
|
|
if got := CostUSD(price, u5); got != 0 {
|
|
t.Fatalf("negative usage must never reduce cost below 0, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestPriceForResponse(t *testing.T) {
|
|
def := ModelPrice{InputPerM: 0.14, OutputPerM: 0.28} // cheap anchor
|
|
p, err := NewPricer(map[string]ModelPrice{
|
|
"deepseek-v4-flash": def,
|
|
"claude-sonnet-5": {InputPerM: 2.0, OutputPerM: 10.0},
|
|
}, def)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The model that actually responded is known — price by it, and say so.
|
|
if price, basis := p.PriceForResponse("claude-sonnet-5", "claude-sonnet-5"); price.InputPerM != 2.0 || basis != PriceByAnswerer {
|
|
t.Fatalf("known actual model must price by actual, got %v basis=%q", price.InputPerM, basis)
|
|
}
|
|
// Provider returned a dated slug of a premium model — NOT the cheap anchor, but
|
|
// the price of the REQUESTED model (otherwise a systematic under-accounting of the premium).
|
|
if price, basis := p.PriceForResponse("claude-sonnet-5", "claude-sonnet-5-20260101"); price.InputPerM != 2.0 || basis != PriceByRequested {
|
|
t.Fatalf("unknown actual must fall back to requested model price (2.0), got %v basis=%q", price.InputPerM, basis)
|
|
}
|
|
// Both unknown — the default anchor (never $0).
|
|
if price, basis := p.PriceForResponse("mystery", "also-mystery"); price.InputPerM != 0.14 || basis != PriceByAnchor {
|
|
t.Fatalf("both unknown must fall back to default anchor, got %v basis=%q", price.InputPerM, basis)
|
|
}
|
|
}
|
|
|
|
// TestTheBillNamesTheModelItCameFrom is the half the price alone cannot carry. A substitution is
|
|
// invisible from the figure: the same call site returns the same type whether the rate belongs to the
|
|
// model that answered or to one that did not, and only the second case can bill a reader for a model
|
|
// they did not get. Measured on the run of 11.09: 28 of 33 rows answered under a slug the catalogue
|
|
// does not list, and every one of them was priced from the request with nothing recording it.
|
|
func TestTheBillNamesTheModelItCameFrom(t *testing.T) {
|
|
cheap := ModelPrice{InputPerM: 0.30, OutputPerM: 1.20}
|
|
dear := ModelPrice{InputPerM: 1.32, OutputPerM: 3.96}
|
|
p, err := NewPricer(map[string]ModelPrice{"pro": dear, "flash": cheap}, cheap)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The direction with no guard on it: a request for the premium model, answered under a slug the
|
|
// catalogue does not carry, billed at the premium pin.
|
|
price, basis := p.PriceForResponse("pro", "pro-0813")
|
|
if !basis.Substituted() {
|
|
t.Fatal("a bill taken from a model that did not answer must say so")
|
|
}
|
|
if price.OutputPerM != dear.OutputPerM {
|
|
t.Fatalf("the fixture must actually price by the request, got %v", price.OutputPerM)
|
|
}
|
|
// The control: the ordinary case must NOT claim a substitution, else the signal is noise and the
|
|
// warning it drives fires on every call.
|
|
if _, basis := p.PriceForResponse("flash", "flash"); basis.Substituted() {
|
|
t.Fatalf("a call priced by the model that answered is not a substitution, got %q", basis)
|
|
}
|
|
}
|
|
|
|
func TestPricerNeverZero(t *testing.T) {
|
|
def := ModelPrice{InputPerM: 1, OutputPerM: 2}
|
|
p, err := NewPricer(map[string]ModelPrice{"known": {InputPerM: 5, CachedPerM: 1, OutputPerM: 10}}, def)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := p.PriceFor("known").InputPerM; got != 5 {
|
|
t.Fatalf("known model price: got %v", got)
|
|
}
|
|
// Unknown model (the actual one after a failover) — the default anchor, not $0.
|
|
if got := p.PriceFor("mystery-model").InputPerM; got != 1 {
|
|
t.Fatalf("unknown model must fall back to default price, got %v", got)
|
|
}
|
|
// A $0 default would blind the ceilings — the constructor must reject it.
|
|
if _, err := NewPricer(nil, ModelPrice{}); err == nil {
|
|
t.Fatal("zero default price must be rejected")
|
|
}
|
|
}
|
|
|
|
func TestEstimateUSDIsPessimistic(t *testing.T) {
|
|
price := ModelPrice{InputPerM: 2.0, CachedPerM: 0.2, OutputPerM: 10.0}
|
|
est := EstimateUSD(price, 1_000, 2_000, 0)
|
|
approx(t, est, 1_000*2.0/1e6+2_000*10.0/1e6, "estimate")
|
|
// A real call with a cache hit must cost no more than the estimate — otherwise
|
|
// settle would inflate committed above a ceiling that was let through on the tolerance.
|
|
real := CostUSD(price, llm.Usage{PromptTokens: 1_000, CachedTokens: 900, CompletionTokens: 2_000})
|
|
if real > est {
|
|
t.Fatalf("real cost %v exceeds reservation estimate %v", real, est)
|
|
}
|
|
}
|
|
|
|
// TestEstimateUSDAdditiveReasoningBuffer covers D13.6: the additive-reasoning buffer is
|
|
// reserved as output tokens, so an xAI-style think-ON call whose reasoning lands on top of
|
|
// completion no longer overshoots the reservation and blinds the ceiling.
|
|
func TestEstimateUSDAdditiveReasoningBuffer(t *testing.T) {
|
|
price := ModelPrice{InputPerM: 2.0, OutputPerM: 10.0}
|
|
base := EstimateUSD(price, 1_000, 2_000, 0)
|
|
withBuf := EstimateUSD(price, 1_000, 2_000, 3_000)
|
|
// The buffer adds exactly 3_000 output-priced tokens over the base estimate.
|
|
approx(t, withBuf-base, 3_000*10.0/1e6, "reasoning buffer term")
|
|
// A real additive call (completion 2_000 + reasoning 2_500 ≤ buffer 3_000) stays within.
|
|
real := CostUSD(price, llm.Usage{PromptTokens: 1_000, CompletionTokens: 2_000, ReasoningTokens: 2_500})
|
|
if real > withBuf {
|
|
t.Fatalf("additive real cost %v exceeds buffered reservation %v", real, withBuf)
|
|
}
|
|
if real <= base {
|
|
t.Fatalf("without the buffer the reservation %v would be blind to the reasoning spend (real %v)", base, real)
|
|
}
|
|
}
|