233 lines
10 KiB
Go
233 lines
10 KiB
Go
package llm
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// systemmessages_test.go pins the SystemMessages capability axis — the fix for the silent loss of
|
||
// the memory-bank injection on an endpoint that carries one system message.
|
||
//
|
||
// ⚠ WHAT THESE TESTS MUST ASSERT, and why the obvious test is worthless: the pipeline has ALWAYS
|
||
// put both system messages in the request, and it does so on unfixed code too — the loss happens
|
||
// at the PROVIDER, not in our assembler. A test that checks "the injection is in the request" is
|
||
// therefore green before the fix and proves nothing. What separates fixed from unfixed is the
|
||
// POST-FIX invariant asserted below: on an endpoint declared single there is EXACTLY ONE system
|
||
// message on the wire and the injection is INSIDE it.
|
||
|
||
// systemRoles returns the roles of a decoded wire body's messages, in order.
|
||
func systemRoles(t *testing.T, body map[string]any) []string {
|
||
t.Helper()
|
||
raw, ok := body["messages"].([]any)
|
||
if !ok {
|
||
t.Fatalf("messages absent or not a list: %v", body["messages"])
|
||
}
|
||
roles := make([]string, 0, len(raw))
|
||
for _, m := range raw {
|
||
roles = append(roles, m.(map[string]any)["role"].(string))
|
||
}
|
||
return roles
|
||
}
|
||
|
||
func messageContent(t *testing.T, body map[string]any, i int) string {
|
||
t.Helper()
|
||
raw := body["messages"].([]any)
|
||
return raw[i].(map[string]any)["content"].(string)
|
||
}
|
||
|
||
// captureWire runs one Complete against a stub endpoint and returns the decoded request body.
|
||
func captureWire(t *testing.T, cap Capability, msgs []Message) map[string]any {
|
||
t.Helper()
|
||
var got map[string]any
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||
t.Errorf("decode request: %v", err)
|
||
}
|
||
openAIOK(t, w, `{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],
|
||
"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12}}`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewOpenAICompatClient(OpenAICompatConfig{
|
||
Name: "stub", BaseURL: srv.URL, APIKey: "k", Profile: fastProfile(), Cap: cap,
|
||
}, nil)
|
||
if _, err := c.Complete(context.Background(), LLMRequest{
|
||
Model: "m", Messages: msgs, MaxTokens: 4096, Temperature: 0.4,
|
||
}); err != nil {
|
||
t.Fatalf("Complete: %v", err)
|
||
}
|
||
return got
|
||
}
|
||
|
||
// bankRun is the message list the pipeline assembler actually produces for a chunk whose memory
|
||
// bank is non-empty (render.go MessagesWithInjection): stable system prefix, injection, user.
|
||
func bankRun() []Message {
|
||
return []Message{
|
||
{Role: "system", Content: "Переводи художественный текст с zh на ru.", CacheBoundary: true},
|
||
{Role: "system", Content: "ГЛОССАРИЙ: 方源 → Фан Юань"},
|
||
{Role: "user", Content: "исходный чанк"},
|
||
}
|
||
}
|
||
|
||
// TestSystemMessagesSingleCarriesTheInjection is THE post-fix invariant: one system message, and
|
||
// the glossary inside it. On unfixed code the wire carries two system messages and this fails.
|
||
func TestSystemMessagesSingleCarriesTheInjection(t *testing.T) {
|
||
body := captureWire(t, Capability{SystemMessages: SystemMessagesSingle}, bankRun())
|
||
|
||
roles := systemRoles(t, body)
|
||
nSystem := 0
|
||
for _, r := range roles {
|
||
if r == "system" {
|
||
nSystem++
|
||
}
|
||
}
|
||
if nSystem != 1 {
|
||
t.Fatalf("a single-system endpoint must receive EXACTLY ONE system message, got %d (roles %v)", nSystem, roles)
|
||
}
|
||
if want := []string{"system", "user"}; len(roles) != 2 || roles[0] != want[0] || roles[1] != want[1] {
|
||
t.Fatalf("roles = %v, want %v (the join must not move the user turn)", roles, want)
|
||
}
|
||
sys := messageContent(t, body, 0)
|
||
if !strings.Contains(sys, "ГЛОССАРИЙ: 方源 → Фан Юань") {
|
||
t.Fatalf("the memory-bank injection is NOT inside the single system message: %q", sys)
|
||
}
|
||
if !strings.Contains(sys, "Переводи художественный текст") {
|
||
t.Fatalf("the base prompt was lost by the join: %q", sys)
|
||
}
|
||
// Order is load-bearing: the stable prefix must stay first, or the prefix cache stops hitting.
|
||
if strings.Index(sys, "Переводи") > strings.Index(sys, "ГЛОССАРИЙ") {
|
||
t.Fatalf("the join reversed the stable prefix and the injection: %q", sys)
|
||
}
|
||
if want := "Переводи художественный текст с zh на ru.\n\nГЛОССАРИЙ: 方源 → Фан Юань"; sys != want {
|
||
t.Fatalf("joined system =\n%q\nwant\n%q", sys, want)
|
||
}
|
||
if got := messageContent(t, body, 1); got != "исходный чанк" {
|
||
t.Fatalf("user turn = %q", got)
|
||
}
|
||
}
|
||
|
||
// TestSystemMessagesMultiIsTheWireWeAlreadyShip is the control: an endpoint that declares nothing
|
||
// keeps the two-message wire byte-for-byte, so declaring the quirk on ONE provider cannot change
|
||
// what every other provider receives.
|
||
func TestSystemMessagesMultiIsTheWireWeAlreadyShip(t *testing.T) {
|
||
for _, cap := range []Capability{{}, {SystemMessages: SystemMessagesMulti}} {
|
||
body := captureWire(t, cap, bankRun())
|
||
roles := systemRoles(t, body)
|
||
if want := []string{"system", "system", "user"}; len(roles) != 3 ||
|
||
roles[0] != want[0] || roles[1] != want[1] || roles[2] != want[2] {
|
||
t.Fatalf("roles = %v, want %v", roles, want)
|
||
}
|
||
if got := messageContent(t, body, 0); got != "Переводи художественный текст с zh на ru." {
|
||
t.Fatalf("system[0] = %q", got)
|
||
}
|
||
if got := messageContent(t, body, 1); got != "ГЛОССАРИЙ: 方源 → Фан Юань" {
|
||
t.Fatalf("system[1] = %q", got)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestSystemMessagesSingleWithNoInjectionIsIdentical pins that the join is invisible when there is
|
||
// nothing to join — a bank-less chunk on a single-system endpoint ships exactly what it shipped.
|
||
func TestSystemMessagesSingleWithNoInjectionIsIdentical(t *testing.T) {
|
||
msgs := []Message{
|
||
{Role: "system", Content: "роль", CacheBoundary: true},
|
||
{Role: "user", Content: "чанк"},
|
||
}
|
||
body := captureWire(t, Capability{SystemMessages: SystemMessagesSingle}, msgs)
|
||
if roles := systemRoles(t, body); len(roles) != 2 || roles[0] != "system" || roles[1] != "user" {
|
||
t.Fatalf("roles = %v", roles)
|
||
}
|
||
if got := messageContent(t, body, 0); got != "роль" {
|
||
t.Fatalf("a lone system message must be untouched by the join, got %q", got)
|
||
}
|
||
}
|
||
|
||
// TestSystemMessagesSingleRefusesMidDialogueSystem: joining a system turn that sits AFTER a user
|
||
// turn would re-order the conversation, so it fails loud — the same refusal the Anthropic adapter
|
||
// makes for the same reason. Our assembler never produces one; this keeps that true by force.
|
||
func TestSystemMessagesSingleRefusesMidDialogueSystem(t *testing.T) {
|
||
_, err := toOpenAIMessages([]Message{
|
||
{Role: "system", Content: "роль"},
|
||
{Role: "user", Content: "чанк"},
|
||
{Role: "system", Content: "поздний системный"},
|
||
}, SystemMessagesSingle)
|
||
if err == nil {
|
||
t.Fatal("a system message after a non-system turn must be refused, not silently joined")
|
||
}
|
||
if !strings.Contains(err.Error(), "single system message") {
|
||
t.Fatalf("refusal must name the reason, got %v", err)
|
||
}
|
||
// The SAME list is legal on a multi endpoint: the refusal is the quirk's, not a new rule.
|
||
if _, err := toOpenAIMessages([]Message{
|
||
{Role: "system", Content: "роль"},
|
||
{Role: "user", Content: "чанк"},
|
||
{Role: "system", Content: "поздний системный"},
|
||
}, SystemMessagesMulti); err != nil {
|
||
t.Fatalf("multi endpoints keep their existing tolerance, got %v", err)
|
||
}
|
||
}
|
||
|
||
// TestSystemMessagesAxisCannotReachTheAnthropicPath is the ⛔ of the order: Anthropic turns the
|
||
// system prefix into separate blocks and CacheBoundary into a real cache_control, so a join there
|
||
// would destroy the cache contract. The axis must not be able to touch it — and structurally it
|
||
// cannot, because the Anthropic client is built without a Capability at all.
|
||
func TestSystemMessagesAxisCannotReachTheAnthropicPath(t *testing.T) {
|
||
var got map[string]any
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||
t.Errorf("decode: %v", err)
|
||
}
|
||
w.Write([]byte(`{"id":"m","model":"c","content":[{"type":"text","text":"ок"}],"stop_reason":"end_turn",
|
||
"usage":{"input_tokens":1,"output_tokens":1}}`))
|
||
}))
|
||
defer srv.Close()
|
||
|
||
c := NewAnthropicClient(AnthropicConfig{BaseURL: srv.URL, APIKey: "k", Profile: fastProfile(), CacheTTL: "5m"}, nil)
|
||
if _, err := c.Complete(context.Background(), LLMRequest{Model: "c", Messages: bankRun(), MaxTokens: 512}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
sys, ok := got["system"].([]any)
|
||
if !ok || len(sys) != 2 {
|
||
t.Fatalf("Anthropic must still receive TWO system blocks, got %v", got["system"])
|
||
}
|
||
first := sys[0].(map[string]any)
|
||
cc, ok := first["cache_control"].(map[string]any)
|
||
if !ok || cc["type"] != "ephemeral" || cc["ttl"] != "5m" {
|
||
t.Fatalf("the CacheBoundary block lost its live cache_control: %v", first)
|
||
}
|
||
if second := sys[1].(map[string]any); second["cache_control"] != nil {
|
||
t.Fatalf("the injection block must carry no cache_control: %v", second)
|
||
}
|
||
if msgs, ok := got["messages"].([]any); !ok || len(msgs) != 1 {
|
||
t.Fatalf("messages = %v", got["messages"])
|
||
}
|
||
}
|
||
|
||
// TestCapabilityWithoutTheAxisMarshalsUnchanged is the DETERMINISM guard. The resolved Capability
|
||
// is folded into the job snapshot, so a new field that marshalled for every model would move every
|
||
// snapshot id and re-buy every book. omitempty is what keeps that from happening; this pins it.
|
||
func TestCapabilityWithoutTheAxisMarshalsUnchanged(t *testing.T) {
|
||
data, err := json.Marshal(Capability{Budget: BudgetMaxTokens, Temp: TempSend, MinMaxTokens: 4000}.withDefaults())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if strings.Contains(string(data), "SystemMessages") {
|
||
t.Fatalf("a capability that does not declare the axis must not carry it into the snapshot: %s", data)
|
||
}
|
||
want := `{"Budget":"max_tokens","Temp":"send","TempValue":0,"Reasoning":{"Control":"none","OffEffort":"","OffExtraBody":null,"OnExtraBody":null},"MinMaxTokens":4000}`
|
||
if string(data) != want {
|
||
t.Fatalf("snapshot bytes drifted:\n got %s\nwant %s", data, want)
|
||
}
|
||
// Declaring it DOES move the bytes — that is the point: the flip is a loud --resnapshot.
|
||
declared, err := json.Marshal(Capability{SystemMessages: SystemMessagesSingle}.withDefaults())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(string(declared), `"SystemMessages":"single"`) {
|
||
t.Fatalf("a declared axis must reach the snapshot: %s", declared)
|
||
}
|
||
}
|