111 lines
4.9 KiB
Go
111 lines
4.9 KiB
Go
//go:build live
|
|
|
|
package pipeline
|
|
|
|
// live_conformance_test.go: per-adapter LIVE conformance (§C, D12 Q3). This file is
|
|
// OUTSIDE the default test run — it is build-tagged `live`, so `go test ./...` never
|
|
// compiles or touches it and CI stays key-free and $0. Each call is REAL MONEY
|
|
// (pennies, but real). Run it manually on backend/.env keys:
|
|
//
|
|
// set -a; . ./.env; set +a
|
|
// TM_LIVE=1 go test -tags live -run TestLive -v ./internal/pipeline/
|
|
//
|
|
// It hits a live provider with a tiny dense-CJK→ru translation and asserts the
|
|
// ADAPTER produced a VALID translation — not an echo (the DeepSeek echo-mine, still
|
|
// disarmed on the real wire), not empty (reasoning ate the budget → read `content`,
|
|
// raise max_tokens), not a truncation — and logs the parsed usage + the response
|
|
// slug (canonicalization). It reuses the runner's own verdict machinery (classify),
|
|
// so "valid" here means exactly what the pipeline means by ok.
|
|
//
|
|
// Division of labour (D12 Q3): the Polygon Python oracle is the CROSS-provider
|
|
// acceptance GATE ("every adapter hits a live provider and gets a valid translation");
|
|
// these Go tests are the per-adapter wire+parse checks. BOTH — the oracle is the gate.
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"textmachine/backend/internal/config"
|
|
"textmachine/backend/internal/llm"
|
|
"textmachine/backend/internal/obs"
|
|
)
|
|
|
|
// liveModels are the boevoy cloud models to probe, each with a dense-CJK fragment
|
|
// that stresses exactly the silent-refusal failure modes (echo on dense CJK, empty
|
|
// on a starved reasoning budget). grok is commented out by default: it bills the
|
|
// clean-prod xAI account (the hygiene contract in models.yaml) — enable deliberately.
|
|
var liveModels = []struct {
|
|
model string
|
|
source string
|
|
}{
|
|
{"deepseek-v4-flash", "祝福是鲁迅创作的短篇小说，收录于《彷徨》。故事以第一人称叙述。"}, // the echo-mine repro family (Lu Xun 祝福)
|
|
{"glm-5", "静かな図書館の朝、彼女は古い本を開いて、ゆっくりと読み始めた。"},
|
|
{"kimi-k2.6", "这是一个用于验证适配器的测试句子，内容需要被完整翻译成俄语。"},
|
|
// {"grok-4.20-0309-non-reasoning", "这是一个测试句子。"}, // clean-prod xAI only
|
|
}
|
|
|
|
func TestLiveAdapterConformance(t *testing.T) {
|
|
if os.Getenv("TM_LIVE") != "1" {
|
|
t.Skip("live conformance is opt-in and spends real money: set TM_LIVE=1 with backend/.env loaded")
|
|
}
|
|
models, err := config.LoadModels("../../configs/models.yaml")
|
|
if err != nil {
|
|
t.Fatalf("load boevoy models.yaml: %v", err)
|
|
}
|
|
|
|
for _, tc := range liveModels {
|
|
t.Run(tc.model, func(t *testing.T) {
|
|
mod, ok := models.Models[tc.model]
|
|
if !ok {
|
|
t.Skipf("%s not defined in configs/models.yaml", tc.model)
|
|
}
|
|
if prov := models.Providers[mod.Provider]; prov.Kind != "local" && prov.APIKey() == "" {
|
|
t.Skipf("%s: provider key %s not set — skipping", tc.model, prov.APIKeyEnv)
|
|
}
|
|
|
|
client, err := BuildClient(models, tc.model, obs.NewLogger())
|
|
if err != nil {
|
|
t.Fatalf("build client: %v", err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
resp, err := client.Complete(ctx, llm.LLMRequest{
|
|
Model: tc.model,
|
|
Messages: []llm.Message{
|
|
{Role: "system", Content: "Ты профессиональный литературный переводчик. Переведи фрагмент на русский язык. Выведи ТОЛЬКО перевод, без комментариев.", CacheBoundary: true},
|
|
{Role: "user", Content: tc.source},
|
|
},
|
|
// ≥8000 so a hybrid-reasoning model's thinking does not empty `content`
|
|
// (the DeepSeek/Kimi finish=length-with-empty-content quirk).
|
|
MaxTokens: 8000,
|
|
Temperature: 0.3,
|
|
ReasoningEffort: "off",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("live call failed (adapter/transport): %v", err)
|
|
}
|
|
|
|
// Validate with the SAME verdict machine the runner uses: a valid
|
|
// translation is ok; an echo → cjk_artifact, empty → empty, refusal →
|
|
// soft_refusal. The response slug is logged for canonicalization review
|
|
// (Gemini returns models/…, xAI redirects retired slugs, etc.).
|
|
cls := classify(classifyInput{Source: tc.source, Output: resp.Text, Finish: resp.FinishReason, TargetLang: "ru"})
|
|
t.Logf("model=%q finish=%q usage=%+v cjk_share=%.2f verdict=%q text=%.120q",
|
|
resp.Model, resp.FinishReason, resp.Usage, cjkShare(resp.Text), cls.Reason, resp.Text)
|
|
|
|
if strings.TrimSpace(resp.Text) == "" {
|
|
t.Fatalf("%s returned EMPTY content (read `content`, raise max_tokens — reasoning likely ate the budget)", tc.model)
|
|
}
|
|
if !cls.ok() {
|
|
t.Fatalf("%s did not produce a valid translation: verdict=%s detail=%q", tc.model, cls.Reason, cls.Detail)
|
|
}
|
|
if resp.Model == "" {
|
|
t.Errorf("%s: empty response model slug — cannot canonicalize price (PriceForResponse would fall back)", tc.model)
|
|
}
|
|
})
|
|
}
|
|
}
|